47 void rds_message_addref(struct rds_message *rm)  in rds_message_addref()  argument
49 	rdsdebug("addref rm %p ref %d\n", rm, atomic_read(&rm->m_refcount));  in rds_message_addref()
50 	atomic_inc(&rm->m_refcount);  in rds_message_addref()
57 static void rds_message_purge(struct rds_message *rm)  in rds_message_purge()  argument
61 	if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags)))  in rds_message_purge()
64 	for (i = 0; i < rm->data.op_nents; i++) {  in rds_message_purge()
65 		rdsdebug("putting data page %p\n", (void *)sg_page(&rm->data.op_sg[i]));  in rds_message_purge()
67 		__free_page(sg_page(&rm->data.op_sg[i]));  in rds_message_purge()
69 	rm->data.op_nents = 0;  in rds_message_purge()
71 	if (rm->rdma.op_active)  in rds_message_purge()
72 		rds_rdma_free_op(&rm->rdma);  in rds_message_purge()
73 	if (rm->rdma.op_rdma_mr)  in rds_message_purge()
74 		rds_mr_put(rm->rdma.op_rdma_mr);  in rds_message_purge()
76 	if (rm->atomic.op_active)  in rds_message_purge()
77 		rds_atomic_free_op(&rm->atomic);  in rds_message_purge()
78 	if (rm->atomic.op_rdma_mr)  in rds_message_purge()
79 		rds_mr_put(rm->atomic.op_rdma_mr);  in rds_message_purge()
82 void rds_message_put(struct rds_message *rm)  in rds_message_put()  argument
84 	rdsdebug("put rm %p ref %d\n", rm, atomic_read(&rm->m_refcount));  in rds_message_put()
85 	WARN(!atomic_read(&rm->m_refcount), "danger refcount zero on %p\n", rm);  in rds_message_put()
86 	if (atomic_dec_and_test(&rm->m_refcount)) {  in rds_message_put()
87 		BUG_ON(!list_empty(&rm->m_sock_item));  in rds_message_put()
88 		BUG_ON(!list_empty(&rm->m_conn_item));  in rds_message_put()
89 		rds_message_purge(rm);  in rds_message_put()
91 		kfree(rm);  in rds_message_put()
195 	struct rds_message *rm;  in rds_message_alloc()  local
200 	rm = kzalloc(sizeof(struct rds_message) + extra_len, gfp);  in rds_message_alloc()
201 	if (!rm)  in rds_message_alloc()
204 	rm->m_used_sgs = 0;  in rds_message_alloc()
205 	rm->m_total_sgs = extra_len / sizeof(struct scatterlist);  in rds_message_alloc()
207 	atomic_set(&rm->m_refcount, 1);  in rds_message_alloc()
208 	INIT_LIST_HEAD(&rm->m_sock_item);  in rds_message_alloc()
209 	INIT_LIST_HEAD(&rm->m_conn_item);  in rds_message_alloc()
210 	spin_lock_init(&rm->m_rs_lock);  in rds_message_alloc()
211 	init_waitqueue_head(&rm->m_flush_wait);  in rds_message_alloc()
214 	return rm;  in rds_message_alloc()
220 struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents)  in rds_message_alloc_sgs()  argument
222 	struct scatterlist *sg_first = (struct scatterlist *) &rm[1];  in rds_message_alloc_sgs()
225 	WARN_ON(rm->m_used_sgs + nents > rm->m_total_sgs);  in rds_message_alloc_sgs()
228 	if (rm->m_used_sgs + nents > rm->m_total_sgs)  in rds_message_alloc_sgs()
231 	sg_ret = &sg_first[rm->m_used_sgs];  in rds_message_alloc_sgs()
233 	rm->m_used_sgs += nents;  in rds_message_alloc_sgs()
240 	struct rds_message *rm;  in rds_message_map_pages()  local
245 	rm = rds_message_alloc(extra_bytes, GFP_NOWAIT);  in rds_message_map_pages()
246 	if (!rm)  in rds_message_map_pages()
249 	set_bit(RDS_MSG_PAGEVEC, &rm->m_flags);  in rds_message_map_pages()
250 	rm->m_inc.i_hdr.h_len = cpu_to_be32(total_len);  in rds_message_map_pages()
251 	rm->data.op_nents = ceil(total_len, PAGE_SIZE);  in rds_message_map_pages()
252 	rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs);  in rds_message_map_pages()
253 	if (!rm->data.op_sg) {  in rds_message_map_pages()
254 		rds_message_put(rm);  in rds_message_map_pages()
258 	for (i = 0; i < rm->data.op_nents; ++i) {  in rds_message_map_pages()
259 		sg_set_page(&rm->data.op_sg[i],  in rds_message_map_pages()
264 	return rm;  in rds_message_map_pages()
267 int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from)  in rds_message_copy_from_user()  argument
274 	rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));  in rds_message_copy_from_user()
279 	sg = rm->data.op_sg;  in rds_message_copy_from_user()
288 			rm->data.op_nents++;  in rds_message_copy_from_user()
312 	struct rds_message *rm;  in rds_message_inc_copy_to_user()  local
320 	rm = container_of(inc, struct rds_message, m_inc);  in rds_message_inc_copy_to_user()
321 	len = be32_to_cpu(rm->m_inc.i_hdr.h_len);  in rds_message_inc_copy_to_user()
323 	sg = rm->data.op_sg;  in rds_message_inc_copy_to_user()
354 void rds_message_wait(struct rds_message *rm)  in rds_message_wait()  argument
356 	wait_event_interruptible(rm->m_flush_wait,  in rds_message_wait()
357 			!test_bit(RDS_MSG_MAPPED, &rm->m_flags));  in rds_message_wait()
360 void rds_message_unmapped(struct rds_message *rm)  in rds_message_unmapped()  argument
362 	clear_bit(RDS_MSG_MAPPED, &rm->m_flags);  in rds_message_unmapped()
363 	wake_up_interruptible(&rm->m_flush_wait);  in rds_message_unmapped()