87 static void rds_ib_cache_xfer_to_ready(struct rds_ib_refill_cache *cache)  in rds_ib_cache_xfer_to_ready()  argument
91 	tmp = xchg(&cache->xfer, NULL);  in rds_ib_cache_xfer_to_ready()
93 		if (cache->ready)  in rds_ib_cache_xfer_to_ready()
94 			list_splice_entire_tail(tmp, cache->ready);  in rds_ib_cache_xfer_to_ready()
96 			cache->ready = tmp;  in rds_ib_cache_xfer_to_ready()
100 static int rds_ib_recv_alloc_cache(struct rds_ib_refill_cache *cache)  in rds_ib_recv_alloc_cache()  argument
105 	cache->percpu = alloc_percpu(struct rds_ib_cache_head);  in rds_ib_recv_alloc_cache()
106 	if (!cache->percpu)  in rds_ib_recv_alloc_cache()
110 		head = per_cpu_ptr(cache->percpu, cpu);  in rds_ib_recv_alloc_cache()
114 	cache->xfer = NULL;  in rds_ib_recv_alloc_cache()
115 	cache->ready = NULL;  in rds_ib_recv_alloc_cache()
134 static void rds_ib_cache_splice_all_lists(struct rds_ib_refill_cache *cache,  in rds_ib_cache_splice_all_lists()  argument
141 		head = per_cpu_ptr(cache->percpu, cpu);  in rds_ib_cache_splice_all_lists()
148 	if (cache->ready) {  in rds_ib_cache_splice_all_lists()
149 		list_splice_entire_tail(cache->ready, caller_list);  in rds_ib_cache_splice_all_lists()
150 		cache->ready = NULL;  in rds_ib_cache_splice_all_lists()
185 				  struct rds_ib_refill_cache *cache);
186 static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache);
465 				 struct rds_ib_refill_cache *cache)  in rds_ib_recv_cache_put()  argument
472 	chpfirst = __this_cpu_read(cache->percpu->first);  in rds_ib_recv_cache_put()
478 	__this_cpu_write(cache->percpu->first, new_item);  in rds_ib_recv_cache_put()
479 	__this_cpu_inc(cache->percpu->count);  in rds_ib_recv_cache_put()
481 	if (__this_cpu_read(cache->percpu->count) < RDS_IB_RECYCLE_BATCH_COUNT)  in rds_ib_recv_cache_put()
491 		old = xchg(&cache->xfer, NULL);  in rds_ib_recv_cache_put()
494 		old = cmpxchg(&cache->xfer, NULL, chpfirst);  in rds_ib_recv_cache_put()
498 	__this_cpu_write(cache->percpu->first, NULL);  in rds_ib_recv_cache_put()
499 	__this_cpu_write(cache->percpu->count, 0);  in rds_ib_recv_cache_put()
504 static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache)  in rds_ib_recv_cache_get()  argument
506 	struct list_head *head = cache->ready;  in rds_ib_recv_cache_get()
510 			cache->ready = head->next;  in rds_ib_recv_cache_get()
513 			cache->ready = NULL;  in rds_ib_recv_cache_get()