36 		       const struct bucket_table *tbl,  in head_hashfn()  argument
39 	return rht_head_hashfn(ht, tbl, he, ht->p);  in head_hashfn()
51 int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash)  in lockdep_rht_bucket_is_held()  argument
53 	spinlock_t *lock = rht_bucket_lock(tbl, hash);  in lockdep_rht_bucket_is_held()
63 static int alloc_bucket_locks(struct rhashtable *ht, struct bucket_table *tbl,  in alloc_bucket_locks()  argument
77 	size = min_t(unsigned int, size, tbl->size >> 1);  in alloc_bucket_locks()
83 			tbl->locks = vmalloc(size * sizeof(spinlock_t));  in alloc_bucket_locks()
86 		tbl->locks = kmalloc_array(size, sizeof(spinlock_t),  in alloc_bucket_locks()
88 		if (!tbl->locks)  in alloc_bucket_locks()
91 			spin_lock_init(&tbl->locks[i]);  in alloc_bucket_locks()
93 	tbl->locks_mask = size - 1;  in alloc_bucket_locks()
98 static void bucket_table_free(const struct bucket_table *tbl)  in bucket_table_free()  argument
100 	if (tbl)  in bucket_table_free()
101 		kvfree(tbl->locks);  in bucket_table_free()
103 	kvfree(tbl);  in bucket_table_free()
115 	struct bucket_table *tbl = NULL;  in bucket_table_alloc()  local
119 	size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]);  in bucket_table_alloc()
122 		tbl = kzalloc(size, gfp | __GFP_NOWARN | __GFP_NORETRY);  in bucket_table_alloc()
123 	if (tbl == NULL && gfp == GFP_KERNEL)  in bucket_table_alloc()
124 		tbl = vzalloc(size);  in bucket_table_alloc()
125 	if (tbl == NULL)  in bucket_table_alloc()
128 	tbl->size = nbuckets;  in bucket_table_alloc()
130 	if (alloc_bucket_locks(ht, tbl, gfp) < 0) {  in bucket_table_alloc()
131 		bucket_table_free(tbl);  in bucket_table_alloc()
135 	INIT_LIST_HEAD(&tbl->walkers);  in bucket_table_alloc()
137 	get_random_bytes(&tbl->hash_rnd, sizeof(tbl->hash_rnd));  in bucket_table_alloc()
140 		INIT_RHT_NULLS_HEAD(tbl->buckets[i], ht, i);  in bucket_table_alloc()
142 	return tbl;  in bucket_table_alloc()
146 						  struct bucket_table *tbl)  in rhashtable_last_table()  argument
151 		new_tbl = tbl;  in rhashtable_last_table()
152 		tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in rhashtable_last_table()
153 	} while (tbl);  in rhashtable_last_table()
160 	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_rehash_one()
207 	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_rehash_chain()
247 	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_rehash_table()
260 	rcu_assign_pointer(ht->tbl, new_tbl);  in rhashtable_rehash_table()
264 		walker->tbl = NULL;  in rhashtable_rehash_table()
293 	struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_expand()
329 	struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_shrink()
359 	struct bucket_table *tbl;  in rht_deferred_worker()  local
365 	tbl = rht_dereference(ht->tbl, ht);  in rht_deferred_worker()
366 	tbl = rhashtable_last_table(ht, tbl);  in rht_deferred_worker()
368 	if (rht_grow_above_75(ht, tbl))  in rht_deferred_worker()
370 	else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl))  in rht_deferred_worker()
382 					struct bucket_table *tbl,  in rhashtable_check_elasticity()  argument
388 	rht_for_each(head, tbl, hash)  in rhashtable_check_elasticity()
399 	struct bucket_table *tbl;  in rhashtable_insert_rehash()  local
403 	old_tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_insert_rehash()
404 	tbl = rhashtable_last_table(ht, old_tbl);  in rhashtable_insert_rehash()
406 	size = tbl->size;  in rhashtable_insert_rehash()
408 	if (rht_grow_above_75(ht, tbl))  in rhashtable_insert_rehash()
411 	else if (old_tbl != tbl)  in rhashtable_insert_rehash()
423 	err = rhashtable_rehash_attach(ht, tbl, new_tbl);  in rhashtable_insert_rehash()
437 			   struct bucket_table *tbl)  in rhashtable_insert_slow()  argument
443 	tbl = rhashtable_last_table(ht, tbl);  in rhashtable_insert_slow()
444 	hash = head_hashfn(ht, tbl, obj);  in rhashtable_insert_slow()
445 	spin_lock_nested(rht_bucket_lock(tbl, hash), SINGLE_DEPTH_NESTING);  in rhashtable_insert_slow()
452 	if (unlikely(rht_grow_above_max(ht, tbl)))  in rhashtable_insert_slow()
456 	if (rhashtable_check_elasticity(ht, tbl, hash) ||  in rhashtable_insert_slow()
457 	    rht_grow_above_100(ht, tbl))  in rhashtable_insert_slow()
462 	head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);  in rhashtable_insert_slow()
466 	rcu_assign_pointer(tbl->buckets[hash], obj);  in rhashtable_insert_slow()
471 	spin_unlock(rht_bucket_lock(tbl, hash));  in rhashtable_insert_slow()
510 	iter->walker->tbl =  in rhashtable_walk_init()
511 		rcu_dereference_protected(ht->tbl, lockdep_is_held(&ht->lock));  in rhashtable_walk_init()
512 	list_add(&iter->walker->list, &iter->walker->tbl->walkers);  in rhashtable_walk_init()
528 	if (iter->walker->tbl)  in rhashtable_walk_exit()
557 	if (iter->walker->tbl)  in rhashtable_walk_start()
561 	if (!iter->walker->tbl) {  in rhashtable_walk_start()
562 		iter->walker->tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_walk_start()
584 	struct bucket_table *tbl = iter->walker->tbl;  in rhashtable_walk_next()  local
590 		p = rht_dereference_bucket_rcu(p->next, tbl, iter->slot);  in rhashtable_walk_next()
594 	for (; iter->slot < tbl->size; iter->slot++) {  in rhashtable_walk_next()
597 		rht_for_each_rcu(p, tbl, iter->slot) {  in rhashtable_walk_next()
619 	iter->walker->tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in rhashtable_walk_next()
620 	if (iter->walker->tbl) {  in rhashtable_walk_next()
642 	struct bucket_table *tbl = iter->walker->tbl;  in rhashtable_walk_stop()  local
644 	if (!tbl)  in rhashtable_walk_stop()
650 	if (tbl->rehash < tbl->size)  in rhashtable_walk_stop()
651 		list_add(&iter->walker->list, &tbl->walkers);  in rhashtable_walk_stop()
653 		iter->walker->tbl = NULL;  in rhashtable_walk_stop()
720 	struct bucket_table *tbl;  in rhashtable_init()  local
784 	tbl = bucket_table_alloc(ht, size, GFP_KERNEL);  in rhashtable_init()
785 	if (tbl == NULL)  in rhashtable_init()
790 	RCU_INIT_POINTER(ht->tbl, tbl);  in rhashtable_init()
817 	const struct bucket_table *tbl;  in rhashtable_free_and_destroy()  local
823 	tbl = rht_dereference(ht->tbl, ht);  in rhashtable_free_and_destroy()
825 		for (i = 0; i < tbl->size; i++) {  in rhashtable_free_and_destroy()
828 			for (pos = rht_dereference(tbl->buckets[i], ht),  in rhashtable_free_and_destroy()
839 	bucket_table_free(tbl);  in rhashtable_free_and_destroy()