144 	struct bucket_table __rcu	*tbl;  member
161 	struct bucket_table *tbl;  member
204 static inline unsigned int rht_bucket_index(const struct bucket_table *tbl,  in rht_bucket_index()  argument
207 	return (hash >> RHT_HASH_RESERVED_SPACE) & (tbl->size - 1);  in rht_bucket_index()
211 	struct rhashtable *ht, const struct bucket_table *tbl,  in rht_key_hashfn()  argument
218 		hash = ht->p.hashfn(key, ht->key_len, tbl->hash_rnd);  in rht_key_hashfn()
223 			hash = params.hashfn(key, key_len, tbl->hash_rnd);  in rht_key_hashfn()
225 			hash = jhash(key, key_len, tbl->hash_rnd);  in rht_key_hashfn()
228 				      tbl->hash_rnd);  in rht_key_hashfn()
233 			hash = params.hashfn(key, key_len, tbl->hash_rnd);  in rht_key_hashfn()
235 			hash = jhash(key, key_len, tbl->hash_rnd);  in rht_key_hashfn()
238 	return rht_bucket_index(tbl, hash);  in rht_key_hashfn()
242 	struct rhashtable *ht, const struct bucket_table *tbl,  in rht_head_hashfn()  argument
248 	       rht_bucket_index(tbl, params.obj_hashfn(ptr, params.key_len ?:  in rht_head_hashfn()
250 						       tbl->hash_rnd)) :  in rht_head_hashfn()
251 	       rht_key_hashfn(ht, tbl, ptr + params.key_offset, params);  in rht_head_hashfn()
260 				     const struct bucket_table *tbl)  in rht_grow_above_75()  argument
263 	return atomic_read(&ht->nelems) > (tbl->size / 4 * 3) &&  in rht_grow_above_75()
264 	       (!ht->p.max_size || tbl->size < ht->p.max_size);  in rht_grow_above_75()
273 				       const struct bucket_table *tbl)  in rht_shrink_below_30()  argument
276 	return atomic_read(&ht->nelems) < (tbl->size * 3 / 10) &&  in rht_shrink_below_30()
277 	       tbl->size > ht->p.min_size;  in rht_shrink_below_30()
286 				      const struct bucket_table *tbl)  in rht_grow_above_100()  argument
288 	return atomic_read(&ht->nelems) > tbl->size &&  in rht_grow_above_100()
289 		(!ht->p.max_size || tbl->size < ht->p.max_size);  in rht_grow_above_100()
298 				      const struct bucket_table *tbl)  in rht_grow_above_max()  argument
317 static inline spinlock_t *rht_bucket_lock(const struct bucket_table *tbl,  in rht_bucket_lock()  argument
320 	return &tbl->locks[hash & tbl->locks_mask];  in rht_bucket_lock()
325 int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash);
332 static inline int lockdep_rht_bucket_is_held(const struct bucket_table *tbl,  in lockdep_rht_bucket_is_held()  argument
364 #define rht_dereference_bucket(p, tbl, hash) \  argument
365 	rcu_dereference_protected(p, lockdep_rht_bucket_is_held(tbl, hash))
367 #define rht_dereference_bucket_rcu(p, tbl, hash) \  argument
368 	rcu_dereference_check(p, lockdep_rht_bucket_is_held(tbl, hash))
380 #define rht_for_each_continue(pos, head, tbl, hash) \  argument
381 	for (pos = rht_dereference_bucket(head, tbl, hash); \
383 	     pos = rht_dereference_bucket((pos)->next, tbl, hash))
391 #define rht_for_each(pos, tbl, hash) \  argument
392 	rht_for_each_continue(pos, (tbl)->buckets[hash], tbl, hash)
403 #define rht_for_each_entry_continue(tpos, pos, head, tbl, hash, member)	\  argument
404 	for (pos = rht_dereference_bucket(head, tbl, hash);		\
406 	     pos = rht_dereference_bucket((pos)->next, tbl, hash))
416 #define rht_for_each_entry(tpos, pos, tbl, hash, member)		\  argument
417 	rht_for_each_entry_continue(tpos, pos, (tbl)->buckets[hash],	\
418 				    tbl, hash, member)
432 #define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member)	    \  argument
433 	for (pos = rht_dereference_bucket((tbl)->buckets[hash], tbl, hash), \
435 		       rht_dereference_bucket(pos->next, tbl, hash) : NULL; \
439 		       rht_dereference_bucket(pos->next, tbl, hash) : NULL)
452 #define rht_for_each_rcu_continue(pos, head, tbl, hash)			\  argument
454 	     pos = rht_dereference_bucket_rcu(head, tbl, hash);		\
468 #define rht_for_each_rcu(pos, tbl, hash)				\  argument
469 	rht_for_each_rcu_continue(pos, (tbl)->buckets[hash], tbl, hash)
484 #define rht_for_each_entry_rcu_continue(tpos, pos, head, tbl, hash, member) \  argument
486 	     pos = rht_dereference_bucket_rcu(head, tbl, hash);		    \
488 	     pos = rht_dereference_bucket_rcu(pos->next, tbl, hash))
502 #define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member)		\  argument
503 	rht_for_each_entry_rcu_continue(tpos, pos, (tbl)->buckets[hash],\
504 					tbl, hash, member)
534 	const struct bucket_table *tbl;  in rhashtable_lookup_fast()  local
540 	tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_lookup_fast()
542 	hash = rht_key_hashfn(ht, tbl, key, params);  in rhashtable_lookup_fast()
543 	rht_for_each_rcu(he, tbl, hash) {  in rhashtable_lookup_fast()
555 	tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in rhashtable_lookup_fast()
556 	if (unlikely(tbl))  in rhashtable_lookup_fast()
572 	struct bucket_table *tbl, *new_tbl;  in __rhashtable_insert_fast()  local
582 	tbl = rht_dereference_rcu(ht->tbl, ht);  in __rhashtable_insert_fast()
588 		hash = rht_head_hashfn(ht, tbl, obj, params);  in __rhashtable_insert_fast()
589 		lock = rht_bucket_lock(tbl, hash);  in __rhashtable_insert_fast()
592 		if (tbl->rehash <= hash)  in __rhashtable_insert_fast()
596 		tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in __rhashtable_insert_fast()
599 	new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in __rhashtable_insert_fast()
608 	if (unlikely(rht_grow_above_max(ht, tbl)))  in __rhashtable_insert_fast()
611 	if (unlikely(rht_grow_above_100(ht, tbl))) {  in __rhashtable_insert_fast()
624 	rht_for_each(head, tbl, hash) {  in __rhashtable_insert_fast()
636 	head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);  in __rhashtable_insert_fast()
640 	rcu_assign_pointer(tbl->buckets[hash], obj);  in __rhashtable_insert_fast()
643 	if (rht_grow_above_75(ht, tbl))  in __rhashtable_insert_fast()
742 	struct rhashtable *ht, struct bucket_table *tbl,  in __rhashtable_remove_fast()  argument
751 	hash = rht_head_hashfn(ht, tbl, obj, params);  in __rhashtable_remove_fast()
752 	lock = rht_bucket_lock(tbl, hash);  in __rhashtable_remove_fast()
756 	pprev = &tbl->buckets[hash];  in __rhashtable_remove_fast()
757 	rht_for_each(he, tbl, hash) {  in __rhashtable_remove_fast()
792 	struct bucket_table *tbl;  in rhashtable_remove_fast()  local
797 	tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_remove_fast()
804 	while ((err = __rhashtable_remove_fast(ht, tbl, obj, params)) &&  in rhashtable_remove_fast()
805 	       (tbl = rht_dereference_rcu(tbl->future_tbl, ht)))  in rhashtable_remove_fast()
813 		     rht_shrink_below_30(ht, tbl)))  in rhashtable_remove_fast()