88 	struct rhashtable *ht;  member
174 	struct rhashtable *ht;  member
181 static inline unsigned long rht_marker(const struct rhashtable *ht, u32 hash)  in rht_marker()  argument
183 	return NULLS_MARKER(ht->p.nulls_base + hash);  in rht_marker()
186 #define INIT_RHT_NULLS_HEAD(ptr, ht, hash) \  argument
187 	((ptr) = (typeof(ptr)) rht_marker(ht, hash))
199 static inline void *rht_obj(const struct rhashtable *ht,  in rht_obj()  argument
202 	return (char *)he - ht->p.head_offset;  in rht_obj()
212 	struct rhashtable *ht, const struct bucket_table *tbl,  in rht_key_hashfn()  argument
219 		hash = ht->p.hashfn(key, ht->key_len, tbl->hash_rnd);  in rht_key_hashfn()
231 		unsigned int key_len = ht->p.key_len;  in rht_key_hashfn()
243 	struct rhashtable *ht, const struct bucket_table *tbl,  in rht_head_hashfn()  argument
246 	const char *ptr = rht_obj(ht, he);  in rht_head_hashfn()
250 							    ht->p.key_len,  in rht_head_hashfn()
252 	       rht_key_hashfn(ht, tbl, ptr + params.key_offset, params);  in rht_head_hashfn()
260 static inline bool rht_grow_above_75(const struct rhashtable *ht,  in rht_grow_above_75()  argument
264 	return atomic_read(&ht->nelems) > (tbl->size / 4 * 3) &&  in rht_grow_above_75()
265 	       (!ht->p.max_size || tbl->size < ht->p.max_size);  in rht_grow_above_75()
273 static inline bool rht_shrink_below_30(const struct rhashtable *ht,  in rht_shrink_below_30()  argument
277 	return atomic_read(&ht->nelems) < (tbl->size * 3 / 10) &&  in rht_shrink_below_30()
278 	       tbl->size > ht->p.min_size;  in rht_shrink_below_30()
286 static inline bool rht_grow_above_100(const struct rhashtable *ht,  in rht_grow_above_100()  argument
289 	return atomic_read(&ht->nelems) > tbl->size &&  in rht_grow_above_100()
290 		(!ht->p.max_size || tbl->size < ht->p.max_size);  in rht_grow_above_100()
298 static inline bool rht_grow_above_max(const struct rhashtable *ht,  in rht_grow_above_max()  argument
301 	return ht->p.insecure_max_entries &&  in rht_grow_above_max()
302 	       atomic_read(&ht->nelems) >= ht->p.insecure_max_entries;  in rht_grow_above_max()
325 int lockdep_rht_mutex_is_held(struct rhashtable *ht);
328 static inline int lockdep_rht_mutex_is_held(struct rhashtable *ht)  in lockdep_rht_mutex_is_held()  argument
340 int rhashtable_init(struct rhashtable *ht,
343 struct bucket_table *rhashtable_insert_slow(struct rhashtable *ht,
347 int rhashtable_insert_rehash(struct rhashtable *ht, struct bucket_table *tbl);
349 int rhashtable_walk_init(struct rhashtable *ht, struct rhashtable_iter *iter);
355 void rhashtable_free_and_destroy(struct rhashtable *ht,
358 void rhashtable_destroy(struct rhashtable *ht);
360 #define rht_dereference(p, ht) \  argument
361 	rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
363 #define rht_dereference_rcu(p, ht) \  argument
364 	rcu_dereference_check(p, lockdep_rht_mutex_is_held(ht))
511 	struct rhashtable *ht = arg->ht;  in rhashtable_compare()  local
514 	return memcmp(ptr + ht->p.key_offset, arg->key, ht->p.key_len);  in rhashtable_compare()
529 	struct rhashtable *ht, const void *key,  in rhashtable_lookup_fast()  argument
533 		.ht = ht,  in rhashtable_lookup_fast()
542 	tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_lookup_fast()
544 	hash = rht_key_hashfn(ht, tbl, key, params);  in rhashtable_lookup_fast()
547 		    params.obj_cmpfn(&arg, rht_obj(ht, he)) :  in rhashtable_lookup_fast()
548 		    rhashtable_compare(&arg, rht_obj(ht, he)))  in rhashtable_lookup_fast()
551 		return rht_obj(ht, he);  in rhashtable_lookup_fast()
557 	tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in rhashtable_lookup_fast()
567 	struct rhashtable *ht, const void *key, struct rhash_head *obj,  in __rhashtable_insert_fast()  argument
571 		.ht = ht,  in __rhashtable_insert_fast()
584 	tbl = rht_dereference_rcu(ht->tbl, ht);  in __rhashtable_insert_fast()
590 		hash = rht_head_hashfn(ht, tbl, obj, params);  in __rhashtable_insert_fast()
598 		tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in __rhashtable_insert_fast()
601 	new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in __rhashtable_insert_fast()
603 		tbl = rhashtable_insert_slow(ht, key, obj, new_tbl);  in __rhashtable_insert_fast()
612 	if (unlikely(rht_grow_above_max(ht, tbl)))  in __rhashtable_insert_fast()
615 	if (unlikely(rht_grow_above_100(ht, tbl))) {  in __rhashtable_insert_fast()
618 		err = rhashtable_insert_rehash(ht, tbl);  in __rhashtable_insert_fast()
627 	elasticity = ht->elasticity;  in __rhashtable_insert_fast()
631 			       params.obj_cmpfn(&arg, rht_obj(ht, head)) :  in __rhashtable_insert_fast()
632 			       rhashtable_compare(&arg, rht_obj(ht, head)))))  in __rhashtable_insert_fast()
646 	atomic_inc(&ht->nelems);  in __rhashtable_insert_fast()
647 	if (rht_grow_above_75(ht, tbl))  in __rhashtable_insert_fast()
648 		schedule_work(&ht->run_work);  in __rhashtable_insert_fast()
674 	struct rhashtable *ht, struct rhash_head *obj,  in rhashtable_insert_fast()  argument
677 	return __rhashtable_insert_fast(ht, NULL, obj, params);  in rhashtable_insert_fast()
702 	struct rhashtable *ht, struct rhash_head *obj,  in rhashtable_lookup_insert_fast()  argument
705 	const char *key = rht_obj(ht, obj);  in rhashtable_lookup_insert_fast()
707 	BUG_ON(ht->p.obj_hashfn);  in rhashtable_lookup_insert_fast()
709 	return __rhashtable_insert_fast(ht, key + ht->p.key_offset, obj,  in rhashtable_lookup_insert_fast()
736 	struct rhashtable *ht, const void *key, struct rhash_head *obj,  in rhashtable_lookup_insert_key()  argument
739 	BUG_ON(!ht->p.obj_hashfn || !key);  in rhashtable_lookup_insert_key()
741 	return __rhashtable_insert_fast(ht, key, obj, params);  in rhashtable_lookup_insert_key()
746 	struct rhashtable *ht, struct bucket_table *tbl,  in __rhashtable_remove_fast()  argument
755 	hash = rht_head_hashfn(ht, tbl, obj, params);  in __rhashtable_remove_fast()
793 	struct rhashtable *ht, struct rhash_head *obj,  in rhashtable_remove_fast()  argument
801 	tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_remove_fast()
808 	while ((err = __rhashtable_remove_fast(ht, tbl, obj, params)) &&  in rhashtable_remove_fast()
809 	       (tbl = rht_dereference_rcu(tbl->future_tbl, ht)))  in rhashtable_remove_fast()
815 	atomic_dec(&ht->nelems);  in rhashtable_remove_fast()
816 	if (unlikely(ht->p.automatic_shrinking &&  in rhashtable_remove_fast()
817 		     rht_shrink_below_30(ht, tbl)))  in rhashtable_remove_fast()
818 		schedule_work(&ht->run_work);  in rhashtable_remove_fast()