Lines Matching refs:s
63 unsigned int kmem_cache_size(struct kmem_cache *s) in kmem_cache_size() argument
65 return s->object_size; in kmem_cache_size()
72 struct kmem_cache *s = NULL; in kmem_cache_sanity_check() local
80 list_for_each_entry(s, &slab_caches, list) { in kmem_cache_sanity_check()
89 res = probe_kernel_address(s->name, tmp); in kmem_cache_sanity_check()
92 s->object_size); in kmem_cache_sanity_check()
107 void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p) in __kmem_cache_free_bulk() argument
112 kmem_cache_free(s, p[i]); in __kmem_cache_free_bulk()
115 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, in __kmem_cache_alloc_bulk() argument
121 void *x = p[i] = kmem_cache_alloc(s, flags); in __kmem_cache_alloc_bulk()
123 __kmem_cache_free_bulk(s, i, p); in __kmem_cache_alloc_bulk()
131 void slab_init_memcg_params(struct kmem_cache *s) in slab_init_memcg_params() argument
133 s->memcg_params.is_root_cache = true; in slab_init_memcg_params()
134 INIT_LIST_HEAD(&s->memcg_params.list); in slab_init_memcg_params()
135 RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL); in slab_init_memcg_params()
138 static int init_memcg_params(struct kmem_cache *s, in init_memcg_params() argument
144 s->memcg_params.is_root_cache = false; in init_memcg_params()
145 s->memcg_params.memcg = memcg; in init_memcg_params()
146 s->memcg_params.root_cache = root_cache; in init_memcg_params()
150 slab_init_memcg_params(s); in init_memcg_params()
161 RCU_INIT_POINTER(s->memcg_params.memcg_caches, arr); in init_memcg_params()
165 static void destroy_memcg_params(struct kmem_cache *s) in destroy_memcg_params() argument
167 if (is_root_cache(s)) in destroy_memcg_params()
168 kfree(rcu_access_pointer(s->memcg_params.memcg_caches)); in destroy_memcg_params()
171 static int update_memcg_params(struct kmem_cache *s, int new_array_size) in update_memcg_params() argument
175 if (!is_root_cache(s)) in update_memcg_params()
183 old = rcu_dereference_protected(s->memcg_params.memcg_caches, in update_memcg_params()
189 rcu_assign_pointer(s->memcg_params.memcg_caches, new); in update_memcg_params()
197 struct kmem_cache *s; in memcg_update_all_caches() local
201 list_for_each_entry(s, &slab_caches, list) { in memcg_update_all_caches()
202 ret = update_memcg_params(s, num_memcgs); in memcg_update_all_caches()
214 static inline int init_memcg_params(struct kmem_cache *s, in init_memcg_params() argument
220 static inline void destroy_memcg_params(struct kmem_cache *s) in destroy_memcg_params() argument
228 int slab_unmergeable(struct kmem_cache *s) in slab_unmergeable() argument
230 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE)) in slab_unmergeable()
233 if (!is_root_cache(s)) in slab_unmergeable()
236 if (s->ctor) in slab_unmergeable()
242 if (s->refcount < 0) in slab_unmergeable()
251 struct kmem_cache *s; in find_mergeable() local
264 list_for_each_entry_reverse(s, &slab_caches, list) { in find_mergeable()
265 if (slab_unmergeable(s)) in find_mergeable()
268 if (size > s->size) in find_mergeable()
271 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME)) in find_mergeable()
277 if ((s->size & ~(align - 1)) != s->size) in find_mergeable()
280 if (s->size - size >= sizeof(void *)) in find_mergeable()
284 (align > s->align || s->align % align)) in find_mergeable()
287 return s; in find_mergeable()
324 struct kmem_cache *s; in create_cache() local
328 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); in create_cache()
329 if (!s) in create_cache()
332 s->name = name; in create_cache()
333 s->object_size = object_size; in create_cache()
334 s->size = size; in create_cache()
335 s->align = align; in create_cache()
336 s->ctor = ctor; in create_cache()
338 err = init_memcg_params(s, memcg, root_cache); in create_cache()
342 err = __kmem_cache_create(s, flags); in create_cache()
346 s->refcount = 1; in create_cache()
347 list_add(&s->list, &slab_caches); in create_cache()
351 return s; in create_cache()
354 destroy_memcg_params(s); in create_cache()
355 kmem_cache_free(kmem_cache, s); in create_cache()
387 struct kmem_cache *s = NULL; in kmem_cache_create() local
410 s = __kmem_cache_alias(name, size, align, flags, ctor); in kmem_cache_create()
411 if (s) in kmem_cache_create()
420 s = create_cache(cache_name, size, size, in kmem_cache_create()
423 if (IS_ERR(s)) { in kmem_cache_create()
424 err = PTR_ERR(s); in kmem_cache_create()
446 return s; in kmem_cache_create()
450 static int shutdown_cache(struct kmem_cache *s, in shutdown_cache() argument
453 if (__kmem_cache_shutdown(s) != 0) in shutdown_cache()
456 if (s->flags & SLAB_DESTROY_BY_RCU) in shutdown_cache()
459 list_move(&s->list, release); in shutdown_cache()
465 struct kmem_cache *s, *s2; in release_caches() local
470 list_for_each_entry_safe(s, s2, release, list) { in release_caches()
472 sysfs_slab_remove(s); in release_caches()
474 slab_kmem_cache_release(s); in release_caches()
495 struct kmem_cache *s = NULL; in memcg_create_kmem_cache() local
529 s = create_cache(cache_name, root_cache->object_size, in memcg_create_kmem_cache()
538 if (IS_ERR(s)) { in memcg_create_kmem_cache()
543 list_add(&s->memcg_params.list, &root_cache->memcg_params.list); in memcg_create_kmem_cache()
551 arr->entries[idx] = s; in memcg_create_kmem_cache()
564 struct kmem_cache *s, *c; in memcg_deactivate_kmem_caches() local
572 list_for_each_entry(s, &slab_caches, list) { in memcg_deactivate_kmem_caches()
573 if (!is_root_cache(s)) in memcg_deactivate_kmem_caches()
576 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, in memcg_deactivate_kmem_caches()
591 static int __shutdown_memcg_cache(struct kmem_cache *s, in __shutdown_memcg_cache() argument
594 BUG_ON(is_root_cache(s)); in __shutdown_memcg_cache()
596 if (shutdown_cache(s, release, need_rcu_barrier)) in __shutdown_memcg_cache()
599 list_del(&s->memcg_params.list); in __shutdown_memcg_cache()
607 struct kmem_cache *s, *s2; in memcg_destroy_kmem_caches() local
613 list_for_each_entry_safe(s, s2, &slab_caches, list) { in memcg_destroy_kmem_caches()
614 if (is_root_cache(s) || s->memcg_params.memcg != memcg) in memcg_destroy_kmem_caches()
620 BUG_ON(__shutdown_memcg_cache(s, &release, &need_rcu_barrier)); in memcg_destroy_kmem_caches()
630 static int shutdown_memcg_caches(struct kmem_cache *s, in shutdown_memcg_caches() argument
638 BUG_ON(!is_root_cache(s)); in shutdown_memcg_caches()
644 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, in shutdown_memcg_caches()
671 list_for_each_entry_safe(c, c2, &s->memcg_params.list, in shutdown_memcg_caches()
675 list_splice(&busy, &s->memcg_params.list); in shutdown_memcg_caches()
681 if (!list_empty(&s->memcg_params.list)) in shutdown_memcg_caches()
686 static inline int shutdown_memcg_caches(struct kmem_cache *s, in shutdown_memcg_caches() argument
693 void slab_kmem_cache_release(struct kmem_cache *s) in slab_kmem_cache_release() argument
695 destroy_memcg_params(s); in slab_kmem_cache_release()
696 kfree_const(s->name); in slab_kmem_cache_release()
697 kmem_cache_free(kmem_cache, s); in slab_kmem_cache_release()
700 void kmem_cache_destroy(struct kmem_cache *s) in kmem_cache_destroy() argument
706 if (unlikely(!s)) in kmem_cache_destroy()
714 s->refcount--; in kmem_cache_destroy()
715 if (s->refcount) in kmem_cache_destroy()
718 err = shutdown_memcg_caches(s, &release, &need_rcu_barrier); in kmem_cache_destroy()
720 err = shutdown_cache(s, &release, &need_rcu_barrier); in kmem_cache_destroy()
724 "Slab cache still has objects\n", s->name); in kmem_cache_destroy()
764 void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size, in create_boot_cache() argument
769 s->name = name; in create_boot_cache()
770 s->size = s->object_size = size; in create_boot_cache()
771 s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size); in create_boot_cache()
773 slab_init_memcg_params(s); in create_boot_cache()
775 err = __kmem_cache_create(s, flags); in create_boot_cache()
781 s->refcount = -1; /* Exempt from merging for now */ in create_boot_cache()
787 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); in create_kmalloc_cache() local
789 if (!s) in create_kmalloc_cache()
792 create_boot_cache(s, name, size, flags); in create_kmalloc_cache()
793 list_add(&s->list, &slab_caches); in create_kmalloc_cache()
794 s->refcount = 1; in create_kmalloc_cache()
795 return s; in create_kmalloc_cache()
980 struct kmem_cache *s = kmalloc_caches[i]; in create_kmalloc_caches() local
982 if (s) { in create_kmalloc_caches()
1073 memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info) in memcg_accumulate_slabinfo() argument
1078 if (!is_root_cache(s)) in memcg_accumulate_slabinfo()
1081 for_each_memcg_cache(c, s) { in memcg_accumulate_slabinfo()
1093 static void cache_show(struct kmem_cache *s, struct seq_file *m) in cache_show() argument
1098 get_slabinfo(s, &sinfo); in cache_show()
1100 memcg_accumulate_slabinfo(s, &sinfo); in cache_show()
1103 cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size, in cache_show()
1110 slabinfo_show_stats(m, s); in cache_show()
1116 struct kmem_cache *s = list_entry(p, struct kmem_cache, list); in slab_show() local
1120 if (is_root_cache(s)) in slab_show()
1121 cache_show(s, m); in slab_show()
1128 struct kmem_cache *s = list_entry(p, struct kmem_cache, list); in memcg_slab_show() local
1133 if (!is_root_cache(s) && s->memcg_params.memcg == memcg) in memcg_slab_show()
1134 cache_show(s, m); in memcg_slab_show()