Lines Matching refs:cachep
247 static void free_block(struct kmem_cache *cachep, void **objpp, int len,
249 static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
250 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
270 #define MAKE_LIST(cachep, listp, slab, nodeid) \ argument
273 list_splice(&get_node(cachep, nodeid)->slab, listp); \
276 #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ argument
278 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
279 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
280 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
355 static int obj_offset(struct kmem_cache *cachep) in obj_offset() argument
357 return cachep->obj_offset; in obj_offset()
360 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) in dbg_redzone1() argument
362 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); in dbg_redzone1()
363 return (unsigned long long*) (objp + obj_offset(cachep) - in dbg_redzone1()
367 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) in dbg_redzone2() argument
369 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); in dbg_redzone2()
370 if (cachep->flags & SLAB_STORE_USER) in dbg_redzone2()
371 return (unsigned long long *)(objp + cachep->size - in dbg_redzone2()
374 return (unsigned long long *) (objp + cachep->size - in dbg_redzone2()
378 static void **dbg_userword(struct kmem_cache *cachep, void *objp) in dbg_userword() argument
380 BUG_ON(!(cachep->flags & SLAB_STORE_USER)); in dbg_userword()
381 return (void **)(objp + cachep->size - BYTES_PER_WORD); in dbg_userword()
387 #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
388 #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
389 #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) argument
402 struct kmem_cache *cachep = page->slab_cache; in set_obj_status() local
404 freelist_size = cachep->num * sizeof(freelist_idx_t); in set_obj_status()
413 struct kmem_cache *cachep = page->slab_cache; in get_obj_status() local
415 freelist_size = cachep->num * sizeof(freelist_idx_t); in get_obj_status()
473 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) in cpu_cache_get() argument
475 return this_cpu_ptr(cachep->cpu_cache); in cpu_cache_get()
563 #define slab_error(cachep, msg) __slab_error(__func__, cachep, msg) argument
565 static void __slab_error(const char *function, struct kmem_cache *cachep, in __slab_error() argument
569 function, cachep->name, msg); in __slab_error()
696 static void recheck_pfmemalloc_active(struct kmem_cache *cachep, in recheck_pfmemalloc_active() argument
699 struct kmem_cache_node *n = get_node(cachep, numa_mem_id()); in recheck_pfmemalloc_active()
724 static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, in __ac_get_obj() argument
754 n = get_node(cachep, numa_mem_id()); in __ac_get_obj()
759 recheck_pfmemalloc_active(cachep, ac); in __ac_get_obj()
771 static inline void *ac_get_obj(struct kmem_cache *cachep, in ac_get_obj() argument
777 objp = __ac_get_obj(cachep, ac, flags, force_refill); in ac_get_obj()
784 static noinline void *__ac_put_obj(struct kmem_cache *cachep, in __ac_put_obj() argument
797 static inline void ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac, in ac_put_obj() argument
801 objp = __ac_put_obj(cachep, ac, objp); in ac_put_obj()
831 #define drain_alien_cache(cachep, alien) do { } while (0) argument
832 #define reap_alien(cachep, n) do { } while (0) argument
844 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
849 static inline void *alternate_node_alloc(struct kmem_cache *cachep, in alternate_node_alloc() argument
855 static inline void *____cache_alloc_node(struct kmem_cache *cachep, in ____cache_alloc_node() argument
920 static void __drain_alien_cache(struct kmem_cache *cachep, in __drain_alien_cache() argument
924 struct kmem_cache_node *n = get_node(cachep, node); in __drain_alien_cache()
936 free_block(cachep, ac->entry, ac->avail, node, list); in __drain_alien_cache()
945 static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n) in reap_alien() argument
958 __drain_alien_cache(cachep, ac, node, &list); in reap_alien()
960 slabs_destroy(cachep, &list); in reap_alien()
966 static void drain_alien_cache(struct kmem_cache *cachep, in drain_alien_cache() argument
981 __drain_alien_cache(cachep, ac, i, &list); in drain_alien_cache()
983 slabs_destroy(cachep, &list); in drain_alien_cache()
988 static int __cache_free_alien(struct kmem_cache *cachep, void *objp, in __cache_free_alien() argument
996 n = get_node(cachep, node); in __cache_free_alien()
997 STATS_INC_NODEFREES(cachep); in __cache_free_alien()
1003 STATS_INC_ACOVERFLOW(cachep); in __cache_free_alien()
1004 __drain_alien_cache(cachep, ac, page_node, &list); in __cache_free_alien()
1006 ac_put_obj(cachep, ac, objp); in __cache_free_alien()
1008 slabs_destroy(cachep, &list); in __cache_free_alien()
1010 n = get_node(cachep, page_node); in __cache_free_alien()
1012 free_block(cachep, &objp, 1, page_node, &list); in __cache_free_alien()
1014 slabs_destroy(cachep, &list); in __cache_free_alien()
1019 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
1030 return __cache_free_alien(cachep, objp, node, page_node); in cache_free_alien()
1054 struct kmem_cache *cachep; in init_cache_node_node() local
1058 list_for_each_entry(cachep, &slab_caches, list) { in init_cache_node_node()
1064 n = get_node(cachep, node); in init_cache_node_node()
1071 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in init_cache_node_node()
1078 cachep->node[node] = n; in init_cache_node_node()
1084 cachep->batchcount + cachep->num; in init_cache_node_node()
1090 static inline int slabs_tofree(struct kmem_cache *cachep, in slabs_tofree() argument
1093 return (n->free_objects + cachep->num - 1) / cachep->num; in slabs_tofree()
1098 struct kmem_cache *cachep; in cpuup_canceled() local
1103 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_canceled()
1109 n = get_node(cachep, node); in cpuup_canceled()
1116 n->free_limit -= cachep->batchcount; in cpuup_canceled()
1119 nc = per_cpu_ptr(cachep->cpu_cache, cpu); in cpuup_canceled()
1121 free_block(cachep, nc->entry, nc->avail, node, &list); in cpuup_canceled()
1132 free_block(cachep, shared->entry, in cpuup_canceled()
1144 drain_alien_cache(cachep, alien); in cpuup_canceled()
1149 slabs_destroy(cachep, &list); in cpuup_canceled()
1156 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_canceled()
1157 n = get_node(cachep, node); in cpuup_canceled()
1160 drain_freelist(cachep, n, slabs_tofree(cachep, n)); in cpuup_canceled()
1166 struct kmem_cache *cachep; in cpuup_prepare() local
1185 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_prepare()
1189 if (cachep->shared) { in cpuup_prepare()
1191 cachep->shared * cachep->batchcount, in cpuup_prepare()
1197 alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL); in cpuup_prepare()
1203 n = get_node(cachep, node); in cpuup_prepare()
1302 struct kmem_cache *cachep; in drain_cache_node_node() local
1305 list_for_each_entry(cachep, &slab_caches, list) { in drain_cache_node_node()
1308 n = get_node(cachep, node); in drain_cache_node_node()
1312 drain_freelist(cachep, n, slabs_tofree(cachep, n)); in drain_cache_node_node()
1359 static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list, in init_list() argument
1373 MAKE_ALL_LISTS(cachep, ptr, nodeid); in init_list()
1374 cachep->node[nodeid] = ptr; in init_list()
1381 static void __init set_up_node(struct kmem_cache *cachep, int index) in set_up_node() argument
1386 cachep->node[node] = &init_kmem_cache_node[index + node]; in set_up_node()
1387 cachep->node[node]->next_reap = jiffies + in set_up_node()
1389 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in set_up_node()
1479 struct kmem_cache *cachep; in kmem_cache_init_late() local
1485 list_for_each_entry(cachep, &slab_caches, list) in kmem_cache_init_late()
1486 if (enable_cpucache(cachep, GFP_NOWAIT)) in kmem_cache_init_late()
1530 slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) in slab_out_of_memory() argument
1547 cachep->name, cachep->size, cachep->gfporder); in slab_out_of_memory()
1549 for_each_kmem_cache_node(cachep, node, n) { in slab_out_of_memory()
1555 active_objs += cachep->num; in slab_out_of_memory()
1569 num_objs = num_slabs * cachep->num; in slab_out_of_memory()
1586 static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, in kmem_getpages() argument
1592 flags |= cachep->allocflags; in kmem_getpages()
1593 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) in kmem_getpages()
1596 page = __alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder); in kmem_getpages()
1598 slab_out_of_memory(cachep, flags, nodeid); in kmem_getpages()
1602 if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) { in kmem_getpages()
1603 __free_pages(page, cachep->gfporder); in kmem_getpages()
1611 nr_pages = (1 << cachep->gfporder); in kmem_getpages()
1612 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) in kmem_getpages()
1622 if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) { in kmem_getpages()
1623 kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid); in kmem_getpages()
1625 if (cachep->ctor) in kmem_getpages()
1637 static void kmem_freepages(struct kmem_cache *cachep, struct page *page) in kmem_freepages() argument
1639 const unsigned long nr_freed = (1 << cachep->gfporder); in kmem_freepages()
1641 kmemcheck_free_shadow(page, cachep->gfporder); in kmem_freepages()
1643 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) in kmem_freepages()
1658 __free_kmem_pages(page, cachep->gfporder); in kmem_freepages()
1663 struct kmem_cache *cachep; in kmem_rcu_free() local
1667 cachep = page->slab_cache; in kmem_rcu_free()
1669 kmem_freepages(cachep, page); in kmem_rcu_free()
1675 static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr, in store_stackinfo() argument
1678 int size = cachep->object_size; in store_stackinfo()
1680 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)]; in store_stackinfo()
1708 static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val) in poison_obj() argument
1710 int size = cachep->object_size; in poison_obj()
1711 addr = &((char *)addr)[obj_offset(cachep)]; in poison_obj()
1751 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) in print_objinfo() argument
1756 if (cachep->flags & SLAB_RED_ZONE) { in print_objinfo()
1758 *dbg_redzone1(cachep, objp), in print_objinfo()
1759 *dbg_redzone2(cachep, objp)); in print_objinfo()
1762 if (cachep->flags & SLAB_STORE_USER) { in print_objinfo()
1764 *dbg_userword(cachep, objp), in print_objinfo()
1765 *dbg_userword(cachep, objp)); in print_objinfo()
1767 realobj = (char *)objp + obj_offset(cachep); in print_objinfo()
1768 size = cachep->object_size; in print_objinfo()
1778 static void check_poison_obj(struct kmem_cache *cachep, void *objp) in check_poison_obj() argument
1784 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1785 size = cachep->object_size; in check_poison_obj()
1798 print_tainted(), cachep->name, realobj, size); in check_poison_obj()
1799 print_objinfo(cachep, objp, 0); in check_poison_obj()
1821 objnr = obj_to_index(cachep, page, objp); in check_poison_obj()
1823 objp = index_to_obj(cachep, page, objnr - 1); in check_poison_obj()
1824 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1827 print_objinfo(cachep, objp, 2); in check_poison_obj()
1829 if (objnr + 1 < cachep->num) { in check_poison_obj()
1830 objp = index_to_obj(cachep, page, objnr + 1); in check_poison_obj()
1831 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1834 print_objinfo(cachep, objp, 2); in check_poison_obj()
1841 static void slab_destroy_debugcheck(struct kmem_cache *cachep, in slab_destroy_debugcheck() argument
1845 for (i = 0; i < cachep->num; i++) { in slab_destroy_debugcheck()
1846 void *objp = index_to_obj(cachep, page, i); in slab_destroy_debugcheck()
1848 if (cachep->flags & SLAB_POISON) { in slab_destroy_debugcheck()
1850 if (cachep->size % PAGE_SIZE == 0 && in slab_destroy_debugcheck()
1851 OFF_SLAB(cachep)) in slab_destroy_debugcheck()
1853 cachep->size / PAGE_SIZE, 1); in slab_destroy_debugcheck()
1855 check_poison_obj(cachep, objp); in slab_destroy_debugcheck()
1857 check_poison_obj(cachep, objp); in slab_destroy_debugcheck()
1860 if (cachep->flags & SLAB_RED_ZONE) { in slab_destroy_debugcheck()
1861 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1862 slab_error(cachep, "start of a freed object " in slab_destroy_debugcheck()
1864 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1865 slab_error(cachep, "end of a freed object " in slab_destroy_debugcheck()
1871 static void slab_destroy_debugcheck(struct kmem_cache *cachep, in slab_destroy_debugcheck() argument
1886 static void slab_destroy(struct kmem_cache *cachep, struct page *page) in slab_destroy() argument
1891 slab_destroy_debugcheck(cachep, page); in slab_destroy()
1892 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) in slab_destroy()
1895 kmem_freepages(cachep, page); in slab_destroy()
1901 if (OFF_SLAB(cachep)) in slab_destroy()
1902 kmem_cache_free(cachep->freelist_cache, freelist); in slab_destroy()
1905 static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list) in slabs_destroy() argument
1911 slab_destroy(cachep, page); in slabs_destroy()
1928 static size_t calculate_slab_order(struct kmem_cache *cachep, in calculate_slab_order() argument
1964 cachep->num = num; in calculate_slab_order()
1965 cachep->gfporder = gfporder; in calculate_slab_order()
1993 struct kmem_cache *cachep, int entries, int batchcount) in alloc_kmem_cache_cpus() argument
2013 static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) in setup_cpu_cache() argument
2016 return enable_cpucache(cachep, gfp); in setup_cpu_cache()
2018 cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1); in setup_cpu_cache()
2019 if (!cachep->cpu_cache) in setup_cpu_cache()
2027 set_up_node(cachep, SIZE_NODE); in setup_cpu_cache()
2032 cachep->node[node] = kmalloc_node( in setup_cpu_cache()
2034 BUG_ON(!cachep->node[node]); in setup_cpu_cache()
2035 kmem_cache_node_init(cachep->node[node]); in setup_cpu_cache()
2039 cachep->node[numa_mem_id()]->next_reap = in setup_cpu_cache()
2041 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in setup_cpu_cache()
2043 cpu_cache_get(cachep)->avail = 0; in setup_cpu_cache()
2044 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; in setup_cpu_cache()
2045 cpu_cache_get(cachep)->batchcount = 1; in setup_cpu_cache()
2046 cpu_cache_get(cachep)->touched = 0; in setup_cpu_cache()
2047 cachep->batchcount = 1; in setup_cpu_cache()
2048 cachep->limit = BOOT_CPUCACHE_ENTRIES; in setup_cpu_cache()
2063 struct kmem_cache *cachep; in __kmem_cache_alias() local
2065 cachep = find_mergeable(size, align, flags, name, ctor); in __kmem_cache_alias()
2066 if (cachep) { in __kmem_cache_alias()
2067 cachep->refcount++; in __kmem_cache_alias()
2073 cachep->object_size = max_t(int, cachep->object_size, size); in __kmem_cache_alias()
2075 return cachep; in __kmem_cache_alias()
2100 __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) in __kmem_cache_create() argument
2106 size_t size = cachep->size; in __kmem_cache_create()
2145 if (ralign < cachep->align) { in __kmem_cache_create()
2146 ralign = cachep->align; in __kmem_cache_create()
2154 cachep->align = ralign; in __kmem_cache_create()
2169 cachep->obj_offset += sizeof(unsigned long long); in __kmem_cache_create()
2191 size >= 256 && cachep->object_size > cache_line_size() && in __kmem_cache_create()
2192 ALIGN(size, cachep->align) < PAGE_SIZE) { in __kmem_cache_create()
2193 cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align); in __kmem_cache_create()
2213 size = ALIGN(size, cachep->align); in __kmem_cache_create()
2219 size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align); in __kmem_cache_create()
2221 left_over = calculate_slab_order(cachep, size, cachep->align, flags); in __kmem_cache_create()
2223 if (!cachep->num) in __kmem_cache_create()
2226 freelist_size = calculate_freelist_size(cachep->num, cachep->align); in __kmem_cache_create()
2239 freelist_size = calculate_freelist_size(cachep->num, 0); in __kmem_cache_create()
2251 cachep->colour_off = cache_line_size(); in __kmem_cache_create()
2253 if (cachep->colour_off < cachep->align) in __kmem_cache_create()
2254 cachep->colour_off = cachep->align; in __kmem_cache_create()
2255 cachep->colour = left_over / cachep->colour_off; in __kmem_cache_create()
2256 cachep->freelist_size = freelist_size; in __kmem_cache_create()
2257 cachep->flags = flags; in __kmem_cache_create()
2258 cachep->allocflags = __GFP_COMP; in __kmem_cache_create()
2260 cachep->allocflags |= GFP_DMA; in __kmem_cache_create()
2261 cachep->size = size; in __kmem_cache_create()
2262 cachep->reciprocal_buffer_size = reciprocal_value(size); in __kmem_cache_create()
2265 cachep->freelist_cache = kmalloc_slab(freelist_size, 0u); in __kmem_cache_create()
2273 BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache)); in __kmem_cache_create()
2276 err = setup_cpu_cache(cachep, gfp); in __kmem_cache_create()
2278 __kmem_cache_shutdown(cachep); in __kmem_cache_create()
2296 static void check_spinlock_acquired(struct kmem_cache *cachep) in check_spinlock_acquired() argument
2300 assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock); in check_spinlock_acquired()
2304 static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) in check_spinlock_acquired_node() argument
2308 assert_spin_locked(&get_node(cachep, node)->list_lock); in check_spinlock_acquired_node()
2319 static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
2325 struct kmem_cache *cachep = arg; in do_drain() local
2332 ac = cpu_cache_get(cachep); in do_drain()
2333 n = get_node(cachep, node); in do_drain()
2335 free_block(cachep, ac->entry, ac->avail, node, &list); in do_drain()
2337 slabs_destroy(cachep, &list); in do_drain()
2341 static void drain_cpu_caches(struct kmem_cache *cachep) in drain_cpu_caches() argument
2346 on_each_cpu(do_drain, cachep, 1); in drain_cpu_caches()
2348 for_each_kmem_cache_node(cachep, node, n) in drain_cpu_caches()
2350 drain_alien_cache(cachep, n->alien); in drain_cpu_caches()
2352 for_each_kmem_cache_node(cachep, node, n) in drain_cpu_caches()
2353 drain_array(cachep, n, n->shared, 1, node); in drain_cpu_caches()
2397 int __kmem_cache_shrink(struct kmem_cache *cachep, bool deactivate) in __kmem_cache_shrink() argument
2403 drain_cpu_caches(cachep); in __kmem_cache_shrink()
2406 for_each_kmem_cache_node(cachep, node, n) { in __kmem_cache_shrink()
2407 drain_freelist(cachep, n, slabs_tofree(cachep, n)); in __kmem_cache_shrink()
2415 int __kmem_cache_shutdown(struct kmem_cache *cachep) in __kmem_cache_shutdown() argument
2419 int rc = __kmem_cache_shrink(cachep, false); in __kmem_cache_shutdown()
2424 free_percpu(cachep->cpu_cache); in __kmem_cache_shutdown()
2427 for_each_kmem_cache_node(cachep, i, n) { in __kmem_cache_shutdown()
2431 cachep->node[i] = NULL; in __kmem_cache_shutdown()
2450 static void *alloc_slabmgmt(struct kmem_cache *cachep, in alloc_slabmgmt() argument
2457 if (OFF_SLAB(cachep)) { in alloc_slabmgmt()
2459 freelist = kmem_cache_alloc_node(cachep->freelist_cache, in alloc_slabmgmt()
2465 colour_off += cachep->freelist_size; in alloc_slabmgmt()
2483 static void cache_init_objs(struct kmem_cache *cachep, in cache_init_objs() argument
2488 for (i = 0; i < cachep->num; i++) { in cache_init_objs()
2489 void *objp = index_to_obj(cachep, page, i); in cache_init_objs()
2492 if (cachep->flags & SLAB_POISON) in cache_init_objs()
2493 poison_obj(cachep, objp, POISON_FREE); in cache_init_objs()
2494 if (cachep->flags & SLAB_STORE_USER) in cache_init_objs()
2495 *dbg_userword(cachep, objp) = NULL; in cache_init_objs()
2497 if (cachep->flags & SLAB_RED_ZONE) { in cache_init_objs()
2498 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_init_objs()
2499 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_init_objs()
2506 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) in cache_init_objs()
2507 cachep->ctor(objp + obj_offset(cachep)); in cache_init_objs()
2509 if (cachep->flags & SLAB_RED_ZONE) { in cache_init_objs()
2510 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in cache_init_objs()
2511 slab_error(cachep, "constructor overwrote the" in cache_init_objs()
2513 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in cache_init_objs()
2514 slab_error(cachep, "constructor overwrote the" in cache_init_objs()
2517 if ((cachep->size % PAGE_SIZE) == 0 && in cache_init_objs()
2518 OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) in cache_init_objs()
2520 cachep->size / PAGE_SIZE, 0); in cache_init_objs()
2522 if (cachep->ctor) in cache_init_objs()
2523 cachep->ctor(objp); in cache_init_objs()
2530 static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags) in kmem_flagcheck() argument
2534 BUG_ON(!(cachep->allocflags & GFP_DMA)); in kmem_flagcheck()
2536 BUG_ON(cachep->allocflags & GFP_DMA); in kmem_flagcheck()
2540 static void *slab_get_obj(struct kmem_cache *cachep, struct page *page, in slab_get_obj() argument
2545 objp = index_to_obj(cachep, page, get_free_obj(page, page->active)); in slab_get_obj()
2554 static void slab_put_obj(struct kmem_cache *cachep, struct page *page, in slab_put_obj() argument
2557 unsigned int objnr = obj_to_index(cachep, page, objp); in slab_put_obj()
2565 for (i = page->active; i < cachep->num; i++) { in slab_put_obj()
2568 "'%s', objp %p\n", cachep->name, objp); in slab_put_obj()
2593 static int cache_grow(struct kmem_cache *cachep, in cache_grow() argument
2613 n = get_node(cachep, nodeid); in cache_grow()
2619 if (n->colour_next >= cachep->colour) in cache_grow()
2623 offset *= cachep->colour_off; in cache_grow()
2634 kmem_flagcheck(cachep, flags); in cache_grow()
2641 page = kmem_getpages(cachep, local_flags, nodeid); in cache_grow()
2646 freelist = alloc_slabmgmt(cachep, page, offset, in cache_grow()
2651 slab_map_pages(cachep, page, freelist); in cache_grow()
2653 cache_init_objs(cachep, page); in cache_grow()
2662 STATS_INC_GROWN(cachep); in cache_grow()
2663 n->free_objects += cachep->num; in cache_grow()
2667 kmem_freepages(cachep, page); in cache_grow()
2712 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, in cache_free_debugcheck() argument
2718 BUG_ON(virt_to_cache(objp) != cachep); in cache_free_debugcheck()
2720 objp -= obj_offset(cachep); in cache_free_debugcheck()
2724 if (cachep->flags & SLAB_RED_ZONE) { in cache_free_debugcheck()
2725 verify_redzone_free(cachep, objp); in cache_free_debugcheck()
2726 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2727 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2729 if (cachep->flags & SLAB_STORE_USER) in cache_free_debugcheck()
2730 *dbg_userword(cachep, objp) = (void *)caller; in cache_free_debugcheck()
2732 objnr = obj_to_index(cachep, page, objp); in cache_free_debugcheck()
2734 BUG_ON(objnr >= cachep->num); in cache_free_debugcheck()
2735 BUG_ON(objp != index_to_obj(cachep, page, objnr)); in cache_free_debugcheck()
2738 if (cachep->flags & SLAB_POISON) { in cache_free_debugcheck()
2740 if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { in cache_free_debugcheck()
2741 store_stackinfo(cachep, objp, caller); in cache_free_debugcheck()
2743 cachep->size / PAGE_SIZE, 0); in cache_free_debugcheck()
2745 poison_obj(cachep, objp, POISON_FREE); in cache_free_debugcheck()
2748 poison_obj(cachep, objp, POISON_FREE); in cache_free_debugcheck()
2759 static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags, in cache_alloc_refill() argument
2772 ac = cpu_cache_get(cachep); in cache_alloc_refill()
2782 n = get_node(cachep, node); in cache_alloc_refill()
2806 check_spinlock_acquired(cachep); in cache_alloc_refill()
2813 BUG_ON(page->active >= cachep->num); in cache_alloc_refill()
2815 while (page->active < cachep->num && batchcount--) { in cache_alloc_refill()
2816 STATS_INC_ALLOCED(cachep); in cache_alloc_refill()
2817 STATS_INC_ACTIVE(cachep); in cache_alloc_refill()
2818 STATS_SET_HIGH(cachep); in cache_alloc_refill()
2820 ac_put_obj(cachep, ac, slab_get_obj(cachep, page, in cache_alloc_refill()
2826 if (page->active == cachep->num) in cache_alloc_refill()
2840 x = cache_grow(cachep, gfp_exact_node(flags), node, NULL); in cache_alloc_refill()
2843 ac = cpu_cache_get(cachep); in cache_alloc_refill()
2855 return ac_get_obj(cachep, ac, flags, force_refill); in cache_alloc_refill()
2858 static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep, in cache_alloc_debugcheck_before() argument
2863 kmem_flagcheck(cachep, flags); in cache_alloc_debugcheck_before()
2868 static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, in cache_alloc_debugcheck_after() argument
2875 if (cachep->flags & SLAB_POISON) { in cache_alloc_debugcheck_after()
2877 if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) in cache_alloc_debugcheck_after()
2879 cachep->size / PAGE_SIZE, 1); in cache_alloc_debugcheck_after()
2881 check_poison_obj(cachep, objp); in cache_alloc_debugcheck_after()
2883 check_poison_obj(cachep, objp); in cache_alloc_debugcheck_after()
2885 poison_obj(cachep, objp, POISON_INUSE); in cache_alloc_debugcheck_after()
2887 if (cachep->flags & SLAB_STORE_USER) in cache_alloc_debugcheck_after()
2888 *dbg_userword(cachep, objp) = (void *)caller; in cache_alloc_debugcheck_after()
2890 if (cachep->flags & SLAB_RED_ZONE) { in cache_alloc_debugcheck_after()
2891 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || in cache_alloc_debugcheck_after()
2892 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { in cache_alloc_debugcheck_after()
2893 slab_error(cachep, "double free, or memory outside" in cache_alloc_debugcheck_after()
2897 objp, *dbg_redzone1(cachep, objp), in cache_alloc_debugcheck_after()
2898 *dbg_redzone2(cachep, objp)); in cache_alloc_debugcheck_after()
2900 *dbg_redzone1(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
2901 *dbg_redzone2(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
2905 set_obj_status(page, obj_to_index(cachep, page, objp), OBJECT_ACTIVE); in cache_alloc_debugcheck_after()
2906 objp += obj_offset(cachep); in cache_alloc_debugcheck_after()
2907 if (cachep->ctor && cachep->flags & SLAB_POISON) in cache_alloc_debugcheck_after()
2908 cachep->ctor(objp); in cache_alloc_debugcheck_after()
2920 static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags) in slab_should_failslab() argument
2922 if (unlikely(cachep == kmem_cache)) in slab_should_failslab()
2925 return should_failslab(cachep->object_size, flags, cachep->flags); in slab_should_failslab()
2928 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) in ____cache_alloc() argument
2936 ac = cpu_cache_get(cachep); in ____cache_alloc()
2939 objp = ac_get_obj(cachep, ac, flags, false); in ____cache_alloc()
2946 STATS_INC_ALLOCHIT(cachep); in ____cache_alloc()
2952 STATS_INC_ALLOCMISS(cachep); in ____cache_alloc()
2953 objp = cache_alloc_refill(cachep, flags, force_refill); in ____cache_alloc()
2958 ac = cpu_cache_get(cachep); in ____cache_alloc()
2978 static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) in alternate_node_alloc() argument
2985 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) in alternate_node_alloc()
2990 return ____cache_alloc_node(cachep, flags, nid_alloc); in alternate_node_alloc()
3085 static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, in ____cache_alloc_node() argument
3095 n = get_node(cachep, nodeid); in ____cache_alloc_node()
3110 check_spinlock_acquired_node(cachep, nodeid); in ____cache_alloc_node()
3112 STATS_INC_NODEALLOCS(cachep); in ____cache_alloc_node()
3113 STATS_INC_ACTIVE(cachep); in ____cache_alloc_node()
3114 STATS_SET_HIGH(cachep); in ____cache_alloc_node()
3116 BUG_ON(page->active == cachep->num); in ____cache_alloc_node()
3118 obj = slab_get_obj(cachep, page, nodeid); in ____cache_alloc_node()
3123 if (page->active == cachep->num) in ____cache_alloc_node()
3133 x = cache_grow(cachep, gfp_exact_node(flags), nodeid, NULL); in ____cache_alloc_node()
3137 return fallback_alloc(cachep, flags); in ____cache_alloc_node()
3144 slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, in slab_alloc_node() argument
3155 if (slab_should_failslab(cachep, flags)) in slab_alloc_node()
3158 cachep = memcg_kmem_get_cache(cachep, flags); in slab_alloc_node()
3160 cache_alloc_debugcheck_before(cachep, flags); in slab_alloc_node()
3166 if (unlikely(!get_node(cachep, nodeid))) { in slab_alloc_node()
3168 ptr = fallback_alloc(cachep, flags); in slab_alloc_node()
3179 ptr = ____cache_alloc(cachep, flags); in slab_alloc_node()
3184 ptr = ____cache_alloc_node(cachep, flags, nodeid); in slab_alloc_node()
3187 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); in slab_alloc_node()
3188 kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags, in slab_alloc_node()
3192 kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size); in slab_alloc_node()
3194 memset(ptr, 0, cachep->object_size); in slab_alloc_node()
3197 memcg_kmem_put_cache(cachep); in slab_alloc_node()
3226 __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) in __do_cache_alloc() argument
3228 return ____cache_alloc(cachep, flags); in __do_cache_alloc()
3234 slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) in slab_alloc() argument
3243 if (slab_should_failslab(cachep, flags)) in slab_alloc()
3246 cachep = memcg_kmem_get_cache(cachep, flags); in slab_alloc()
3248 cache_alloc_debugcheck_before(cachep, flags); in slab_alloc()
3250 objp = __do_cache_alloc(cachep, flags); in slab_alloc()
3252 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); in slab_alloc()
3253 kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags, in slab_alloc()
3258 kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size); in slab_alloc()
3260 memset(objp, 0, cachep->object_size); in slab_alloc()
3263 memcg_kmem_put_cache(cachep); in slab_alloc()
3271 static void free_block(struct kmem_cache *cachep, void **objpp, in free_block() argument
3275 struct kmem_cache_node *n = get_node(cachep, node); in free_block()
3286 check_spinlock_acquired_node(cachep, node); in free_block()
3287 slab_put_obj(cachep, page, objp, node); in free_block()
3288 STATS_DEC_ACTIVE(cachep); in free_block()
3294 n->free_objects -= cachep->num; in free_block()
3309 static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) in cache_flusharray() argument
3321 n = get_node(cachep, node); in cache_flusharray()
3336 free_block(cachep, ac->entry, batchcount, node, &list); in cache_flusharray()
3353 STATS_SET_FREEABLE(cachep, i); in cache_flusharray()
3357 slabs_destroy(cachep, &list); in cache_flusharray()
3366 static inline void __cache_free(struct kmem_cache *cachep, void *objp, in __cache_free() argument
3369 struct array_cache *ac = cpu_cache_get(cachep); in __cache_free()
3372 kmemleak_free_recursive(objp, cachep->flags); in __cache_free()
3373 objp = cache_free_debugcheck(cachep, objp, caller); in __cache_free()
3375 kmemcheck_slab_free(cachep, objp, cachep->object_size); in __cache_free()
3384 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) in __cache_free()
3388 STATS_INC_FREEHIT(cachep); in __cache_free()
3390 STATS_INC_FREEMISS(cachep); in __cache_free()
3391 cache_flusharray(cachep, ac); in __cache_free()
3394 ac_put_obj(cachep, ac, objp); in __cache_free()
3405 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) in kmem_cache_alloc() argument
3407 void *ret = slab_alloc(cachep, flags, _RET_IP_); in kmem_cache_alloc()
3410 cachep->object_size, cachep->size, flags); in kmem_cache_alloc()
3431 kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) in kmem_cache_alloc_trace() argument
3435 ret = slab_alloc(cachep, flags, _RET_IP_); in kmem_cache_alloc_trace()
3438 size, cachep->size, flags); in kmem_cache_alloc_trace()
3456 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) in kmem_cache_alloc_node() argument
3458 void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); in kmem_cache_alloc_node()
3461 cachep->object_size, cachep->size, in kmem_cache_alloc_node()
3469 void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, in kmem_cache_alloc_node_trace() argument
3476 ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); in kmem_cache_alloc_node_trace()
3479 size, cachep->size, in kmem_cache_alloc_node_trace()
3489 struct kmem_cache *cachep; in __do_kmalloc_node() local
3491 cachep = kmalloc_slab(size, flags); in __do_kmalloc_node()
3492 if (unlikely(ZERO_OR_NULL_PTR(cachep))) in __do_kmalloc_node()
3493 return cachep; in __do_kmalloc_node()
3494 return kmem_cache_alloc_node_trace(cachep, flags, node, size); in __do_kmalloc_node()
3520 struct kmem_cache *cachep; in __do_kmalloc() local
3523 cachep = kmalloc_slab(size, flags); in __do_kmalloc()
3524 if (unlikely(ZERO_OR_NULL_PTR(cachep))) in __do_kmalloc()
3525 return cachep; in __do_kmalloc()
3526 ret = slab_alloc(cachep, flags, caller); in __do_kmalloc()
3529 size, cachep->size, flags); in __do_kmalloc()
3554 void kmem_cache_free(struct kmem_cache *cachep, void *objp) in kmem_cache_free() argument
3557 cachep = cache_from_obj(cachep, objp); in kmem_cache_free()
3558 if (!cachep) in kmem_cache_free()
3562 debug_check_no_locks_freed(objp, cachep->object_size); in kmem_cache_free()
3563 if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) in kmem_cache_free()
3564 debug_check_no_obj_freed(objp, cachep->object_size); in kmem_cache_free()
3565 __cache_free(cachep, objp, _RET_IP_); in kmem_cache_free()
3604 static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp) in alloc_kmem_cache_node() argument
3614 new_alien = alloc_alien_cache(node, cachep->limit, gfp); in alloc_kmem_cache_node()
3620 if (cachep->shared) { in alloc_kmem_cache_node()
3622 cachep->shared*cachep->batchcount, in alloc_kmem_cache_node()
3630 n = get_node(cachep, node); in alloc_kmem_cache_node()
3638 free_block(cachep, shared->entry, in alloc_kmem_cache_node()
3647 cachep->batchcount + cachep->num; in alloc_kmem_cache_node()
3649 slabs_destroy(cachep, &list); in alloc_kmem_cache_node()
3663 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in alloc_kmem_cache_node()
3667 cachep->batchcount + cachep->num; in alloc_kmem_cache_node()
3668 cachep->node[node] = n; in alloc_kmem_cache_node()
3673 if (!cachep->list.next) { in alloc_kmem_cache_node()
3677 n = get_node(cachep, node); in alloc_kmem_cache_node()
3682 cachep->node[node] = NULL; in alloc_kmem_cache_node()
3691 static int __do_tune_cpucache(struct kmem_cache *cachep, int limit, in __do_tune_cpucache() argument
3697 cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount); in __do_tune_cpucache()
3701 prev = cachep->cpu_cache; in __do_tune_cpucache()
3702 cachep->cpu_cache = cpu_cache; in __do_tune_cpucache()
3706 cachep->batchcount = batchcount; in __do_tune_cpucache()
3707 cachep->limit = limit; in __do_tune_cpucache()
3708 cachep->shared = shared; in __do_tune_cpucache()
3720 n = get_node(cachep, node); in __do_tune_cpucache()
3722 free_block(cachep, ac->entry, ac->avail, node, &list); in __do_tune_cpucache()
3724 slabs_destroy(cachep, &list); in __do_tune_cpucache()
3729 return alloc_kmem_cache_node(cachep, gfp); in __do_tune_cpucache()
3732 static int do_tune_cpucache(struct kmem_cache *cachep, int limit, in do_tune_cpucache() argument
3738 ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp); in do_tune_cpucache()
3743 if ((ret < 0) || !is_root_cache(cachep)) in do_tune_cpucache()
3747 for_each_memcg_cache(c, cachep) { in do_tune_cpucache()
3756 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) in enable_cpucache() argument
3763 if (!is_root_cache(cachep)) { in enable_cpucache()
3764 struct kmem_cache *root = memcg_root_cache(cachep); in enable_cpucache()
3781 if (cachep->size > 131072) in enable_cpucache()
3783 else if (cachep->size > PAGE_SIZE) in enable_cpucache()
3785 else if (cachep->size > 1024) in enable_cpucache()
3787 else if (cachep->size > 256) in enable_cpucache()
3802 if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1) in enable_cpucache()
3815 err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp); in enable_cpucache()
3818 cachep->name, -err); in enable_cpucache()
3827 static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, in drain_array() argument
3843 free_block(cachep, ac->entry, tofree, node, &list); in drain_array()
3849 slabs_destroy(cachep, &list); in drain_array()
3922 void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) in get_slabinfo() argument
3936 for_each_kmem_cache_node(cachep, node, n) { in get_slabinfo()
3942 if (page->active != cachep->num && !error) in get_slabinfo()
3944 active_objs += cachep->num; in get_slabinfo()
3948 if (page->active == cachep->num && !error) in get_slabinfo()
3967 num_objs = num_slabs * cachep->num; in get_slabinfo()
3971 name = cachep->name; in get_slabinfo()
3980 sinfo->limit = cachep->limit; in get_slabinfo()
3981 sinfo->batchcount = cachep->batchcount; in get_slabinfo()
3982 sinfo->shared = cachep->shared; in get_slabinfo()
3983 sinfo->objects_per_slab = cachep->num; in get_slabinfo()
3984 sinfo->cache_order = cachep->gfporder; in get_slabinfo()
3987 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep) in slabinfo_show_stats() argument
3991 unsigned long high = cachep->high_mark; in slabinfo_show_stats()
3992 unsigned long allocs = cachep->num_allocations; in slabinfo_show_stats()
3993 unsigned long grown = cachep->grown; in slabinfo_show_stats()
3994 unsigned long reaped = cachep->reaped; in slabinfo_show_stats()
3995 unsigned long errors = cachep->errors; in slabinfo_show_stats()
3996 unsigned long max_freeable = cachep->max_freeable; in slabinfo_show_stats()
3997 unsigned long node_allocs = cachep->node_allocs; in slabinfo_show_stats()
3998 unsigned long node_frees = cachep->node_frees; in slabinfo_show_stats()
3999 unsigned long overflows = cachep->node_overflow; in slabinfo_show_stats()
4009 unsigned long allochit = atomic_read(&cachep->allochit); in slabinfo_show_stats()
4010 unsigned long allocmiss = atomic_read(&cachep->allocmiss); in slabinfo_show_stats()
4011 unsigned long freehit = atomic_read(&cachep->freehit); in slabinfo_show_stats()
4012 unsigned long freemiss = atomic_read(&cachep->freemiss); in slabinfo_show_stats()
4033 struct kmem_cache *cachep; in slabinfo_write() local
4052 list_for_each_entry(cachep, &slab_caches, list) { in slabinfo_write()
4053 if (!strcmp(cachep->name, kbuf)) { in slabinfo_write()
4058 res = do_tune_cpucache(cachep, limit, in slabinfo_write()
4138 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); in leaks_show() local
4146 if (!(cachep->flags & SLAB_STORE_USER)) in leaks_show()
4148 if (!(cachep->flags & SLAB_RED_ZONE)) in leaks_show()
4155 for_each_kmem_cache_node(cachep, node, n) { in leaks_show()
4161 handle_slab(x, cachep, page); in leaks_show()
4163 handle_slab(x, cachep, page); in leaks_show()
4166 name = cachep->name; in leaks_show()