Lines Matching refs:objp
211 static inline bool is_obj_pfmemalloc(void *objp) in is_obj_pfmemalloc() argument
213 return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC; in is_obj_pfmemalloc()
216 static inline void set_obj_pfmemalloc(void **objp) in set_obj_pfmemalloc() argument
218 *objp = (void *)((unsigned long)*objp | SLAB_OBJ_PFMEMALLOC); in set_obj_pfmemalloc()
222 static inline void clear_obj_pfmemalloc(void **objp) in clear_obj_pfmemalloc() argument
224 *objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC); in clear_obj_pfmemalloc()
359 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) in dbg_redzone1() argument
362 return (unsigned long long*) (objp + obj_offset(cachep) - in dbg_redzone1()
366 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) in dbg_redzone2() argument
370 return (unsigned long long *)(objp + cachep->size - in dbg_redzone2()
373 return (unsigned long long *) (objp + cachep->size - in dbg_redzone2()
377 static void **dbg_userword(struct kmem_cache *cachep, void *objp) in dbg_userword() argument
380 return (void **)(objp + cachep->size - BYTES_PER_WORD); in dbg_userword()
386 #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
387 #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
388 #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) argument
727 void *objp = ac->entry[--ac->avail]; in __ac_get_obj() local
730 if (unlikely(is_obj_pfmemalloc(objp))) { in __ac_get_obj()
734 clear_obj_pfmemalloc(&objp); in __ac_get_obj()
735 return objp; in __ac_get_obj()
742 objp = ac->entry[i]; in __ac_get_obj()
744 ac->entry[ac->avail] = objp; in __ac_get_obj()
745 return objp; in __ac_get_obj()
755 struct page *page = virt_to_head_page(objp); in __ac_get_obj()
757 clear_obj_pfmemalloc(&objp); in __ac_get_obj()
759 return objp; in __ac_get_obj()
764 objp = NULL; in __ac_get_obj()
767 return objp; in __ac_get_obj()
773 void *objp; in ac_get_obj() local
776 objp = __ac_get_obj(cachep, ac, flags, force_refill); in ac_get_obj()
778 objp = ac->entry[--ac->avail]; in ac_get_obj()
780 return objp; in ac_get_obj()
784 struct array_cache *ac, void *objp) in __ac_put_obj() argument
788 struct page *page = virt_to_head_page(objp); in __ac_put_obj()
790 set_obj_pfmemalloc(&objp); in __ac_put_obj()
793 return objp; in __ac_put_obj()
797 void *objp) in ac_put_obj() argument
800 objp = __ac_put_obj(cachep, ac, objp); in ac_put_obj()
802 ac->entry[ac->avail++] = objp; in ac_put_obj()
843 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
987 static int __cache_free_alien(struct kmem_cache *cachep, void *objp, in __cache_free_alien() argument
1005 ac_put_obj(cachep, ac, objp); in __cache_free_alien()
1011 free_block(cachep, &objp, 1, page_node, &list); in __cache_free_alien()
1018 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
1020 int page_node = page_to_nid(virt_to_page(objp)); in cache_free_alien()
1029 return __cache_free_alien(cachep, objp, node, page_node); in cache_free_alien()
1749 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) in print_objinfo() argument
1756 *dbg_redzone1(cachep, objp), in print_objinfo()
1757 *dbg_redzone2(cachep, objp)); in print_objinfo()
1762 *dbg_userword(cachep, objp), in print_objinfo()
1763 *dbg_userword(cachep, objp)); in print_objinfo()
1765 realobj = (char *)objp + obj_offset(cachep); in print_objinfo()
1776 static void check_poison_obj(struct kmem_cache *cachep, void *objp) in check_poison_obj() argument
1782 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1797 print_objinfo(cachep, objp, 0); in check_poison_obj()
1816 struct page *page = virt_to_head_page(objp); in check_poison_obj()
1819 objnr = obj_to_index(cachep, page, objp); in check_poison_obj()
1821 objp = index_to_obj(cachep, page, objnr - 1); in check_poison_obj()
1822 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1825 print_objinfo(cachep, objp, 2); in check_poison_obj()
1828 objp = index_to_obj(cachep, page, objnr + 1); in check_poison_obj()
1829 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1832 print_objinfo(cachep, objp, 2); in check_poison_obj()
1844 void *objp = index_to_obj(cachep, page, i); in slab_destroy_debugcheck() local
1850 kernel_map_pages(virt_to_page(objp), in slab_destroy_debugcheck()
1853 check_poison_obj(cachep, objp); in slab_destroy_debugcheck()
1855 check_poison_obj(cachep, objp); in slab_destroy_debugcheck()
1859 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1862 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
2498 void *objp = index_to_obj(cachep, page, i); in cache_init_objs() local
2502 poison_obj(cachep, objp, POISON_FREE); in cache_init_objs()
2504 *dbg_userword(cachep, objp) = NULL; in cache_init_objs()
2507 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_init_objs()
2508 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_init_objs()
2516 cachep->ctor(objp + obj_offset(cachep)); in cache_init_objs()
2519 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in cache_init_objs()
2522 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in cache_init_objs()
2528 kernel_map_pages(virt_to_page(objp), in cache_init_objs()
2532 cachep->ctor(objp); in cache_init_objs()
2552 void *objp; in slab_get_obj() local
2554 objp = index_to_obj(cachep, page, get_free_obj(page, page->active)); in slab_get_obj()
2557 WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid); in slab_get_obj()
2560 return objp; in slab_get_obj()
2564 void *objp, int nodeid) in slab_put_obj() argument
2566 unsigned int objnr = obj_to_index(cachep, page, objp); in slab_put_obj()
2571 WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid); in slab_put_obj()
2577 "'%s', objp %p\n", cachep->name, objp); in slab_put_obj()
2690 static void kfree_debugcheck(const void *objp) in kfree_debugcheck() argument
2692 if (!virt_addr_valid(objp)) { in kfree_debugcheck()
2694 (unsigned long)objp); in kfree_debugcheck()
2721 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, in cache_free_debugcheck() argument
2727 BUG_ON(virt_to_cache(objp) != cachep); in cache_free_debugcheck()
2729 objp -= obj_offset(cachep); in cache_free_debugcheck()
2730 kfree_debugcheck(objp); in cache_free_debugcheck()
2731 page = virt_to_head_page(objp); in cache_free_debugcheck()
2734 verify_redzone_free(cachep, objp); in cache_free_debugcheck()
2735 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2736 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2739 *dbg_userword(cachep, objp) = (void *)caller; in cache_free_debugcheck()
2741 objnr = obj_to_index(cachep, page, objp); in cache_free_debugcheck()
2744 BUG_ON(objp != index_to_obj(cachep, page, objnr)); in cache_free_debugcheck()
2750 store_stackinfo(cachep, objp, caller); in cache_free_debugcheck()
2751 kernel_map_pages(virt_to_page(objp), in cache_free_debugcheck()
2754 poison_obj(cachep, objp, POISON_FREE); in cache_free_debugcheck()
2757 poison_obj(cachep, objp, POISON_FREE); in cache_free_debugcheck()
2760 return objp; in cache_free_debugcheck()
2765 #define cache_free_debugcheck(x,objp,z) (objp) argument
2878 gfp_t flags, void *objp, unsigned long caller) in cache_alloc_debugcheck_after() argument
2882 if (!objp) in cache_alloc_debugcheck_after()
2883 return objp; in cache_alloc_debugcheck_after()
2887 kernel_map_pages(virt_to_page(objp), in cache_alloc_debugcheck_after()
2890 check_poison_obj(cachep, objp); in cache_alloc_debugcheck_after()
2892 check_poison_obj(cachep, objp); in cache_alloc_debugcheck_after()
2894 poison_obj(cachep, objp, POISON_INUSE); in cache_alloc_debugcheck_after()
2897 *dbg_userword(cachep, objp) = (void *)caller; in cache_alloc_debugcheck_after()
2900 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || in cache_alloc_debugcheck_after()
2901 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { in cache_alloc_debugcheck_after()
2906 objp, *dbg_redzone1(cachep, objp), in cache_alloc_debugcheck_after()
2907 *dbg_redzone2(cachep, objp)); in cache_alloc_debugcheck_after()
2909 *dbg_redzone1(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
2910 *dbg_redzone2(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
2913 page = virt_to_head_page(objp); in cache_alloc_debugcheck_after()
2914 set_obj_status(page, obj_to_index(cachep, page, objp), OBJECT_ACTIVE); in cache_alloc_debugcheck_after()
2915 objp += obj_offset(cachep); in cache_alloc_debugcheck_after()
2917 cachep->ctor(objp); in cache_alloc_debugcheck_after()
2919 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { in cache_alloc_debugcheck_after()
2921 objp, (int)ARCH_SLAB_MINALIGN); in cache_alloc_debugcheck_after()
2923 return objp; in cache_alloc_debugcheck_after()
2926 #define cache_alloc_debugcheck_after(a,b,objp,d) (objp) argument
2939 void *objp; in ____cache_alloc() local
2948 objp = ac_get_obj(cachep, ac, flags, false); in ____cache_alloc()
2954 if (objp) { in ____cache_alloc()
2962 objp = cache_alloc_refill(cachep, flags, force_refill); in ____cache_alloc()
2975 if (objp) in ____cache_alloc()
2977 return objp; in ____cache_alloc()
3213 void *objp; in __do_cache_alloc() local
3216 objp = alternate_node_alloc(cache, flags); in __do_cache_alloc()
3217 if (objp) in __do_cache_alloc()
3220 objp = ____cache_alloc(cache, flags); in __do_cache_alloc()
3226 if (!objp) in __do_cache_alloc()
3227 objp = ____cache_alloc_node(cache, flags, numa_mem_id()); in __do_cache_alloc()
3230 return objp; in __do_cache_alloc()
3246 void *objp; in slab_alloc() local
3259 objp = __do_cache_alloc(cachep, flags); in slab_alloc()
3261 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); in slab_alloc()
3262 kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags, in slab_alloc()
3264 prefetchw(objp); in slab_alloc()
3266 if (likely(objp)) { in slab_alloc()
3267 kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size); in slab_alloc()
3269 memset(objp, 0, cachep->object_size); in slab_alloc()
3273 return objp; in slab_alloc()
3287 void *objp; in free_block() local
3291 objp = objpp[i]; in free_block()
3293 page = virt_to_head_page(objp); in free_block()
3296 slab_put_obj(cachep, page, objp, node); in free_block()
3375 static inline void __cache_free(struct kmem_cache *cachep, void *objp, in __cache_free() argument
3381 kmemleak_free_recursive(objp, cachep->flags); in __cache_free()
3382 objp = cache_free_debugcheck(cachep, objp, caller); in __cache_free()
3384 kmemcheck_slab_free(cachep, objp, cachep->object_size); in __cache_free()
3393 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) in __cache_free()
3403 ac_put_obj(cachep, ac, objp); in __cache_free()
3550 void kmem_cache_free(struct kmem_cache *cachep, void *objp) in kmem_cache_free() argument
3553 cachep = cache_from_obj(cachep, objp); in kmem_cache_free()
3558 debug_check_no_locks_freed(objp, cachep->object_size); in kmem_cache_free()
3560 debug_check_no_obj_freed(objp, cachep->object_size); in kmem_cache_free()
3561 __cache_free(cachep, objp, _RET_IP_); in kmem_cache_free()
3564 trace_kmem_cache_free(_RET_IP_, objp); in kmem_cache_free()
3577 void kfree(const void *objp) in kfree() argument
3582 trace_kfree(_RET_IP_, objp); in kfree()
3584 if (unlikely(ZERO_OR_NULL_PTR(objp))) in kfree()
3587 kfree_debugcheck(objp); in kfree()
3588 c = virt_to_cache(objp); in kfree()
3589 debug_check_no_locks_freed(objp, c->object_size); in kfree()
3591 debug_check_no_obj_freed(objp, c->object_size); in kfree()
3592 __cache_free(c, (void *)objp, _RET_IP_); in kfree()
4239 size_t ksize(const void *objp) in ksize() argument
4241 BUG_ON(!objp); in ksize()
4242 if (unlikely(objp == ZERO_SIZE_PTR)) in ksize()
4245 return virt_to_cache(objp)->object_size; in ksize()