229 				struct page *page, const void *object)  in check_valid_pointer()  argument
233 	if (!object)  in check_valid_pointer()
237 	if (object < base || object >= base + page->objects * s->size ||  in check_valid_pointer()
238 		(object - base) % s->size) {  in check_valid_pointer()
245 static inline void *get_freepointer(struct kmem_cache *s, void *object)  in get_freepointer()  argument
247 	return *(void **)(object + s->offset);  in get_freepointer()
250 static void prefetch_freepointer(const struct kmem_cache *s, void *object)  in prefetch_freepointer()  argument
252 	prefetch(object + s->offset);  in prefetch_freepointer()
255 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)  in get_freepointer_safe()  argument
260 	probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));  in get_freepointer_safe()
262 	p = get_freepointer(s, object);  in get_freepointer_safe()
267 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)  in set_freepointer()  argument
269 	*(void **)(object + s->offset) = fp;  in set_freepointer()
498 static struct track *get_track(struct kmem_cache *s, void *object,  in get_track()  argument
504 		p = object + s->offset + sizeof(void *);  in get_track()
506 		p = object + s->inuse;  in get_track()
511 static void set_track(struct kmem_cache *s, void *object,  in set_track()  argument
514 	struct track *p = get_track(s, object, alloc);  in set_track()
545 static void init_tracking(struct kmem_cache *s, void *object)  in init_tracking()  argument
550 	set_track(s, object, TRACK_FREE, 0UL);  in init_tracking()
551 	set_track(s, object, TRACK_ALLOC, 0UL);  in init_tracking()
573 static void print_tracking(struct kmem_cache *s, void *object)  in print_tracking()  argument
578 	print_track("Allocated", get_track(s, object, TRACK_ALLOC));  in print_tracking()
579 	print_track("Freed", get_track(s, object, TRACK_FREE));  in print_tracking()
654 			u8 *object, char *reason)  in object_err()  argument
657 	print_trailer(s, page, object);  in object_err()
674 static void init_object(struct kmem_cache *s, void *object, u8 val)  in init_object()  argument
676 	u8 *p = object;  in init_object()
695 			u8 *object, char *what,  in check_bytes_and_report()  argument
714 	print_trailer(s, page, object);  in check_bytes_and_report()
812 					void *object, u8 val)  in check_object()  argument
814 	u8 *p = object;  in check_object()
815 	u8 *endobject = object + s->object_size;  in check_object()
818 		if (!check_bytes_and_report(s, page, object, "Redzone",  in check_object()
898 	void *object = NULL;  in on_freelist()  local
906 			if (object) {  in on_freelist()
907 				object_err(s, page, object,  in on_freelist()
909 				set_freepointer(s, object, NULL);  in on_freelist()
919 		object = fp;  in on_freelist()
920 		fp = get_freepointer(s, object);  in on_freelist()
943 static void trace(struct kmem_cache *s, struct page *page, void *object,  in trace()  argument
950 			object, page->inuse,  in trace()
954 			print_section("Object ", (void *)object,  in trace()
1021 								void *object)  in setup_object_debug()  argument
1026 	init_object(s, object, SLUB_RED_INACTIVE);  in setup_object_debug()
1027 	init_tracking(s, object);  in setup_object_debug()
1032 					void *object, unsigned long addr)  in alloc_debug_processing()  argument
1037 	if (!check_valid_pointer(s, page, object)) {  in alloc_debug_processing()
1038 		object_err(s, page, object, "Freelist Pointer check fails");  in alloc_debug_processing()
1042 	if (!check_object(s, page, object, SLUB_RED_INACTIVE))  in alloc_debug_processing()
1047 		set_track(s, object, TRACK_ALLOC, addr);  in alloc_debug_processing()
1048 	trace(s, page, object, 1);  in alloc_debug_processing()
1049 	init_object(s, object, SLUB_RED_ACTIVE);  in alloc_debug_processing()
1067 	struct kmem_cache *s, struct page *page, void *object,  in free_debug_processing()  argument
1078 	if (!check_valid_pointer(s, page, object)) {  in free_debug_processing()
1079 		slab_err(s, page, "Invalid object pointer 0x%p", object);  in free_debug_processing()
1083 	if (on_freelist(s, page, object)) {  in free_debug_processing()
1084 		object_err(s, page, object, "Object already free");  in free_debug_processing()
1088 	if (!check_object(s, page, object, SLUB_RED_ACTIVE))  in free_debug_processing()
1094 				"outside of slab", object);  in free_debug_processing()
1097 			       object);  in free_debug_processing()
1100 			object_err(s, page, object,  in free_debug_processing()
1106 		set_track(s, object, TRACK_FREE, addr);  in free_debug_processing()
1107 	trace(s, page, object, 0);  in free_debug_processing()
1108 	init_object(s, object, SLUB_RED_INACTIVE);  in free_debug_processing()
1120 	slab_fix(s, "Object at 0x%p not freed", object);  in free_debug_processing()
1207 			struct page *page, void *object) {}  in setup_object_debug()  argument
1210 	struct page *page, void *object, unsigned long addr) { return 0; }  in alloc_debug_processing()  argument
1213 	struct kmem_cache *s, struct page *page, void *object,  in free_debug_processing()  argument
1219 			void *object, u8 val) { return 1; }  in check_object()  argument
1275 					gfp_t flags, void *object)  in slab_post_alloc_hook()  argument
1278 	kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));  in slab_post_alloc_hook()
1279 	kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags);  in slab_post_alloc_hook()
1281 	kasan_slab_alloc(s, object);  in slab_post_alloc_hook()
1398 				void *object)  in setup_object()  argument
1400 	setup_object_debug(s, page, object);  in setup_object()
1402 		kasan_unpoison_object_data(s, object);  in setup_object()
1403 		s->ctor(object);  in setup_object()
1404 		kasan_poison_object_data(s, object);  in setup_object()
1620 	void *object = NULL;  in get_partial_node()  local
1640 		t = acquire_slab(s, n, page, object == NULL, &objects);  in get_partial_node()
1645 		if (!object) {  in get_partial_node()
1648 			object = t;  in get_partial_node()
1659 	return object;  in get_partial_node()
1673 	void *object;  in get_any_partial()  local
1708 				object = get_partial_node(s, n, c, flags);  in get_any_partial()
1709 				if (object) {  in get_any_partial()
1717 					return object;  in get_any_partial()
1732 	void *object;  in get_partial()  local
1740 	object = get_partial_node(s, get_node(s, searchnode), c, flags);  in get_partial()
1741 	if (object || node != NUMA_NO_NODE)  in get_partial()
1742 		return object;  in get_partial()
2428 	void **object;  in slab_alloc_node()  local
2470 	object = c->freelist;  in slab_alloc_node()
2472 	if (unlikely(!object || !node_match(page, node))) {  in slab_alloc_node()
2473 		object = __slab_alloc(s, gfpflags, node, addr, c);  in slab_alloc_node()
2476 		void *next_object = get_freepointer_safe(s, object);  in slab_alloc_node()
2494 				object, tid,  in slab_alloc_node()
2504 	if (unlikely(gfpflags & __GFP_ZERO) && object)  in slab_alloc_node()
2505 		memset(object, 0, s->object_size);  in slab_alloc_node()
2507 	slab_post_alloc_hook(s, gfpflags, object);  in slab_alloc_node()
2509 	return object;  in slab_alloc_node()
2581 	void **object = (void *)x;  in __slab_free()  local
2601 		set_freepointer(s, object, prior);  in __slab_free()
2635 		object, new.counters,  in __slab_free()
2704 	void **object = (void *)x;  in slab_free()  local
2727 		set_freepointer(s, object, c->freelist);  in slab_free()
2732 				object, next_tid(tid)))) {  in slab_free()
3374 static size_t __ksize(const void *object)  in __ksize()  argument
3378 	if (unlikely(object == ZERO_SIZE_PTR))  in __ksize()
3381 	page = virt_to_head_page(object);  in __ksize()
3391 size_t ksize(const void *object)  in ksize()  argument
3393 	size_t size = __ksize(object);  in ksize()
3396 	kasan_krealloc(object, size);  in ksize()
3404 	void *object = (void *)x;  in kfree()  local
3418 	slab_free(page->slab_cache, page, object, _RET_IP_);  in kfree()