295 			    struct kmemleak_object *object)  in hex_dump_object()  argument
297 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
303 		min(object->size, (size_t)(HEX_MAX_LINES * HEX_ROW_SIZE));  in hex_dump_object()
327 static bool color_white(const struct kmemleak_object *object)  in color_white()  argument
329 	return object->count != KMEMLEAK_BLACK &&  in color_white()
330 		object->count < object->min_count;  in color_white()
333 static bool color_gray(const struct kmemleak_object *object)  in color_gray()  argument
335 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
336 		object->count >= object->min_count;  in color_gray()
344 static bool unreferenced_object(struct kmemleak_object *object)  in unreferenced_object()  argument
346 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
347 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
356 			       struct kmemleak_object *object)  in print_unreferenced()  argument
359 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
362 		   object->pointer, object->size);  in print_unreferenced()
364 		   object->comm, object->pid, object->jiffies,  in print_unreferenced()
366 	hex_dump_object(seq, object);  in print_unreferenced()
369 	for (i = 0; i < object->trace_len; i++) {  in print_unreferenced()
370 		void *ptr = (void *)object->trace[i];  in print_unreferenced()
380 static void dump_object_info(struct kmemleak_object *object)  in dump_object_info()  argument
384 	trace.nr_entries = object->trace_len;  in dump_object_info()
385 	trace.entries = object->trace;  in dump_object_info()
388 		  object->pointer, object->size);  in dump_object_info()
390 		  object->comm, object->pid, object->jiffies);  in dump_object_info()
391 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
392 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
393 	pr_notice("  flags = 0x%lx\n", object->flags);  in dump_object_info()
394 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
410 		struct kmemleak_object *object =  in lookup_object()  local
412 		if (ptr < object->pointer)  in lookup_object()
413 			rb = object->rb_node.rb_left;  in lookup_object()
414 		else if (object->pointer + object->size <= ptr)  in lookup_object()
415 			rb = object->rb_node.rb_right;  in lookup_object()
416 		else if (object->pointer == ptr || alias)  in lookup_object()
417 			return object;  in lookup_object()
421 			dump_object_info(object);  in lookup_object()
434 static int get_object(struct kmemleak_object *object)  in get_object()  argument
436 	return atomic_inc_not_zero(&object->use_count);  in get_object()
446 	struct kmemleak_object *object =  in free_object_rcu()  local
453 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
457 	kmem_cache_free(object_cache, object);  in free_object_rcu()
467 static void put_object(struct kmemleak_object *object)  in put_object()  argument
469 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
473 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
475 	call_rcu(&object->rcu, free_object_rcu);  in put_object()
484 	struct kmemleak_object *object = NULL;  in find_and_get_object()  local
489 		object = lookup_object(ptr, alias);  in find_and_get_object()
493 	if (object && !get_object(object))  in find_and_get_object()
494 		object = NULL;  in find_and_get_object()
497 	return object;  in find_and_get_object()
524 	struct kmemleak_object *object, *parent;  in create_object()  local
527 	object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));  in create_object()
528 	if (!object) {  in create_object()
534 	INIT_LIST_HEAD(&object->object_list);  in create_object()
535 	INIT_LIST_HEAD(&object->gray_list);  in create_object()
536 	INIT_HLIST_HEAD(&object->area_list);  in create_object()
537 	spin_lock_init(&object->lock);  in create_object()
538 	atomic_set(&object->use_count, 1);  in create_object()
539 	object->flags = OBJECT_ALLOCATED;  in create_object()
540 	object->pointer = ptr;  in create_object()
541 	object->size = size;  in create_object()
542 	object->min_count = min_count;  in create_object()
543 	object->count = 0;			/* white color initially */  in create_object()
544 	object->jiffies = jiffies;  in create_object()
545 	object->checksum = 0;  in create_object()
549 		object->pid = 0;  in create_object()
550 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in create_object()
552 		object->pid = 0;  in create_object()
553 		strncpy(object->comm, "softirq", sizeof(object->comm));  in create_object()
555 		object->pid = current->pid;  in create_object()
562 		strncpy(object->comm, current->comm, sizeof(object->comm));  in create_object()
566 	object->trace_len = __save_stack_trace(object->trace);  in create_object()
585 			kmem_cache_free(object_cache, object);  in create_object()
586 			object = parent;  in create_object()
587 			spin_lock(&object->lock);  in create_object()
588 			dump_object_info(object);  in create_object()
589 			spin_unlock(&object->lock);  in create_object()
593 	rb_link_node(&object->rb_node, rb_parent, link);  in create_object()
594 	rb_insert_color(&object->rb_node, &object_tree_root);  in create_object()
596 	list_add_tail_rcu(&object->object_list, &object_list);  in create_object()
599 	return object;  in create_object()
606 static void __delete_object(struct kmemleak_object *object)  in __delete_object()  argument
611 	rb_erase(&object->rb_node, &object_tree_root);  in __delete_object()
612 	list_del_rcu(&object->object_list);  in __delete_object()
615 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
616 	WARN_ON(atomic_read(&object->use_count) < 2);  in __delete_object()
622 	spin_lock_irqsave(&object->lock, flags);  in __delete_object()
623 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
624 	spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
625 	put_object(object);  in __delete_object()
634 	struct kmemleak_object *object;  in delete_object_full()  local
636 	object = find_and_get_object(ptr, 0);  in delete_object_full()
637 	if (!object) {  in delete_object_full()
644 	__delete_object(object);  in delete_object_full()
645 	put_object(object);  in delete_object_full()
655 	struct kmemleak_object *object;  in delete_object_part()  local
658 	object = find_and_get_object(ptr, 1);  in delete_object_part()
659 	if (!object) {  in delete_object_part()
666 	__delete_object(object);  in delete_object_part()
675 	start = object->pointer;  in delete_object_part()
676 	end = object->pointer + object->size;  in delete_object_part()
678 		create_object(start, ptr - start, object->min_count,  in delete_object_part()
681 		create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
684 	put_object(object);  in delete_object_part()
687 static void __paint_it(struct kmemleak_object *object, int color)  in __paint_it()  argument
689 	object->min_count = color;  in __paint_it()
691 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
694 static void paint_it(struct kmemleak_object *object, int color)  in paint_it()  argument
698 	spin_lock_irqsave(&object->lock, flags);  in paint_it()
699 	__paint_it(object, color);  in paint_it()
700 	spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
705 	struct kmemleak_object *object;  in paint_ptr()  local
707 	object = find_and_get_object(ptr, 0);  in paint_ptr()
708 	if (!object) {  in paint_ptr()
715 	paint_it(object, color);  in paint_ptr()
716 	put_object(object);  in paint_ptr()
744 	struct kmemleak_object *object;  in add_scan_area()  local
747 	object = find_and_get_object(ptr, 1);  in add_scan_area()
748 	if (!object) {  in add_scan_area()
760 	spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
762 		size = object->pointer + object->size - ptr;  in add_scan_area()
763 	} else if (ptr + size > object->pointer + object->size) {  in add_scan_area()
765 		dump_object_info(object);  in add_scan_area()
774 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
776 	spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
778 	put_object(object);  in add_scan_area()
789 	struct kmemleak_object *object;  in object_no_scan()  local
791 	object = find_and_get_object(ptr, 0);  in object_no_scan()
792 	if (!object) {  in object_no_scan()
797 	spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
798 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
799 	spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
800 	put_object(object);  in object_no_scan()
844 	struct kmemleak_object *object;  in early_alloc()  local
855 	object = create_object((unsigned long)log->ptr, log->size,  in early_alloc()
857 	if (!object)  in early_alloc()
859 	spin_lock_irqsave(&object->lock, flags);  in early_alloc()
861 		object->trace[i] = log->trace[i];  in early_alloc()
862 	object->trace_len = log->trace_len;  in early_alloc()
863 	spin_unlock_irqrestore(&object->lock, flags);  in early_alloc()
1006 	struct kmemleak_object *object;  in kmemleak_update_trace()  local
1014 	object = find_and_get_object((unsigned long)ptr, 1);  in kmemleak_update_trace()
1015 	if (!object) {  in kmemleak_update_trace()
1023 	spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1024 	object->trace_len = __save_stack_trace(object->trace);  in kmemleak_update_trace()
1025 	spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1027 	put_object(object);  in kmemleak_update_trace()
1114 static bool update_checksum(struct kmemleak_object *object)  in update_checksum()  argument
1116 	u32 old_csum = object->checksum;  in update_checksum()
1118 	if (!kmemcheck_is_obj_initialized(object->pointer, object->size))  in update_checksum()
1122 	object->checksum = crc32(0, (void *)object->pointer, object->size);  in update_checksum()
1125 	return object->checksum != old_csum;  in update_checksum()
1161 		struct kmemleak_object *object;  in scan_block()  local
1179 		object = find_and_get_object(pointer, 1);  in scan_block()
1180 		if (!object)  in scan_block()
1182 		if (object == scanned) {  in scan_block()
1184 			put_object(object);  in scan_block()
1193 		spin_lock_irqsave_nested(&object->lock, flags,  in scan_block()
1195 		if (!color_white(object)) {  in scan_block()
1197 			spin_unlock_irqrestore(&object->lock, flags);  in scan_block()
1198 			put_object(object);  in scan_block()
1208 		object->count++;  in scan_block()
1209 		if (color_gray(object)) {  in scan_block()
1210 			list_add_tail(&object->gray_list, &gray_list);  in scan_block()
1211 			spin_unlock_irqrestore(&object->lock, flags);  in scan_block()
1215 		spin_unlock_irqrestore(&object->lock, flags);  in scan_block()
1216 		put_object(object);  in scan_block()
1224 static void scan_object(struct kmemleak_object *object)  in scan_object()  argument
1233 	spin_lock_irqsave(&object->lock, flags);  in scan_object()
1234 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1236 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1239 	if (hlist_empty(&object->area_list)) {  in scan_object()
1240 		void *start = (void *)object->pointer;  in scan_object()
1241 		void *end = (void *)(object->pointer + object->size);  in scan_object()
1243 		while (start < end && (object->flags & OBJECT_ALLOCATED) &&  in scan_object()
1244 		       !(object->flags & OBJECT_NO_SCAN)) {  in scan_object()
1246 				   object, 0);  in scan_object()
1249 			spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1251 			spin_lock_irqsave(&object->lock, flags);  in scan_object()
1254 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1257 				   object, 0);  in scan_object()
1259 	spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1268 	struct kmemleak_object *object, *tmp;  in scan_gray_list()  local
1275 	object = list_entry(gray_list.next, typeof(*object), gray_list);  in scan_gray_list()
1276 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1281 			scan_object(object);  in scan_gray_list()
1283 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1287 		list_del(&object->gray_list);  in scan_gray_list()
1288 		put_object(object);  in scan_gray_list()
1290 		object = tmp;  in scan_gray_list()
1303 	struct kmemleak_object *object;  in kmemleak_scan()  local
1311 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1312 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1318 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1320 				 atomic_read(&object->use_count));  in kmemleak_scan()
1321 			dump_object_info(object);  in kmemleak_scan()
1325 		object->count = 0;  in kmemleak_scan()
1326 		if (color_gray(object) && get_object(object))  in kmemleak_scan()
1327 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1329 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1392 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1393 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1394 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1395 		    && update_checksum(object) && get_object(object)) {  in kmemleak_scan()
1397 			object->count = object->min_count;  in kmemleak_scan()
1398 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1400 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1419 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1420 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1421 		if (unreferenced_object(object) &&  in kmemleak_scan()
1422 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1423 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1426 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1509 	struct kmemleak_object *object;  in kmemleak_seq_start()  local
1518 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_seq_start()
1521 		if (get_object(object))  in kmemleak_seq_start()
1524 	object = NULL;  in kmemleak_seq_start()
1526 	return object;  in kmemleak_seq_start()
1574 	struct kmemleak_object *object = v;  in kmemleak_seq_show()  local
1577 	spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1578 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1579 		print_unreferenced(seq, object);  in kmemleak_seq_show()
1580 	spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1599 	struct kmemleak_object *object;  in dump_str_object_info()  local
1604 	object = find_and_get_object(addr, 0);  in dump_str_object_info()
1605 	if (!object) {  in dump_str_object_info()
1610 	spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1611 	dump_object_info(object);  in dump_str_object_info()
1612 	spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1614 	put_object(object);  in dump_str_object_info()
1626 	struct kmemleak_object *object;  in kmemleak_clear()  local
1630 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_clear()
1631 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_clear()
1632 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1633 		    unreferenced_object(object))  in kmemleak_clear()
1634 			__paint_it(object, KMEMLEAK_GREY);  in kmemleak_clear()
1635 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_clear()
1738 	struct kmemleak_object *object;  in __kmemleak_do_cleanup()  local
1741 	list_for_each_entry_rcu(object, &object_list, object_list)  in __kmemleak_do_cleanup()
1742 		delete_object_full(object->pointer);  in __kmemleak_do_cleanup()