302 			    struct kmemleak_object *object)  in hex_dump_object()  argument
304 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
308 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
325 static bool color_white(const struct kmemleak_object *object)  in color_white()  argument
327 	return object->count != KMEMLEAK_BLACK &&  in color_white()
328 		object->count < object->min_count;  in color_white()
331 static bool color_gray(const struct kmemleak_object *object)  in color_gray()  argument
333 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
334 		object->count >= object->min_count;  in color_gray()
342 static bool unreferenced_object(struct kmemleak_object *object)  in unreferenced_object()  argument
344 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
345 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
354 			       struct kmemleak_object *object)  in print_unreferenced()  argument
357 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
360 		   object->pointer, object->size);  in print_unreferenced()
362 		   object->comm, object->pid, object->jiffies,  in print_unreferenced()
364 	hex_dump_object(seq, object);  in print_unreferenced()
367 	for (i = 0; i < object->trace_len; i++) {  in print_unreferenced()
368 		void *ptr = (void *)object->trace[i];  in print_unreferenced()
378 static void dump_object_info(struct kmemleak_object *object)  in dump_object_info()  argument
382 	trace.nr_entries = object->trace_len;  in dump_object_info()
383 	trace.entries = object->trace;  in dump_object_info()
386 		  object->pointer, object->size);  in dump_object_info()
388 		  object->comm, object->pid, object->jiffies);  in dump_object_info()
389 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
390 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
391 	pr_notice("  flags = 0x%lx\n", object->flags);  in dump_object_info()
392 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
408 		struct kmemleak_object *object =  in lookup_object()  local
410 		if (ptr < object->pointer)  in lookup_object()
411 			rb = object->rb_node.rb_left;  in lookup_object()
412 		else if (object->pointer + object->size <= ptr)  in lookup_object()
413 			rb = object->rb_node.rb_right;  in lookup_object()
414 		else if (object->pointer == ptr || alias)  in lookup_object()
415 			return object;  in lookup_object()
419 			dump_object_info(object);  in lookup_object()
432 static int get_object(struct kmemleak_object *object)  in get_object()  argument
434 	return atomic_inc_not_zero(&object->use_count);  in get_object()
444 	struct kmemleak_object *object =  in free_object_rcu()  local
451 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
455 	kmem_cache_free(object_cache, object);  in free_object_rcu()
465 static void put_object(struct kmemleak_object *object)  in put_object()  argument
467 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
471 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
473 	call_rcu(&object->rcu, free_object_rcu);  in put_object()
482 	struct kmemleak_object *object;  in find_and_get_object()  local
486 	object = lookup_object(ptr, alias);  in find_and_get_object()
490 	if (object && !get_object(object))  in find_and_get_object()
491 		object = NULL;  in find_and_get_object()
494 	return object;  in find_and_get_object()
505 	struct kmemleak_object *object;  in find_and_remove_object()  local
508 	object = lookup_object(ptr, alias);  in find_and_remove_object()
509 	if (object) {  in find_and_remove_object()
510 		rb_erase(&object->rb_node, &object_tree_root);  in find_and_remove_object()
511 		list_del_rcu(&object->object_list);  in find_and_remove_object()
515 	return object;  in find_and_remove_object()
542 	struct kmemleak_object *object, *parent;  in create_object()  local
545 	object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));  in create_object()
546 	if (!object) {  in create_object()
552 	INIT_LIST_HEAD(&object->object_list);  in create_object()
553 	INIT_LIST_HEAD(&object->gray_list);  in create_object()
554 	INIT_HLIST_HEAD(&object->area_list);  in create_object()
555 	spin_lock_init(&object->lock);  in create_object()
556 	atomic_set(&object->use_count, 1);  in create_object()
557 	object->flags = OBJECT_ALLOCATED;  in create_object()
558 	object->pointer = ptr;  in create_object()
559 	object->size = size;  in create_object()
560 	object->min_count = min_count;  in create_object()
561 	object->count = 0;			/* white color initially */  in create_object()
562 	object->jiffies = jiffies;  in create_object()
563 	object->checksum = 0;  in create_object()
567 		object->pid = 0;  in create_object()
568 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in create_object()
570 		object->pid = 0;  in create_object()
571 		strncpy(object->comm, "softirq", sizeof(object->comm));  in create_object()
573 		object->pid = current->pid;  in create_object()
580 		strncpy(object->comm, current->comm, sizeof(object->comm));  in create_object()
584 	object->trace_len = __save_stack_trace(object->trace);  in create_object()
608 			kmem_cache_free(object_cache, object);  in create_object()
609 			object = NULL;  in create_object()
613 	rb_link_node(&object->rb_node, rb_parent, link);  in create_object()
614 	rb_insert_color(&object->rb_node, &object_tree_root);  in create_object()
616 	list_add_tail_rcu(&object->object_list, &object_list);  in create_object()
619 	return object;  in create_object()
625 static void __delete_object(struct kmemleak_object *object)  in __delete_object()  argument
629 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
630 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
636 	spin_lock_irqsave(&object->lock, flags);  in __delete_object()
637 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
638 	spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
639 	put_object(object);  in __delete_object()
648 	struct kmemleak_object *object;  in delete_object_full()  local
650 	object = find_and_remove_object(ptr, 0);  in delete_object_full()
651 	if (!object) {  in delete_object_full()
658 	__delete_object(object);  in delete_object_full()
668 	struct kmemleak_object *object;  in delete_object_part()  local
671 	object = find_and_remove_object(ptr, 1);  in delete_object_part()
672 	if (!object) {  in delete_object_part()
687 	start = object->pointer;  in delete_object_part()
688 	end = object->pointer + object->size;  in delete_object_part()
690 		create_object(start, ptr - start, object->min_count,  in delete_object_part()
693 		create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
696 	__delete_object(object);  in delete_object_part()
699 static void __paint_it(struct kmemleak_object *object, int color)  in __paint_it()  argument
701 	object->min_count = color;  in __paint_it()
703 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
706 static void paint_it(struct kmemleak_object *object, int color)  in paint_it()  argument
710 	spin_lock_irqsave(&object->lock, flags);  in paint_it()
711 	__paint_it(object, color);  in paint_it()
712 	spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
717 	struct kmemleak_object *object;  in paint_ptr()  local
719 	object = find_and_get_object(ptr, 0);  in paint_ptr()
720 	if (!object) {  in paint_ptr()
727 	paint_it(object, color);  in paint_ptr()
728 	put_object(object);  in paint_ptr()
756 	struct kmemleak_object *object;  in add_scan_area()  local
759 	object = find_and_get_object(ptr, 1);  in add_scan_area()
760 	if (!object) {  in add_scan_area()
772 	spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
774 		size = object->pointer + object->size - ptr;  in add_scan_area()
775 	} else if (ptr + size > object->pointer + object->size) {  in add_scan_area()
777 		dump_object_info(object);  in add_scan_area()
786 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
788 	spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
790 	put_object(object);  in add_scan_area()
801 	struct kmemleak_object *object;  in object_no_scan()  local
803 	object = find_and_get_object(ptr, 0);  in object_no_scan()
804 	if (!object) {  in object_no_scan()
809 	spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
810 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
811 	spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
812 	put_object(object);  in object_no_scan()
857 	struct kmemleak_object *object;  in early_alloc()  local
868 	object = create_object((unsigned long)log->ptr, log->size,  in early_alloc()
870 	if (!object)  in early_alloc()
872 	spin_lock_irqsave(&object->lock, flags);  in early_alloc()
874 		object->trace[i] = log->trace[i];  in early_alloc()
875 	object->trace_len = log->trace_len;  in early_alloc()
876 	spin_unlock_irqrestore(&object->lock, flags);  in early_alloc()
1019 	struct kmemleak_object *object;  in kmemleak_update_trace()  local
1027 	object = find_and_get_object((unsigned long)ptr, 1);  in kmemleak_update_trace()
1028 	if (!object) {  in kmemleak_update_trace()
1036 	spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1037 	object->trace_len = __save_stack_trace(object->trace);  in kmemleak_update_trace()
1038 	spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1040 	put_object(object);  in kmemleak_update_trace()
1127 static bool update_checksum(struct kmemleak_object *object)  in update_checksum()  argument
1129 	u32 old_csum = object->checksum;  in update_checksum()
1131 	if (!kmemcheck_is_obj_initialized(object->pointer, object->size))  in update_checksum()
1135 	object->checksum = crc32(0, (void *)object->pointer, object->size);  in update_checksum()
1138 	return object->checksum != old_csum;  in update_checksum()
1176 		struct kmemleak_object *object;  in scan_block()  local
1200 		object = lookup_object(pointer, 1);  in scan_block()
1201 		if (!object)  in scan_block()
1203 		if (object == scanned)  in scan_block()
1212 		spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1213 		if (!color_white(object)) {  in scan_block()
1215 			spin_unlock(&object->lock);  in scan_block()
1225 		object->count++;  in scan_block()
1226 		if (color_gray(object)) {  in scan_block()
1228 			WARN_ON(!get_object(object));  in scan_block()
1229 			list_add_tail(&object->gray_list, &gray_list);  in scan_block()
1231 		spin_unlock(&object->lock);  in scan_block()
1255 static void scan_object(struct kmemleak_object *object)  in scan_object()  argument
1264 	spin_lock_irqsave(&object->lock, flags);  in scan_object()
1265 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1267 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1270 	if (hlist_empty(&object->area_list)) {  in scan_object()
1271 		void *start = (void *)object->pointer;  in scan_object()
1272 		void *end = (void *)(object->pointer + object->size);  in scan_object()
1277 			scan_block(start, next, object);  in scan_object()
1283 			spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1285 			spin_lock_irqsave(&object->lock, flags);  in scan_object()
1286 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1288 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1291 				   object);  in scan_object()
1293 	spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1302 	struct kmemleak_object *object, *tmp;  in scan_gray_list()  local
1309 	object = list_entry(gray_list.next, typeof(*object), gray_list);  in scan_gray_list()
1310 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1315 			scan_object(object);  in scan_gray_list()
1317 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1321 		list_del(&object->gray_list);  in scan_gray_list()
1322 		put_object(object);  in scan_gray_list()
1324 		object = tmp;  in scan_gray_list()
1337 	struct kmemleak_object *object;  in kmemleak_scan()  local
1345 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1346 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1352 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1354 				 atomic_read(&object->use_count));  in kmemleak_scan()
1355 			dump_object_info(object);  in kmemleak_scan()
1359 		object->count = 0;  in kmemleak_scan()
1360 		if (color_gray(object) && get_object(object))  in kmemleak_scan()
1361 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1363 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1426 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1427 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1428 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1429 		    && update_checksum(object) && get_object(object)) {  in kmemleak_scan()
1431 			object->count = object->min_count;  in kmemleak_scan()
1432 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1434 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1453 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1454 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_scan()
1455 		if (unreferenced_object(object) &&  in kmemleak_scan()
1456 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1457 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1460 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_scan()
1543 	struct kmemleak_object *object;  in kmemleak_seq_start()  local
1552 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_seq_start()
1555 		if (get_object(object))  in kmemleak_seq_start()
1558 	object = NULL;  in kmemleak_seq_start()
1560 	return object;  in kmemleak_seq_start()
1608 	struct kmemleak_object *object = v;  in kmemleak_seq_show()  local
1611 	spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1612 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1613 		print_unreferenced(seq, object);  in kmemleak_seq_show()
1614 	spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1633 	struct kmemleak_object *object;  in dump_str_object_info()  local
1638 	object = find_and_get_object(addr, 0);  in dump_str_object_info()
1639 	if (!object) {  in dump_str_object_info()
1644 	spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1645 	dump_object_info(object);  in dump_str_object_info()
1646 	spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1648 	put_object(object);  in dump_str_object_info()
1660 	struct kmemleak_object *object;  in kmemleak_clear()  local
1664 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_clear()
1665 		spin_lock_irqsave(&object->lock, flags);  in kmemleak_clear()
1666 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1667 		    unreferenced_object(object))  in kmemleak_clear()
1668 			__paint_it(object, KMEMLEAK_GREY);  in kmemleak_clear()
1669 		spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_clear()
1772 	struct kmemleak_object *object;  in __kmemleak_do_cleanup()  local
1775 	list_for_each_entry_rcu(object, &object_list, object_list)  in __kmemleak_do_cleanup()
1776 		delete_object_full(object->pointer);  in __kmemleak_do_cleanup()