246 					    &block_group->io_ctl, path,  in btrfs_truncate_free_space_cache()
309 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode,  in io_ctl_init()  argument
325 	memset(io_ctl, 0, sizeof(struct btrfs_io_ctl));  in io_ctl_init()
327 	io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS);  in io_ctl_init()
328 	if (!io_ctl->pages)  in io_ctl_init()
331 	io_ctl->num_pages = num_pages;  in io_ctl_init()
332 	io_ctl->root = root;  in io_ctl_init()
333 	io_ctl->check_crcs = check_crcs;  in io_ctl_init()
334 	io_ctl->inode = inode;  in io_ctl_init()
339 static void io_ctl_free(struct btrfs_io_ctl *io_ctl)  in io_ctl_free()  argument
341 	kfree(io_ctl->pages);  in io_ctl_free()
342 	io_ctl->pages = NULL;  in io_ctl_free()
345 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl)  in io_ctl_unmap_page()  argument
347 	if (io_ctl->cur) {  in io_ctl_unmap_page()
348 		io_ctl->cur = NULL;  in io_ctl_unmap_page()
349 		io_ctl->orig = NULL;  in io_ctl_unmap_page()
353 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear)  in io_ctl_map_page()  argument
355 	ASSERT(io_ctl->index < io_ctl->num_pages);  in io_ctl_map_page()
356 	io_ctl->page = io_ctl->pages[io_ctl->index++];  in io_ctl_map_page()
357 	io_ctl->cur = page_address(io_ctl->page);  in io_ctl_map_page()
358 	io_ctl->orig = io_ctl->cur;  in io_ctl_map_page()
359 	io_ctl->size = PAGE_CACHE_SIZE;  in io_ctl_map_page()
361 		memset(io_ctl->cur, 0, PAGE_CACHE_SIZE);  in io_ctl_map_page()
364 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_drop_pages()  argument
368 	io_ctl_unmap_page(io_ctl);  in io_ctl_drop_pages()
370 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_drop_pages()
371 		if (io_ctl->pages[i]) {  in io_ctl_drop_pages()
372 			ClearPageChecked(io_ctl->pages[i]);  in io_ctl_drop_pages()
373 			unlock_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
374 			page_cache_release(io_ctl->pages[i]);  in io_ctl_drop_pages()
379 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, struct inode *inode,  in io_ctl_prepare_pages()  argument
386 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
389 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
392 		io_ctl->pages[i] = page;  in io_ctl_prepare_pages()
399 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
405 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
406 		clear_page_dirty_for_io(io_ctl->pages[i]);  in io_ctl_prepare_pages()
407 		set_page_extent_mapped(io_ctl->pages[i]);  in io_ctl_prepare_pages()
413 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_set_generation()  argument
417 	io_ctl_map_page(io_ctl, 1);  in io_ctl_set_generation()
423 	if (io_ctl->check_crcs) {  in io_ctl_set_generation()
424 		io_ctl->cur += (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
425 		io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
427 		io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
428 		io_ctl->size -= sizeof(u64) * 2;  in io_ctl_set_generation()
431 	val = io_ctl->cur;  in io_ctl_set_generation()
433 	io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
436 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_check_generation()  argument
444 	if (io_ctl->check_crcs) {  in io_ctl_check_generation()
445 		io_ctl->cur += sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_generation()
446 		io_ctl->size -= sizeof(u64) +  in io_ctl_check_generation()
447 			(sizeof(u32) * io_ctl->num_pages);  in io_ctl_check_generation()
449 		io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
450 		io_ctl->size -= sizeof(u64) * 2;  in io_ctl_check_generation()
453 	gen = io_ctl->cur;  in io_ctl_check_generation()
458 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_generation()
461 	io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
465 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_set_crc()  argument
471 	if (!io_ctl->check_crcs) {  in io_ctl_set_crc()
472 		io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
477 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_set_crc()
479 	crc = btrfs_csum_data(io_ctl->orig + offset, crc,  in io_ctl_set_crc()
482 	io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
483 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_set_crc()
488 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_check_crc()  argument
494 	if (!io_ctl->check_crcs) {  in io_ctl_check_crc()
495 		io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
500 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_crc()
502 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_check_crc()
506 	io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
507 	crc = btrfs_csum_data(io_ctl->orig + offset, crc,  in io_ctl_check_crc()
513 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_crc()
520 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes,  in io_ctl_add_entry()  argument
525 	if (!io_ctl->cur)  in io_ctl_add_entry()
528 	entry = io_ctl->cur;  in io_ctl_add_entry()
533 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
534 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
536 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_add_entry()
539 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_entry()
542 	if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_entry()
546 	io_ctl_map_page(io_ctl, 1);  in io_ctl_add_entry()
550 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap)  in io_ctl_add_bitmap()  argument
552 	if (!io_ctl->cur)  in io_ctl_add_bitmap()
559 	if (io_ctl->cur != io_ctl->orig) {  in io_ctl_add_bitmap()
560 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
561 		if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_bitmap()
563 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
566 	memcpy(io_ctl->cur, bitmap, PAGE_CACHE_SIZE);  in io_ctl_add_bitmap()
567 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
568 	if (io_ctl->index < io_ctl->num_pages)  in io_ctl_add_bitmap()
569 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
573 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_zero_remaining_pages()  argument
579 	if (io_ctl->cur != io_ctl->orig)  in io_ctl_zero_remaining_pages()
580 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
582 		io_ctl_unmap_page(io_ctl);  in io_ctl_zero_remaining_pages()
584 	while (io_ctl->index < io_ctl->num_pages) {  in io_ctl_zero_remaining_pages()
585 		io_ctl_map_page(io_ctl, 1);  in io_ctl_zero_remaining_pages()
586 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
590 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_entry()  argument
596 	if (!io_ctl->cur) {  in io_ctl_read_entry()
597 		ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_entry()
602 	e = io_ctl->cur;  in io_ctl_read_entry()
606 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
607 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
609 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_read_entry()
612 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_entry()
617 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_bitmap()  argument
622 	ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_bitmap()
626 	memcpy(entry->bitmap, io_ctl->cur, PAGE_CACHE_SIZE);  in io_ctl_read_bitmap()
627 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_bitmap()
676 	struct btrfs_io_ctl io_ctl;  in __load_free_space_cache()  local
730 	ret = io_ctl_init(&io_ctl, inode, root, 0);  in __load_free_space_cache()
738 	ret = io_ctl_prepare_pages(&io_ctl, inode, 1);  in __load_free_space_cache()
742 	ret = io_ctl_check_crc(&io_ctl, 0);  in __load_free_space_cache()
746 	ret = io_ctl_check_generation(&io_ctl, generation);  in __load_free_space_cache()
756 		ret = io_ctl_read_entry(&io_ctl, e, &type);  in __load_free_space_cache()
803 	io_ctl_unmap_page(&io_ctl);  in __load_free_space_cache()
811 		ret = io_ctl_read_bitmap(&io_ctl, e);  in __load_free_space_cache()
816 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
820 	io_ctl_free(&io_ctl);  in __load_free_space_cache()
823 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
905 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,  in write_cache_extent_entries()  argument
938 		ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes,  in write_cache_extent_entries()
967 		ret = io_ctl_add_entry(io_ctl, trim_entry->start,  in write_cache_extent_entries()
1039 			    struct btrfs_io_ctl *io_ctl,  in write_pinned_extent_entries()  argument
1078 		ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL);  in write_pinned_extent_entries()
1089 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list)  in write_bitmap_entries()  argument
1099 		ret = io_ctl_add_bitmap(io_ctl, entry->bitmap);  in write_bitmap_entries()
1135 			   struct btrfs_io_ctl *io_ctl,  in cleanup_write_cache_enospc()  argument
1139 	io_ctl_drop_pages(io_ctl);  in cleanup_write_cache_enospc()
1148 			struct btrfs_io_ctl *io_ctl,  in btrfs_wait_cache_io()  argument
1152 	struct inode *inode = io_ctl->inode;  in btrfs_wait_cache_io()
1167 				io_ctl->entries, io_ctl->bitmaps);  in btrfs_wait_cache_io()
1169 	io_ctl_free(io_ctl);  in btrfs_wait_cache_io()
1202 		io_ctl->inode = NULL;  in btrfs_wait_cache_io()
1226 				   struct btrfs_io_ctl *io_ctl,  in __btrfs_write_out_cache()  argument
1240 	WARN_ON(io_ctl->pages);  in __btrfs_write_out_cache()
1241 	ret = io_ctl_init(io_ctl, inode, root, 1);  in __btrfs_write_out_cache()
1261 	ret = io_ctl_prepare_pages(io_ctl, inode, 0);  in __btrfs_write_out_cache()
1268 	io_ctl_set_generation(io_ctl, trans->transid);  in __btrfs_write_out_cache()
1273 	ret = write_cache_extent_entries(io_ctl, ctl,  in __btrfs_write_out_cache()
1287 	ret = write_pinned_extent_entries(root, block_group, io_ctl, &entries);  in __btrfs_write_out_cache()
1296 	ret = write_bitmap_entries(io_ctl, &bitmap_list);  in __btrfs_write_out_cache()
1303 	io_ctl_zero_remaining_pages(io_ctl);  in __btrfs_write_out_cache()
1306 	ret = btrfs_dirty_pages(root, inode, io_ctl->pages, io_ctl->num_pages,  in __btrfs_write_out_cache()
1317 	io_ctl_drop_pages(io_ctl);  in __btrfs_write_out_cache()
1327 	io_ctl->entries = entries;  in __btrfs_write_out_cache()
1328 	io_ctl->bitmaps = bitmaps;  in __btrfs_write_out_cache()
1337 	io_ctl->inode = NULL;  in __btrfs_write_out_cache()
1338 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1354 	cleanup_write_cache_enospc(inode, io_ctl, &cached_state, &bitmap_list);  in __btrfs_write_out_cache()
1385 				      &block_group->io_ctl, trans,  in btrfs_write_out_cache()
1397 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()
3468 	struct btrfs_io_ctl io_ctl;  in btrfs_write_out_ino_cache()  local
3474 	memset(&io_ctl, 0, sizeof(io_ctl));  in btrfs_write_out_ino_cache()
3475 	ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl,  in btrfs_write_out_ino_cache()
3485 		ret = btrfs_wait_cache_io(root, trans, NULL, &io_ctl, path, 0);  in btrfs_write_out_ino_cache()