60 	struct inode vfs_inode;
63 static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)  in HUGETLBFS_I()  argument
65 	return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);  in HUGETLBFS_I()
90 					struct inode *inode, pgoff_t index)  in hugetlb_set_vma_policy()  argument
92 	vma->vm_policy = mpol_shared_policy_lookup(&HUGETLBFS_I(inode)->policy,  in hugetlb_set_vma_policy()
102 					struct inode *inode, pgoff_t index)  in hugetlb_set_vma_policy()  argument
123 	struct inode *inode = file_inode(file);  in hugetlbfs_file_mmap()  local
144 	mutex_lock(&inode->i_mutex);  in hugetlbfs_file_mmap()
150 	if (hugetlb_reserve_pages(inode,  in hugetlbfs_file_mmap()
157 	if (vma->vm_flags & VM_WRITE && inode->i_size < len)  in hugetlbfs_file_mmap()
158 		inode->i_size = len;  in hugetlbfs_file_mmap()
160 	mutex_unlock(&inode->i_mutex);  in hugetlbfs_file_mmap()
247 	struct inode *inode = mapping->host;  in hugetlbfs_read_iter()  local
260 		isize = i_size_read(inode);  in hugetlbfs_read_iter()
349 static void remove_inode_hugepages(struct inode *inode, loff_t lstart,  in remove_inode_hugepages()  argument
352 	struct hstate *h = hstate_inode(inode);  in remove_inode_hugepages()
353 	struct address_space *mapping = &inode->i_data;  in remove_inode_hugepages()
415 							inode, next,  in remove_inode_hugepages()
418 							inode, rsv_on_error);  in remove_inode_hugepages()
440 		(void)hugetlb_unreserve_pages(inode, start, LONG_MAX, freed);  in remove_inode_hugepages()
443 static void hugetlbfs_evict_inode(struct inode *inode)  in hugetlbfs_evict_inode()  argument
447 	remove_inode_hugepages(inode, 0, LLONG_MAX);  in hugetlbfs_evict_inode()
448 	resv_map = (struct resv_map *)inode->i_mapping->private_data;  in hugetlbfs_evict_inode()
452 	clear_inode(inode);  in hugetlbfs_evict_inode()
493 static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)  in hugetlb_vmtruncate()  argument
496 	struct address_space *mapping = inode->i_mapping;  in hugetlb_vmtruncate()
497 	struct hstate *h = hstate_inode(inode);  in hugetlb_vmtruncate()
502 	i_size_write(inode, offset);  in hugetlb_vmtruncate()
507 	remove_inode_hugepages(inode, offset, LLONG_MAX);  in hugetlb_vmtruncate()
511 static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)  in hugetlbfs_punch_hole()  argument
513 	struct hstate *h = hstate_inode(inode);  in hugetlbfs_punch_hole()
525 		struct address_space *mapping = inode->i_mapping;  in hugetlbfs_punch_hole()
527 		mutex_lock(&inode->i_mutex);  in hugetlbfs_punch_hole()
534 		remove_inode_hugepages(inode, hole_start, hole_end);  in hugetlbfs_punch_hole()
535 		mutex_unlock(&inode->i_mutex);  in hugetlbfs_punch_hole()
544 	struct inode *inode = file_inode(file);  in hugetlbfs_fallocate()  local
545 	struct address_space *mapping = inode->i_mapping;  in hugetlbfs_fallocate()
546 	struct hstate *h = hstate_inode(inode);  in hugetlbfs_fallocate()
559 		return hugetlbfs_punch_hole(inode, offset, len);  in hugetlbfs_fallocate()
569 	mutex_lock(&inode->i_mutex);  in hugetlbfs_fallocate()
572 	error = inode_newsize_ok(inode, offset + len);  in hugetlbfs_fallocate()
606 		hugetlb_set_vma_policy(&pseudo_vma, inode, index);  in hugetlbfs_fallocate()
652 	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)  in hugetlbfs_fallocate()
653 		i_size_write(inode, offset + len);  in hugetlbfs_fallocate()
654 	inode->i_ctime = CURRENT_TIME;  in hugetlbfs_fallocate()
656 	mutex_unlock(&inode->i_mutex);  in hugetlbfs_fallocate()
662 	struct inode *inode = d_inode(dentry);  in hugetlbfs_setattr()  local
663 	struct hstate *h = hstate_inode(inode);  in hugetlbfs_setattr()
667 	BUG_ON(!inode);  in hugetlbfs_setattr()
669 	error = inode_change_ok(inode, attr);  in hugetlbfs_setattr()
677 		error = hugetlb_vmtruncate(inode, attr->ia_size);  in hugetlbfs_setattr()
682 	setattr_copy(inode, attr);  in hugetlbfs_setattr()
683 	mark_inode_dirty(inode);  in hugetlbfs_setattr()
687 static struct inode *hugetlbfs_get_root(struct super_block *sb,  in hugetlbfs_get_root()
690 	struct inode *inode;  in hugetlbfs_get_root()  local
692 	inode = new_inode(sb);  in hugetlbfs_get_root()
693 	if (inode) {  in hugetlbfs_get_root()
695 		inode->i_ino = get_next_ino();  in hugetlbfs_get_root()
696 		inode->i_mode = S_IFDIR | config->mode;  in hugetlbfs_get_root()
697 		inode->i_uid = config->uid;  in hugetlbfs_get_root()
698 		inode->i_gid = config->gid;  in hugetlbfs_get_root()
699 		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;  in hugetlbfs_get_root()
700 		info = HUGETLBFS_I(inode);  in hugetlbfs_get_root()
702 		inode->i_op = &hugetlbfs_dir_inode_operations;  in hugetlbfs_get_root()
703 		inode->i_fop = &simple_dir_operations;  in hugetlbfs_get_root()
705 		inc_nlink(inode);  in hugetlbfs_get_root()
706 		lockdep_annotate_inode_mutex_key(inode);  in hugetlbfs_get_root()
708 	return inode;  in hugetlbfs_get_root()
719 static struct inode *hugetlbfs_get_inode(struct super_block *sb,  in hugetlbfs_get_inode()
720 					struct inode *dir,  in hugetlbfs_get_inode()
723 	struct inode *inode;  in hugetlbfs_get_inode()  local
730 	inode = new_inode(sb);  in hugetlbfs_get_inode()
731 	if (inode) {  in hugetlbfs_get_inode()
733 		inode->i_ino = get_next_ino();  in hugetlbfs_get_inode()
734 		inode_init_owner(inode, dir, mode);  in hugetlbfs_get_inode()
735 		lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,  in hugetlbfs_get_inode()
737 		inode->i_mapping->a_ops = &hugetlbfs_aops;  in hugetlbfs_get_inode()
738 		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;  in hugetlbfs_get_inode()
739 		inode->i_mapping->private_data = resv_map;  in hugetlbfs_get_inode()
740 		info = HUGETLBFS_I(inode);  in hugetlbfs_get_inode()
751 			init_special_inode(inode, mode, dev);  in hugetlbfs_get_inode()
754 			inode->i_op = &hugetlbfs_inode_operations;  in hugetlbfs_get_inode()
755 			inode->i_fop = &hugetlbfs_file_operations;  in hugetlbfs_get_inode()
758 			inode->i_op = &hugetlbfs_dir_inode_operations;  in hugetlbfs_get_inode()
759 			inode->i_fop = &simple_dir_operations;  in hugetlbfs_get_inode()
762 			inc_nlink(inode);  in hugetlbfs_get_inode()
765 			inode->i_op = &page_symlink_inode_operations;  in hugetlbfs_get_inode()
768 		lockdep_annotate_inode_mutex_key(inode);  in hugetlbfs_get_inode()
772 	return inode;  in hugetlbfs_get_inode()
778 static int hugetlbfs_mknod(struct inode *dir,  in hugetlbfs_mknod()
781 	struct inode *inode;  in hugetlbfs_mknod()  local
784 	inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);  in hugetlbfs_mknod()
785 	if (inode) {  in hugetlbfs_mknod()
787 		d_instantiate(dentry, inode);  in hugetlbfs_mknod()
794 static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)  in hugetlbfs_mkdir()
802 static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl)  in hugetlbfs_create()
807 static int hugetlbfs_symlink(struct inode *dir,  in hugetlbfs_symlink()
810 	struct inode *inode;  in hugetlbfs_symlink()  local
813 	inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);  in hugetlbfs_symlink()
814 	if (inode) {  in hugetlbfs_symlink()
816 		error = page_symlink(inode, symname, l);  in hugetlbfs_symlink()
818 			d_instantiate(dentry, inode);  in hugetlbfs_symlink()
821 			iput(inode);  in hugetlbfs_symlink()
923 static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)  in hugetlbfs_alloc_inode()
940 	struct inode *inode = container_of(head, struct inode, i_rcu);  in hugetlbfs_i_callback()  local
941 	kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));  in hugetlbfs_i_callback()
944 static void hugetlbfs_destroy_inode(struct inode *inode)  in hugetlbfs_destroy_inode()  argument
946 	hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));  in hugetlbfs_destroy_inode()
947 	mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);  in hugetlbfs_destroy_inode()
948 	call_rcu(&inode->i_rcu, hugetlbfs_i_callback);  in hugetlbfs_destroy_inode()
1240 	struct inode *inode;  in hugetlb_file_setup()  local
1278 	inode = hugetlbfs_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0);  in hugetlb_file_setup()
1279 	if (!inode)  in hugetlb_file_setup()
1282 		inode->i_flags |= S_PRIVATE;  in hugetlb_file_setup()
1285 	if (hugetlb_reserve_pages(inode, 0,  in hugetlb_file_setup()
1286 			size >> huge_page_shift(hstate_inode(inode)), NULL,  in hugetlb_file_setup()
1290 	d_instantiate(path.dentry, inode);  in hugetlb_file_setup()
1291 	inode->i_size = size;  in hugetlb_file_setup()
1292 	clear_nlink(inode);  in hugetlb_file_setup()
1302 	iput(inode);  in hugetlb_file_setup()