397 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)  in dup_mmap()  argument
407 	uprobe_dup_mmap(oldmm, mm);  in dup_mmap()
411 	down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);  in dup_mmap()
414 	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));  in dup_mmap()
416 	mm->total_vm = oldmm->total_vm;  in dup_mmap()
417 	mm->shared_vm = oldmm->shared_vm;  in dup_mmap()
418 	mm->exec_vm = oldmm->exec_vm;  in dup_mmap()
419 	mm->stack_vm = oldmm->stack_vm;  in dup_mmap()
421 	rb_link = &mm->mm_rb.rb_node;  in dup_mmap()
423 	pprev = &mm->mmap;  in dup_mmap()
424 	retval = ksm_fork(mm, oldmm);  in dup_mmap()
427 	retval = khugepaged_fork(mm, oldmm);  in dup_mmap()
436 			vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,  in dup_mmap()
456 		tmp->vm_mm = mm;  in dup_mmap()
498 		__vma_link_rb(mm, tmp, rb_link, rb_parent);  in dup_mmap()
502 		mm->map_count++;  in dup_mmap()
503 		retval = copy_page_range(mm, oldmm, mpnt);  in dup_mmap()
512 	arch_dup_mmap(oldmm, mm);  in dup_mmap()
515 	up_write(&mm->mmap_sem);  in dup_mmap()
530 static inline int mm_alloc_pgd(struct mm_struct *mm)  in mm_alloc_pgd()  argument
532 	mm->pgd = pgd_alloc(mm);  in mm_alloc_pgd()
533 	if (unlikely(!mm->pgd))  in mm_alloc_pgd()
538 static inline void mm_free_pgd(struct mm_struct *mm)  in mm_free_pgd()  argument
540 	pgd_free(mm, mm->pgd);  in mm_free_pgd()
543 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)  in dup_mmap()  argument
546 	RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));  in dup_mmap()
550 #define mm_alloc_pgd(mm)	(0)  argument
551 #define mm_free_pgd(mm)  argument
557 #define free_mm(mm)	(kmem_cache_free(mm_cachep, (mm)))  argument
573 static void mm_init_aio(struct mm_struct *mm)  in mm_init_aio()  argument
576 	spin_lock_init(&mm->ioctx_lock);  in mm_init_aio()
577 	mm->ioctx_table = NULL;  in mm_init_aio()
581 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)  in mm_init_owner()  argument
584 	mm->owner = p;  in mm_init_owner()
588 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)  in mm_init()  argument
590 	mm->mmap = NULL;  in mm_init()
591 	mm->mm_rb = RB_ROOT;  in mm_init()
592 	mm->vmacache_seqnum = 0;  in mm_init()
593 	atomic_set(&mm->mm_users, 1);  in mm_init()
594 	atomic_set(&mm->mm_count, 1);  in mm_init()
595 	init_rwsem(&mm->mmap_sem);  in mm_init()
596 	INIT_LIST_HEAD(&mm->mmlist);  in mm_init()
597 	mm->core_state = NULL;  in mm_init()
598 	atomic_long_set(&mm->nr_ptes, 0);  in mm_init()
599 	mm_nr_pmds_init(mm);  in mm_init()
600 	mm->map_count = 0;  in mm_init()
601 	mm->locked_vm = 0;  in mm_init()
602 	mm->pinned_vm = 0;  in mm_init()
603 	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));  in mm_init()
604 	spin_lock_init(&mm->page_table_lock);  in mm_init()
605 	mm_init_cpumask(mm);  in mm_init()
606 	mm_init_aio(mm);  in mm_init()
607 	mm_init_owner(mm, p);  in mm_init()
608 	mmu_notifier_mm_init(mm);  in mm_init()
609 	clear_tlb_flush_pending(mm);  in mm_init()
611 	mm->pmd_huge_pte = NULL;  in mm_init()
614 	if (current->mm) {  in mm_init()
615 		mm->flags = current->mm->flags & MMF_INIT_MASK;  in mm_init()
616 		mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;  in mm_init()
618 		mm->flags = default_dump_filter;  in mm_init()
619 		mm->def_flags = 0;  in mm_init()
622 	if (mm_alloc_pgd(mm))  in mm_init()
625 	if (init_new_context(p, mm))  in mm_init()
628 	return mm;  in mm_init()
631 	mm_free_pgd(mm);  in mm_init()
633 	free_mm(mm);  in mm_init()
637 static void check_mm(struct mm_struct *mm)  in check_mm()  argument
642 		long x = atomic_long_read(&mm->rss_stat.count[i]);  in check_mm()
646 					  "mm:%p idx:%d val:%ld\n", mm, i, x);  in check_mm()
649 	if (atomic_long_read(&mm->nr_ptes))  in check_mm()
651 				atomic_long_read(&mm->nr_ptes));  in check_mm()
652 	if (mm_nr_pmds(mm))  in check_mm()
654 				mm_nr_pmds(mm));  in check_mm()
657 	VM_BUG_ON_MM(mm->pmd_huge_pte, mm);  in check_mm()
666 	struct mm_struct *mm;  in mm_alloc()  local
668 	mm = allocate_mm();  in mm_alloc()
669 	if (!mm)  in mm_alloc()
672 	memset(mm, 0, sizeof(*mm));  in mm_alloc()
673 	return mm_init(mm, current);  in mm_alloc()
681 void __mmdrop(struct mm_struct *mm)  in __mmdrop()  argument
683 	BUG_ON(mm == &init_mm);  in __mmdrop()
684 	mm_free_pgd(mm);  in __mmdrop()
685 	destroy_context(mm);  in __mmdrop()
686 	mmu_notifier_mm_destroy(mm);  in __mmdrop()
687 	check_mm(mm);  in __mmdrop()
688 	free_mm(mm);  in __mmdrop()
695 void mmput(struct mm_struct *mm)  in mmput()  argument
699 	if (atomic_dec_and_test(&mm->mm_users)) {  in mmput()
700 		uprobe_clear_state(mm);  in mmput()
701 		exit_aio(mm);  in mmput()
702 		ksm_exit(mm);  in mmput()
703 		khugepaged_exit(mm); /* must run before exit_mmap */  in mmput()
704 		exit_mmap(mm);  in mmput()
705 		set_mm_exe_file(mm, NULL);  in mmput()
706 		if (!list_empty(&mm->mmlist)) {  in mmput()
708 			list_del(&mm->mmlist);  in mmput()
711 		if (mm->binfmt)  in mmput()
712 			module_put(mm->binfmt->module);  in mmput()
713 		mmdrop(mm);  in mmput()
729 void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)  in set_mm_exe_file()  argument
738 	old_exe_file = rcu_dereference_raw(mm->exe_file);  in set_mm_exe_file()
742 	rcu_assign_pointer(mm->exe_file, new_exe_file);  in set_mm_exe_file()
753 struct file *get_mm_exe_file(struct mm_struct *mm)  in get_mm_exe_file()  argument
758 	exe_file = rcu_dereference(mm->exe_file);  in get_mm_exe_file()
777 	struct mm_struct *mm;  in get_task_mm()  local
780 	mm = task->mm;  in get_task_mm()
781 	if (mm) {  in get_task_mm()
783 			mm = NULL;  in get_task_mm()
785 			atomic_inc(&mm->mm_users);  in get_task_mm()
788 	return mm;  in get_task_mm()
794 	struct mm_struct *mm;  in mm_access()  local
801 	mm = get_task_mm(task);  in mm_access()
802 	if (mm && mm != current->mm &&  in mm_access()
804 		mmput(mm);  in mm_access()
805 		mm = ERR_PTR(-EACCES);  in mm_access()
809 	return mm;  in mm_access()
857 void mm_release(struct task_struct *tsk, struct mm_struct *mm)  in mm_release()  argument
878 	deactivate_mm(tsk, mm);  in mm_release()
889 		    atomic_read(&mm->mm_users) > 1) {  in mm_release()
915 	struct mm_struct *mm, *oldmm = current->mm;  in dup_mm()  local
918 	mm = allocate_mm();  in dup_mm()
919 	if (!mm)  in dup_mm()
922 	memcpy(mm, oldmm, sizeof(*mm));  in dup_mm()
924 	if (!mm_init(mm, tsk))  in dup_mm()
927 	err = dup_mmap(mm, oldmm);  in dup_mm()
931 	mm->hiwater_rss = get_mm_rss(mm);  in dup_mm()
932 	mm->hiwater_vm = mm->total_vm;  in dup_mm()
934 	if (mm->binfmt && !try_module_get(mm->binfmt->module))  in dup_mm()
937 	return mm;  in dup_mm()
941 	mm->binfmt = NULL;  in dup_mm()
942 	mmput(mm);  in dup_mm()
950 	struct mm_struct *mm, *oldmm;  in copy_mm()  local
959 	tsk->mm = NULL;  in copy_mm()
967 	oldmm = current->mm;  in copy_mm()
976 		mm = oldmm;  in copy_mm()
981 	mm = dup_mm(tsk);  in copy_mm()
982 	if (!mm)  in copy_mm()
986 	tsk->mm = mm;  in copy_mm()
987 	tsk->active_mm = mm;  in copy_mm()
1632 	if (p->mm)  in copy_process()
1633 		mmput(p->mm);  in copy_process()