128 	struct mempolicy *pol = p->mempolicy;  in get_task_policy()  local
131 	if (pol)  in get_task_policy()
132 		return pol;  in get_task_policy()
136 		pol = &preferred_node_policy[node];  in get_task_policy()
138 		if (pol->mode)  in get_task_policy()
139 			return pol;  in get_task_policy()
146 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
161 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
165 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)  in mpol_store_user_nodemask()  argument
167 	return pol->flags & MPOL_MODE_FLAGS;  in mpol_store_user_nodemask()
178 static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_interleave()  argument
182 	pol->v.nodes = *nodes;  in mpol_new_interleave()
186 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_preferred()  argument
189 		pol->flags |= MPOL_F_LOCAL;	/* local allocation */  in mpol_new_preferred()
193 		pol->v.preferred_node = first_node(*nodes);  in mpol_new_preferred()
197 static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_bind()  argument
201 	pol->v.nodes = *nodes;  in mpol_new_bind()
214 static int mpol_set_nodemask(struct mempolicy *pol,  in mpol_set_nodemask()  argument
220 	if (pol == NULL)  in mpol_set_nodemask()
227 	if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))  in mpol_set_nodemask()
230 		if (pol->flags & MPOL_F_RELATIVE_NODES)  in mpol_set_nodemask()
235 		if (mpol_store_user_nodemask(pol))  in mpol_set_nodemask()
236 			pol->w.user_nodemask = *nodes;  in mpol_set_nodemask()
238 			pol->w.cpuset_mems_allowed =  in mpol_set_nodemask()
243 		ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);  in mpol_set_nodemask()
245 		ret = mpol_ops[pol->mode].create(pol, NULL);  in mpol_set_nodemask()
303 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,  in mpol_rebind_default()  argument
314 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,  in mpol_rebind_nodemask()  argument
319 	if (pol->flags & MPOL_F_STATIC_NODES)  in mpol_rebind_nodemask()
320 		nodes_and(tmp, pol->w.user_nodemask, *nodes);  in mpol_rebind_nodemask()
321 	else if (pol->flags & MPOL_F_RELATIVE_NODES)  in mpol_rebind_nodemask()
322 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_nodemask()
329 			nodes_remap(tmp, pol->v.nodes,  in mpol_rebind_nodemask()
330 					pol->w.cpuset_mems_allowed, *nodes);  in mpol_rebind_nodemask()
331 			pol->w.cpuset_mems_allowed = step ? tmp : *nodes;  in mpol_rebind_nodemask()
333 			tmp = pol->w.cpuset_mems_allowed;  in mpol_rebind_nodemask()
334 			pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_nodemask()
343 		nodes_or(pol->v.nodes, pol->v.nodes, tmp);  in mpol_rebind_nodemask()
345 		pol->v.nodes = tmp;  in mpol_rebind_nodemask()
358 static void mpol_rebind_preferred(struct mempolicy *pol,  in mpol_rebind_preferred()  argument
364 	if (pol->flags & MPOL_F_STATIC_NODES) {  in mpol_rebind_preferred()
365 		int node = first_node(pol->w.user_nodemask);  in mpol_rebind_preferred()
368 			pol->v.preferred_node = node;  in mpol_rebind_preferred()
369 			pol->flags &= ~MPOL_F_LOCAL;  in mpol_rebind_preferred()
371 			pol->flags |= MPOL_F_LOCAL;  in mpol_rebind_preferred()
372 	} else if (pol->flags & MPOL_F_RELATIVE_NODES) {  in mpol_rebind_preferred()
373 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_preferred()
374 		pol->v.preferred_node = first_node(tmp);  in mpol_rebind_preferred()
375 	} else if (!(pol->flags & MPOL_F_LOCAL)) {  in mpol_rebind_preferred()
376 		pol->v.preferred_node = node_remap(pol->v.preferred_node,  in mpol_rebind_preferred()
377 						   pol->w.cpuset_mems_allowed,  in mpol_rebind_preferred()
379 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_preferred()
399 static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,  in mpol_rebind_policy()  argument
402 	if (!pol)  in mpol_rebind_policy()
404 	if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&  in mpol_rebind_policy()
405 	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))  in mpol_rebind_policy()
408 	if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))  in mpol_rebind_policy()
411 	if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))  in mpol_rebind_policy()
415 		pol->flags |= MPOL_F_REBINDING;  in mpol_rebind_policy()
417 		pol->flags &= ~MPOL_F_REBINDING;  in mpol_rebind_policy()
421 	mpol_ops[pol->mode].rebind(pol, newmask, step);  in mpol_rebind_policy()
660 						struct mempolicy *pol)  in vma_replace_policy()  argument
671 	new = mpol_dup(pol);  in vma_replace_policy()
836 	struct mempolicy *pol = current->mempolicy;  in do_get_mempolicy()  local
865 			pol = vma->vm_ops->get_policy(vma, addr);  in do_get_mempolicy()
867 			pol = vma->vm_policy;  in do_get_mempolicy()
871 	if (!pol)  in do_get_mempolicy()
872 		pol = &default_policy;	/* indicates default behavior */  in do_get_mempolicy()
880 		} else if (pol == current->mempolicy &&  in do_get_mempolicy()
881 				pol->mode == MPOL_INTERLEAVE) {  in do_get_mempolicy()
888 		*policy = pol == &default_policy ? MPOL_DEFAULT :  in do_get_mempolicy()
889 						pol->mode;  in do_get_mempolicy()
894 		*policy |= (pol->flags & MPOL_MODE_FLAGS);  in do_get_mempolicy()
904 		if (mpol_store_user_nodemask(pol)) {  in do_get_mempolicy()
905 			*nmask = pol->w.user_nodemask;  in do_get_mempolicy()
908 			get_policy_nodemask(pol, nmask);  in do_get_mempolicy()
914 	mpol_cond_put(pol);  in do_get_mempolicy()
1544 	struct mempolicy *pol = NULL;  in __get_vma_policy()  local
1548 			pol = vma->vm_ops->get_policy(vma, addr);  in __get_vma_policy()
1550 			pol = vma->vm_policy;  in __get_vma_policy()
1558 			if (mpol_needs_cond_ref(pol))  in __get_vma_policy()
1559 				mpol_get(pol);  in __get_vma_policy()
1563 	return pol;  in __get_vma_policy()
1581 	struct mempolicy *pol = __get_vma_policy(vma, addr);  in get_vma_policy()  local
1583 	if (!pol)  in get_vma_policy()
1584 		pol = get_task_policy(current);  in get_vma_policy()
1586 	return pol;  in get_vma_policy()
1591 	struct mempolicy *pol;  in vma_policy_mof()  local
1596 		pol = vma->vm_ops->get_policy(vma, vma->vm_start);  in vma_policy_mof()
1597 		if (pol && (pol->flags & MPOL_F_MOF))  in vma_policy_mof()
1599 		mpol_cond_put(pol);  in vma_policy_mof()
1604 	pol = vma->vm_policy;  in vma_policy_mof()
1605 	if (!pol)  in vma_policy_mof()
1606 		pol = get_task_policy(current);  in vma_policy_mof()
1608 	return pol->flags & MPOL_F_MOF;  in vma_policy_mof()
1734 static unsigned offset_il_node(struct mempolicy *pol,  in offset_il_node()  argument
1737 	unsigned nnodes = nodes_weight(pol->v.nodes);  in offset_il_node()
1747 		nid = next_node(nid, pol->v.nodes);  in offset_il_node()
1754 static inline unsigned interleave_nid(struct mempolicy *pol,  in interleave_nid()  argument
1770 		return offset_il_node(pol, vma, off);  in interleave_nid()
1772 		return interleave_nodes(pol);  in interleave_nid()
1962 	struct mempolicy *pol;  in alloc_pages_vma()  local
1969 	pol = get_vma_policy(vma, addr);  in alloc_pages_vma()
1972 	if (pol->mode == MPOL_INTERLEAVE) {  in alloc_pages_vma()
1975 		nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);  in alloc_pages_vma()
1976 		mpol_cond_put(pol);  in alloc_pages_vma()
1994 		if (pol->mode == MPOL_PREFERRED &&  in alloc_pages_vma()
1995 						!(pol->flags & MPOL_F_LOCAL))  in alloc_pages_vma()
1996 			hpage_node = pol->v.preferred_node;  in alloc_pages_vma()
1998 		nmask = policy_nodemask(gfp, pol);  in alloc_pages_vma()
2000 			mpol_cond_put(pol);  in alloc_pages_vma()
2007 	nmask = policy_nodemask(gfp, pol);  in alloc_pages_vma()
2008 	zl = policy_zonelist(gfp, pol, node);  in alloc_pages_vma()
2009 	mpol_cond_put(pol);  in alloc_pages_vma()
2038 	struct mempolicy *pol = &default_policy;  in alloc_pages_current()  local
2043 		pol = get_task_policy(current);  in alloc_pages_current()
2052 	if (pol->mode == MPOL_INTERLEAVE)  in alloc_pages_current()
2053 		page = alloc_page_interleave(gfp, order, interleave_nodes(pol));  in alloc_pages_current()
2056 				policy_zonelist(gfp, pol, numa_node_id()),  in alloc_pages_current()
2057 				policy_nodemask(gfp, pol));  in alloc_pages_current()
2068 	struct mempolicy *pol = mpol_dup(vma_policy(src));  in vma_dup_policy()  local
2070 	if (IS_ERR(pol))  in vma_dup_policy()
2071 		return PTR_ERR(pol);  in vma_dup_policy()
2072 	dst->vm_policy = pol;  in vma_dup_policy()
2209 	struct mempolicy *pol = NULL;  in mpol_shared_policy_lookup()  local
2218 		pol = sn->policy;  in mpol_shared_policy_lookup()
2221 	return pol;  in mpol_shared_policy_lookup()
2249 	struct mempolicy *pol;  in mpol_misplaced()  local
2260 	pol = get_vma_policy(vma, addr);  in mpol_misplaced()
2261 	if (!(pol->flags & MPOL_F_MOF))  in mpol_misplaced()
2264 	switch (pol->mode) {  in mpol_misplaced()
2271 		polnid = offset_il_node(pol, vma, pgoff);  in mpol_misplaced()
2275 		if (pol->flags & MPOL_F_LOCAL)  in mpol_misplaced()
2278 			polnid = pol->v.preferred_node;  in mpol_misplaced()
2288 		if (node_isset(curnid, pol->v.nodes))  in mpol_misplaced()
2293 				&pol->v.nodes, &zone);  in mpol_misplaced()
2302 	if (pol->flags & MPOL_F_MORON) {  in mpol_misplaced()
2312 	mpol_cond_put(pol);  in mpol_misplaced()
2325 			unsigned long end, struct mempolicy *pol)  in sp_node_init()  argument
2329 	node->policy = pol;  in sp_node_init()
2333 				struct mempolicy *pol)  in sp_alloc()  argument
2342 	newpol = mpol_dup(pol);  in sp_alloc()
2780 void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)  in mpol_to_str()  argument
2787 	if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {  in mpol_to_str()
2788 		mode = pol->mode;  in mpol_to_str()
2789 		flags = pol->flags;  in mpol_to_str()
2799 			node_set(pol->v.preferred_node, nodes);  in mpol_to_str()
2803 		nodes = pol->v.nodes;  in mpol_to_str()