840 	int nid = page_to_nid(page);  in enqueue_huge_page()  local
841 	list_move(&page->lru, &h->hugepage_freelists[nid]);  in enqueue_huge_page()
843 	h->free_huge_pages_node[nid]++;  in enqueue_huge_page()
846 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)  in dequeue_huge_page_node()  argument
850 	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)  in dequeue_huge_page_node()
857 	if (&h->hugepage_freelists[nid] == &page->lru)  in dequeue_huge_page_node()
862 	h->free_huge_pages_node[nid]--;  in dequeue_huge_page_node()
939 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)  in next_node_allowed()  argument
941 	nid = next_node(nid, *nodes_allowed);  in next_node_allowed()
942 	if (nid == MAX_NUMNODES)  in next_node_allowed()
943 		nid = first_node(*nodes_allowed);  in next_node_allowed()
944 	VM_BUG_ON(nid >= MAX_NUMNODES);  in next_node_allowed()
946 	return nid;  in next_node_allowed()
949 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)  in get_valid_node_allowed()  argument
951 	if (!node_isset(nid, *nodes_allowed))  in get_valid_node_allowed()
952 		nid = next_node_allowed(nid, nodes_allowed);  in get_valid_node_allowed()
953 	return nid;  in get_valid_node_allowed()
965 	int nid;  in hstate_next_node_to_alloc()  local
969 	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);  in hstate_next_node_to_alloc()
970 	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);  in hstate_next_node_to_alloc()
972 	return nid;  in hstate_next_node_to_alloc()
983 	int nid;  in hstate_next_node_to_free()  local
987 	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);  in hstate_next_node_to_free()
988 	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);  in hstate_next_node_to_free()
990 	return nid;  in hstate_next_node_to_free()
1066 static struct page *alloc_gigantic_page(int nid, unsigned int order)  in alloc_gigantic_page()  argument
1072 	z = NODE_DATA(nid)->node_zones;  in alloc_gigantic_page()
1073 	for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {  in alloc_gigantic_page()
1101 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1104 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)  in alloc_fresh_gigantic_page_node()  argument
1108 	page = alloc_gigantic_page(nid, huge_page_order(h));  in alloc_fresh_gigantic_page_node()
1111 		prep_new_huge_page(h, page, nid);  in alloc_fresh_gigantic_page_node()
1211 	int nid = page_to_nid(page);  in free_huge_page()  local
1238 	if (h->surplus_huge_pages_node[nid]) {  in free_huge_page()
1243 		h->surplus_huge_pages_node[nid]--;  in free_huge_page()
1251 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)  in prep_new_huge_page()  argument
1258 	h->nr_huge_pages_node[nid]++;  in prep_new_huge_page()
1336 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)  in alloc_fresh_huge_page_node()  argument
1340 	page = __alloc_pages_node(nid,  in alloc_fresh_huge_page_node()
1345 		prep_new_huge_page(h, page, nid);  in alloc_fresh_huge_page_node()
1420 		int nid = page_to_nid(page);  in dissolve_free_huge_page()  local
1423 		h->free_huge_pages_node[nid]--;  in dissolve_free_huge_page()
1456 		struct vm_area_struct *vma, unsigned long addr, int nid)  in __hugetlb_alloc_buddy_huge_page()  argument
1477 		if (nid != NUMA_NO_NODE)  in __hugetlb_alloc_buddy_huge_page()
1483 		return alloc_pages_node(nid, gfp, order);  in __hugetlb_alloc_buddy_huge_page()
1521 		struct vm_area_struct *vma, unsigned long addr, int nid)  in __alloc_buddy_huge_page()  argument
1536 		VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);  in __alloc_buddy_huge_page()
1571 	page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);  in __alloc_buddy_huge_page()
1601 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)  in __alloc_buddy_huge_page_no_mpol()  argument
1605 	return __alloc_buddy_huge_page(h, NULL, addr, nid);  in __alloc_buddy_huge_page_no_mpol()
1623 struct page *alloc_huge_page_node(struct hstate *h, int nid)  in alloc_huge_page_node()  argument
1629 		page = dequeue_huge_page_node(h, nid);  in alloc_huge_page_node()
1633 		page = __alloc_buddy_huge_page_no_mpol(h, nid);  in alloc_huge_page_node()
2250 	int nid;  in nr_hugepages_show_common()  local
2252 	h = kobj_to_hstate(kobj, &nid);  in nr_hugepages_show_common()
2253 	if (nid == NUMA_NO_NODE)  in nr_hugepages_show_common()
2256 		nr_huge_pages = h->nr_huge_pages_node[nid];  in nr_hugepages_show_common()
2262 					   struct hstate *h, int nid,  in __nr_hugepages_store_common()  argument
2273 	if (nid == NUMA_NO_NODE) {  in __nr_hugepages_store_common()
2287 		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];  in __nr_hugepages_store_common()
2288 		init_nodemask_of_node(nodes_allowed, nid);  in __nr_hugepages_store_common()
2309 	int nid;  in nr_hugepages_store_common()  local
2316 	h = kobj_to_hstate(kobj, &nid);  in nr_hugepages_store_common()
2317 	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);  in nr_hugepages_store_common()
2388 	int nid;  in free_hugepages_show()  local
2390 	h = kobj_to_hstate(kobj, &nid);  in free_hugepages_show()
2391 	if (nid == NUMA_NO_NODE)  in free_hugepages_show()
2394 		free_huge_pages = h->free_huge_pages_node[nid];  in free_hugepages_show()
2413 	int nid;  in surplus_hugepages_show()  local
2415 	h = kobj_to_hstate(kobj, &nid);  in surplus_hugepages_show()
2416 	if (nid == NUMA_NO_NODE)  in surplus_hugepages_show()
2419 		surplus_huge_pages = h->surplus_huge_pages_node[nid];  in surplus_hugepages_show()
2511 	int nid;  in kobj_to_node_hstate()  local
2513 	for (nid = 0; nid < nr_node_ids; nid++) {  in kobj_to_node_hstate()
2514 		struct node_hstate *nhs = &node_hstates[nid];  in kobj_to_node_hstate()
2519 					*nidp = nid;  in kobj_to_node_hstate()
2558 	int nid;  in hugetlb_unregister_all_nodes()  local
2568 	for (nid = 0; nid < nr_node_ids; nid++)  in hugetlb_unregister_all_nodes()
2569 		hugetlb_unregister_node(node_devices[nid]);  in hugetlb_unregister_all_nodes()
2610 	int nid;  in hugetlb_register_all_nodes()  local
2612 	for_each_node_state(nid, N_MEMORY) {  in hugetlb_register_all_nodes()
2613 		struct node *node = node_devices[nid];  in hugetlb_register_all_nodes()
2614 		if (node->dev.id == nid)  in hugetlb_register_all_nodes()
2871 int hugetlb_report_node_meminfo(int nid, char *buf)  in hugetlb_report_node_meminfo()  argument
2880 		nid, h->nr_huge_pages_node[nid],  in hugetlb_report_node_meminfo()
2881 		nid, h->free_huge_pages_node[nid],  in hugetlb_report_node_meminfo()
2882 		nid, h->surplus_huge_pages_node[nid]);  in hugetlb_report_node_meminfo()
2888 	int nid;  in hugetlb_show_meminfo()  local
2893 	for_each_node_state(nid, N_MEMORY)  in hugetlb_show_meminfo()
2896 				nid,  in hugetlb_show_meminfo()
2897 				h->nr_huge_pages_node[nid],  in hugetlb_show_meminfo()
2898 				h->free_huge_pages_node[nid],  in hugetlb_show_meminfo()
2899 				h->surplus_huge_pages_node[nid],  in hugetlb_show_meminfo()
4390 	int nid = page_to_nid(hpage);  in dequeue_hwpoisoned_huge_page()  local
4408 		h->free_huge_pages_node[nid]--;  in dequeue_hwpoisoned_huge_page()