591 	int nid = page_to_nid(page);  in enqueue_huge_page()  local
592 	list_move(&page->lru, &h->hugepage_freelists[nid]);  in enqueue_huge_page()
594 	h->free_huge_pages_node[nid]++;  in enqueue_huge_page()
597 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)  in dequeue_huge_page_node()  argument
601 	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)  in dequeue_huge_page_node()
608 	if (&h->hugepage_freelists[nid] == &page->lru)  in dequeue_huge_page_node()
613 	h->free_huge_pages_node[nid]--;  in dequeue_huge_page_node()
690 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)  in next_node_allowed()  argument
692 	nid = next_node(nid, *nodes_allowed);  in next_node_allowed()
693 	if (nid == MAX_NUMNODES)  in next_node_allowed()
694 		nid = first_node(*nodes_allowed);  in next_node_allowed()
695 	VM_BUG_ON(nid >= MAX_NUMNODES);  in next_node_allowed()
697 	return nid;  in next_node_allowed()
700 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)  in get_valid_node_allowed()  argument
702 	if (!node_isset(nid, *nodes_allowed))  in get_valid_node_allowed()
703 		nid = next_node_allowed(nid, nodes_allowed);  in get_valid_node_allowed()
704 	return nid;  in get_valid_node_allowed()
716 	int nid;  in hstate_next_node_to_alloc()  local
720 	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);  in hstate_next_node_to_alloc()
721 	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);  in hstate_next_node_to_alloc()
723 	return nid;  in hstate_next_node_to_alloc()
734 	int nid;  in hstate_next_node_to_free()  local
738 	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);  in hstate_next_node_to_free()
739 	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);  in hstate_next_node_to_free()
741 	return nid;  in hstate_next_node_to_free()
818 static struct page *alloc_gigantic_page(int nid, unsigned int order)  in alloc_gigantic_page()  argument
824 	z = NODE_DATA(nid)->node_zones;  in alloc_gigantic_page()
825 	for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {  in alloc_gigantic_page()
853 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
856 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)  in alloc_fresh_gigantic_page_node()  argument
860 	page = alloc_gigantic_page(nid, huge_page_order(h));  in alloc_fresh_gigantic_page_node()
863 		prep_new_huge_page(h, page, nid);  in alloc_fresh_gigantic_page_node()
964 	int nid = page_to_nid(page);  in free_huge_page()  local
991 	if (h->surplus_huge_pages_node[nid]) {  in free_huge_page()
996 		h->surplus_huge_pages_node[nid]--;  in free_huge_page()
1004 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)  in prep_new_huge_page()  argument
1011 	h->nr_huge_pages_node[nid]++;  in prep_new_huge_page()
1092 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)  in alloc_fresh_huge_page_node()  argument
1096 	page = alloc_pages_exact_node(nid,  in alloc_fresh_huge_page_node()
1105 		prep_new_huge_page(h, page, nid);  in alloc_fresh_huge_page_node()
1180 		int nid = page_to_nid(page);  in dissolve_free_huge_page()  local
1183 		h->free_huge_pages_node[nid]--;  in dissolve_free_huge_page()
1206 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)  in alloc_buddy_huge_page()  argument
1247 	if (nid == NUMA_NO_NODE)  in alloc_buddy_huge_page()
1252 		page = alloc_pages_exact_node(nid,  in alloc_buddy_huge_page()
1288 struct page *alloc_huge_page_node(struct hstate *h, int nid)  in alloc_huge_page_node()  argument
1294 		page = dequeue_huge_page_node(h, nid);  in alloc_huge_page_node()
1298 		page = alloc_buddy_huge_page(h, nid);  in alloc_huge_page_node()
1836 	int nid;  in nr_hugepages_show_common()  local
1838 	h = kobj_to_hstate(kobj, &nid);  in nr_hugepages_show_common()
1839 	if (nid == NUMA_NO_NODE)  in nr_hugepages_show_common()
1842 		nr_huge_pages = h->nr_huge_pages_node[nid];  in nr_hugepages_show_common()
1848 					   struct hstate *h, int nid,  in __nr_hugepages_store_common()  argument
1859 	if (nid == NUMA_NO_NODE) {  in __nr_hugepages_store_common()
1873 		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];  in __nr_hugepages_store_common()
1874 		init_nodemask_of_node(nodes_allowed, nid);  in __nr_hugepages_store_common()
1895 	int nid;  in nr_hugepages_store_common()  local
1902 	h = kobj_to_hstate(kobj, &nid);  in nr_hugepages_store_common()
1903 	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);  in nr_hugepages_store_common()
1974 	int nid;  in free_hugepages_show()  local
1976 	h = kobj_to_hstate(kobj, &nid);  in free_hugepages_show()
1977 	if (nid == NUMA_NO_NODE)  in free_hugepages_show()
1980 		free_huge_pages = h->free_huge_pages_node[nid];  in free_hugepages_show()
1999 	int nid;  in surplus_hugepages_show()  local
2001 	h = kobj_to_hstate(kobj, &nid);  in surplus_hugepages_show()
2002 	if (nid == NUMA_NO_NODE)  in surplus_hugepages_show()
2005 		surplus_huge_pages = h->surplus_huge_pages_node[nid];  in surplus_hugepages_show()
2097 	int nid;  in kobj_to_node_hstate()  local
2099 	for (nid = 0; nid < nr_node_ids; nid++) {  in kobj_to_node_hstate()
2100 		struct node_hstate *nhs = &node_hstates[nid];  in kobj_to_node_hstate()
2105 					*nidp = nid;  in kobj_to_node_hstate()
2144 	int nid;  in hugetlb_unregister_all_nodes()  local
2154 	for (nid = 0; nid < nr_node_ids; nid++)  in hugetlb_unregister_all_nodes()
2155 		hugetlb_unregister_node(node_devices[nid]);  in hugetlb_unregister_all_nodes()
2196 	int nid;  in hugetlb_register_all_nodes()  local
2198 	for_each_node_state(nid, N_MEMORY) {  in hugetlb_register_all_nodes()
2199 		struct node *node = node_devices[nid];  in hugetlb_register_all_nodes()
2200 		if (node->dev.id == nid)  in hugetlb_register_all_nodes()
2457 int hugetlb_report_node_meminfo(int nid, char *buf)  in hugetlb_report_node_meminfo()  argument
2466 		nid, h->nr_huge_pages_node[nid],  in hugetlb_report_node_meminfo()
2467 		nid, h->free_huge_pages_node[nid],  in hugetlb_report_node_meminfo()
2468 		nid, h->surplus_huge_pages_node[nid]);  in hugetlb_report_node_meminfo()
2474 	int nid;  in hugetlb_show_meminfo()  local
2479 	for_each_node_state(nid, N_MEMORY)  in hugetlb_show_meminfo()
2482 				nid,  in hugetlb_show_meminfo()
2483 				h->nr_huge_pages_node[nid],  in hugetlb_show_meminfo()
2484 				h->free_huge_pages_node[nid],  in hugetlb_show_meminfo()
2485 				h->surplus_huge_pages_node[nid],  in hugetlb_show_meminfo()
3918 	int nid = page_to_nid(hpage);  in dequeue_hwpoisoned_huge_page()  local
3936 		h->free_huge_pages_node[nid]--;  in dequeue_hwpoisoned_huge_page()