28 	.bdata = &bootmem_node_data[0]
78 static void __init link_bootmem(bootmem_data_t *bdata)  in link_bootmem()  argument
83 		if (bdata->node_min_pfn < ent->node_min_pfn) {  in link_bootmem()
84 			list_add_tail(&bdata->list, &ent->list);  in link_bootmem()
89 	list_add_tail(&bdata->list, &bdata_list);  in link_bootmem()
95 static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,  in init_bootmem_core()  argument
101 	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));  in init_bootmem_core()
102 	bdata->node_min_pfn = start;  in init_bootmem_core()
103 	bdata->node_low_pfn = end;  in init_bootmem_core()
104 	link_bootmem(bdata);  in init_bootmem_core()
111 	memset(bdata->node_bootmem_map, 0xff, mapsize);  in init_bootmem_core()
114 		bdata - bootmem_node_data, start, mapstart, end, mapsize);  in init_bootmem_core()
131 	return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);  in init_bootmem_node()
145 	return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);  in init_bootmem()
172 static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)  in free_all_bootmem_core()  argument
177 	if (!bdata->node_bootmem_map)  in free_all_bootmem_core()
180 	map = bdata->node_bootmem_map;  in free_all_bootmem_core()
181 	start = bdata->node_min_pfn;  in free_all_bootmem_core()
182 	end = bdata->node_low_pfn;  in free_all_bootmem_core()
185 		bdata - bootmem_node_data, start, end);  in free_all_bootmem_core()
191 		idx = start - bdata->node_min_pfn;  in free_all_bootmem_core()
232 	cur = bdata->node_min_pfn;  in free_all_bootmem_core()
233 	page = virt_to_page(bdata->node_bootmem_map);  in free_all_bootmem_core()
234 	pages = bdata->node_low_pfn - bdata->node_min_pfn;  in free_all_bootmem_core()
240 	bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);  in free_all_bootmem_core()
276 	bootmem_data_t *bdata;  in free_all_bootmem()  local
280 	list_for_each_entry(bdata, &bdata_list, list)  in free_all_bootmem()
281 		total_pages += free_all_bootmem_core(bdata);  in free_all_bootmem()
288 static void __init __free(bootmem_data_t *bdata,  in __free()  argument
293 	bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,  in __free()
294 		sidx + bdata->node_min_pfn,  in __free()
295 		eidx + bdata->node_min_pfn);  in __free()
297 	if (bdata->hint_idx > sidx)  in __free()
298 		bdata->hint_idx = sidx;  in __free()
301 		if (!test_and_clear_bit(idx, bdata->node_bootmem_map))  in __free()
305 static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx,  in __reserve()  argument
312 		bdata - bootmem_node_data,  in __reserve()
313 		sidx + bdata->node_min_pfn,  in __reserve()
314 		eidx + bdata->node_min_pfn,  in __reserve()
318 		if (test_and_set_bit(idx, bdata->node_bootmem_map)) {  in __reserve()
320 				__free(bdata, sidx, idx);  in __reserve()
324 				idx + bdata->node_min_pfn);  in __reserve()
329 static int __init mark_bootmem_node(bootmem_data_t *bdata,  in mark_bootmem_node()  argument
336 		bdata - bootmem_node_data, start, end, reserve, flags);  in mark_bootmem_node()
338 	BUG_ON(start < bdata->node_min_pfn);  in mark_bootmem_node()
339 	BUG_ON(end > bdata->node_low_pfn);  in mark_bootmem_node()
341 	sidx = start - bdata->node_min_pfn;  in mark_bootmem_node()
342 	eidx = end - bdata->node_min_pfn;  in mark_bootmem_node()
345 		return __reserve(bdata, sidx, eidx, flags);  in mark_bootmem_node()
347 		__free(bdata, sidx, eidx);  in mark_bootmem_node()
355 	bootmem_data_t *bdata;  in mark_bootmem()  local
358 	list_for_each_entry(bdata, &bdata_list, list) {  in mark_bootmem()
362 		if (pos < bdata->node_min_pfn ||  in mark_bootmem()
363 		    pos >= bdata->node_low_pfn) {  in mark_bootmem()
368 		max = min(bdata->node_low_pfn, end);  in mark_bootmem()
370 		err = mark_bootmem_node(bdata, pos, max, reserve, flags);  in mark_bootmem()
378 		pos = bdata->node_low_pfn;  in mark_bootmem()
403 	mark_bootmem_node(pgdat->bdata, start, end, 0, 0);  in free_bootmem_node()
446 	return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);  in reserve_bootmem_node()
470 static unsigned long __init align_idx(struct bootmem_data *bdata,  in align_idx()  argument
473 	unsigned long base = bdata->node_min_pfn;  in align_idx()
483 static unsigned long __init align_off(struct bootmem_data *bdata,  in align_off()  argument
486 	unsigned long base = PFN_PHYS(bdata->node_min_pfn);  in align_off()
493 static void * __init alloc_bootmem_bdata(struct bootmem_data *bdata,  in alloc_bootmem_bdata()  argument
501 		bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,  in alloc_bootmem_bdata()
508 	if (!bdata->node_bootmem_map)  in alloc_bootmem_bdata()
511 	min = bdata->node_min_pfn;  in alloc_bootmem_bdata()
512 	max = bdata->node_low_pfn;  in alloc_bootmem_bdata()
529 	sidx = start - bdata->node_min_pfn;  in alloc_bootmem_bdata()
530 	midx = max - bdata->node_min_pfn;  in alloc_bootmem_bdata()
532 	if (bdata->hint_idx > sidx) {  in alloc_bootmem_bdata()
538 		sidx = align_idx(bdata, bdata->hint_idx, step);  in alloc_bootmem_bdata()
546 		sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);  in alloc_bootmem_bdata()
547 		sidx = align_idx(bdata, sidx, step);  in alloc_bootmem_bdata()
554 			if (test_bit(i, bdata->node_bootmem_map)) {  in alloc_bootmem_bdata()
555 				sidx = align_idx(bdata, i, step);  in alloc_bootmem_bdata()
561 		if (bdata->last_end_off & (PAGE_SIZE - 1) &&  in alloc_bootmem_bdata()
562 				PFN_DOWN(bdata->last_end_off) + 1 == sidx)  in alloc_bootmem_bdata()
563 			start_off = align_off(bdata, bdata->last_end_off, align);  in alloc_bootmem_bdata()
570 		bdata->last_end_off = end_off;  in alloc_bootmem_bdata()
571 		bdata->hint_idx = PFN_UP(end_off);  in alloc_bootmem_bdata()
576 		if (__reserve(bdata, PFN_DOWN(start_off) + merge,  in alloc_bootmem_bdata()
580 		region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +  in alloc_bootmem_bdata()
592 		sidx = align_idx(bdata, fallback - 1, step);  in alloc_bootmem_bdata()
605 	bootmem_data_t *bdata;  in alloc_bootmem_core()  local
611 	list_for_each_entry(bdata, &bdata_list, list) {  in alloc_bootmem_core()
612 		if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))  in alloc_bootmem_core()
614 		if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))  in alloc_bootmem_core()
617 		region = alloc_bootmem_bdata(bdata, size, align, goal, limit);  in alloc_bootmem_core()
715 	ptr = alloc_bootmem_bdata(pgdat->bdata, size, align, goal, limit);  in ___alloc_bootmem_node_nopanic()
797 		ptr = alloc_bootmem_bdata(pgdat->bdata, size, align,  in __alloc_bootmem_node_high()