28 	.bdata = &bootmem_node_data[0]
78 static void __init link_bootmem(bootmem_data_t *bdata)  in link_bootmem()  argument
83 		if (bdata->node_min_pfn < ent->node_min_pfn) {  in link_bootmem()
84 			list_add_tail(&bdata->list, &ent->list);  in link_bootmem()
89 	list_add_tail(&bdata->list, &bdata_list);  in link_bootmem()
95 static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,  in init_bootmem_core()  argument
101 	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));  in init_bootmem_core()
102 	bdata->node_min_pfn = start;  in init_bootmem_core()
103 	bdata->node_low_pfn = end;  in init_bootmem_core()
104 	link_bootmem(bdata);  in init_bootmem_core()
111 	memset(bdata->node_bootmem_map, 0xff, mapsize);  in init_bootmem_core()
114 		bdata - bootmem_node_data, start, mapstart, end, mapsize);  in init_bootmem_core()
131 	return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);  in init_bootmem_node()
145 	return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);  in init_bootmem()
172 static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)  in free_all_bootmem_core()  argument
177 	if (!bdata->node_bootmem_map)  in free_all_bootmem_core()
180 	map = bdata->node_bootmem_map;  in free_all_bootmem_core()
181 	start = bdata->node_min_pfn;  in free_all_bootmem_core()
182 	end = bdata->node_low_pfn;  in free_all_bootmem_core()
185 		bdata - bootmem_node_data, start, end);  in free_all_bootmem_core()
191 		idx = start - bdata->node_min_pfn;  in free_all_bootmem_core()
232 	cur = bdata->node_min_pfn;  in free_all_bootmem_core()
233 	page = virt_to_page(bdata->node_bootmem_map);  in free_all_bootmem_core()
234 	pages = bdata->node_low_pfn - bdata->node_min_pfn;  in free_all_bootmem_core()
239 	bdata->node_bootmem_map = NULL;  in free_all_bootmem_core()
241 	bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);  in free_all_bootmem_core()
277 	bootmem_data_t *bdata;  in free_all_bootmem()  local
281 	list_for_each_entry(bdata, &bdata_list, list)  in free_all_bootmem()
282 		total_pages += free_all_bootmem_core(bdata);  in free_all_bootmem()
289 static void __init __free(bootmem_data_t *bdata,  in __free()  argument
294 	bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,  in __free()
295 		sidx + bdata->node_min_pfn,  in __free()
296 		eidx + bdata->node_min_pfn);  in __free()
298 	if (WARN_ON(bdata->node_bootmem_map == NULL))  in __free()
301 	if (bdata->hint_idx > sidx)  in __free()
302 		bdata->hint_idx = sidx;  in __free()
305 		if (!test_and_clear_bit(idx, bdata->node_bootmem_map))  in __free()
309 static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx,  in __reserve()  argument
316 		bdata - bootmem_node_data,  in __reserve()
317 		sidx + bdata->node_min_pfn,  in __reserve()
318 		eidx + bdata->node_min_pfn,  in __reserve()
321 	if (WARN_ON(bdata->node_bootmem_map == NULL))  in __reserve()
325 		if (test_and_set_bit(idx, bdata->node_bootmem_map)) {  in __reserve()
327 				__free(bdata, sidx, idx);  in __reserve()
331 				idx + bdata->node_min_pfn);  in __reserve()
336 static int __init mark_bootmem_node(bootmem_data_t *bdata,  in mark_bootmem_node()  argument
343 		bdata - bootmem_node_data, start, end, reserve, flags);  in mark_bootmem_node()
345 	BUG_ON(start < bdata->node_min_pfn);  in mark_bootmem_node()
346 	BUG_ON(end > bdata->node_low_pfn);  in mark_bootmem_node()
348 	sidx = start - bdata->node_min_pfn;  in mark_bootmem_node()
349 	eidx = end - bdata->node_min_pfn;  in mark_bootmem_node()
352 		return __reserve(bdata, sidx, eidx, flags);  in mark_bootmem_node()
354 		__free(bdata, sidx, eidx);  in mark_bootmem_node()
362 	bootmem_data_t *bdata;  in mark_bootmem()  local
365 	list_for_each_entry(bdata, &bdata_list, list) {  in mark_bootmem()
369 		if (pos < bdata->node_min_pfn ||  in mark_bootmem()
370 		    pos >= bdata->node_low_pfn) {  in mark_bootmem()
375 		max = min(bdata->node_low_pfn, end);  in mark_bootmem()
377 		err = mark_bootmem_node(bdata, pos, max, reserve, flags);  in mark_bootmem()
385 		pos = bdata->node_low_pfn;  in mark_bootmem()
410 	mark_bootmem_node(pgdat->bdata, start, end, 0, 0);  in free_bootmem_node()
453 	return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);  in reserve_bootmem_node()
477 static unsigned long __init align_idx(struct bootmem_data *bdata,  in align_idx()  argument
480 	unsigned long base = bdata->node_min_pfn;  in align_idx()
490 static unsigned long __init align_off(struct bootmem_data *bdata,  in align_off()  argument
493 	unsigned long base = PFN_PHYS(bdata->node_min_pfn);  in align_off()
500 static void * __init alloc_bootmem_bdata(struct bootmem_data *bdata,  in alloc_bootmem_bdata()  argument
508 		bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,  in alloc_bootmem_bdata()
515 	if (!bdata->node_bootmem_map)  in alloc_bootmem_bdata()
518 	min = bdata->node_min_pfn;  in alloc_bootmem_bdata()
519 	max = bdata->node_low_pfn;  in alloc_bootmem_bdata()
536 	sidx = start - bdata->node_min_pfn;  in alloc_bootmem_bdata()
537 	midx = max - bdata->node_min_pfn;  in alloc_bootmem_bdata()
539 	if (bdata->hint_idx > sidx) {  in alloc_bootmem_bdata()
545 		sidx = align_idx(bdata, bdata->hint_idx, step);  in alloc_bootmem_bdata()
553 		sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);  in alloc_bootmem_bdata()
554 		sidx = align_idx(bdata, sidx, step);  in alloc_bootmem_bdata()
561 			if (test_bit(i, bdata->node_bootmem_map)) {  in alloc_bootmem_bdata()
562 				sidx = align_idx(bdata, i, step);  in alloc_bootmem_bdata()
568 		if (bdata->last_end_off & (PAGE_SIZE - 1) &&  in alloc_bootmem_bdata()
569 				PFN_DOWN(bdata->last_end_off) + 1 == sidx)  in alloc_bootmem_bdata()
570 			start_off = align_off(bdata, bdata->last_end_off, align);  in alloc_bootmem_bdata()
577 		bdata->last_end_off = end_off;  in alloc_bootmem_bdata()
578 		bdata->hint_idx = PFN_UP(end_off);  in alloc_bootmem_bdata()
583 		if (__reserve(bdata, PFN_DOWN(start_off) + merge,  in alloc_bootmem_bdata()
587 		region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +  in alloc_bootmem_bdata()
599 		sidx = align_idx(bdata, fallback - 1, step);  in alloc_bootmem_bdata()
612 	bootmem_data_t *bdata;  in alloc_bootmem_core()  local
618 	list_for_each_entry(bdata, &bdata_list, list) {  in alloc_bootmem_core()
619 		if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))  in alloc_bootmem_core()
621 		if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))  in alloc_bootmem_core()
624 		region = alloc_bootmem_bdata(bdata, size, align, goal, limit);  in alloc_bootmem_core()
722 	ptr = alloc_bootmem_bdata(pgdat->bdata, size, align, goal, limit);  in ___alloc_bootmem_node_nopanic()
804 		ptr = alloc_bootmem_bdata(pgdat->bdata, size, align,  in __alloc_bootmem_node_high()