44 	struct ttm_mem_global *glob;  member
92 	spin_lock(&zone->glob->lock);  in ttm_mem_zone_show()
103 	spin_unlock(&zone->glob->lock);  in ttm_mem_zone_show()
109 static void ttm_check_swapping(struct ttm_mem_global *glob);
129 	spin_lock(&zone->glob->lock);  in ttm_mem_zone_store()
142 	spin_unlock(&zone->glob->lock);  in ttm_mem_zone_store()
144 	ttm_check_swapping(zone->glob);  in ttm_mem_zone_store()
171 	struct ttm_mem_global *glob =  in ttm_mem_global_kobj_release()  local
174 	kfree(glob);  in ttm_mem_global_kobj_release()
181 static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,  in ttm_zones_above_swap_target()  argument
188 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_zones_above_swap_target()
189 		zone = glob->zones[i];  in ttm_zones_above_swap_target()
213 static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,  in ttm_shrink()  argument
219 	spin_lock(&glob->lock);  in ttm_shrink()
220 	if (glob->shrink == NULL)  in ttm_shrink()
223 	while (ttm_zones_above_swap_target(glob, from_wq, extra)) {  in ttm_shrink()
224 		shrink = glob->shrink;  in ttm_shrink()
225 		spin_unlock(&glob->lock);  in ttm_shrink()
227 		spin_lock(&glob->lock);  in ttm_shrink()
232 	spin_unlock(&glob->lock);  in ttm_shrink()
239 	struct ttm_mem_global *glob =  in ttm_shrink_work()  local
242 	ttm_shrink(glob, true, 0ULL);  in ttm_shrink_work()
245 static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,  in ttm_mem_init_kernel_zone()  argument
264 	zone->glob = glob;  in ttm_mem_init_kernel_zone()
265 	glob->zone_kernel = zone;  in ttm_mem_init_kernel_zone()
267 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);  in ttm_mem_init_kernel_zone()
272 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_kernel_zone()
277 static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob,  in ttm_mem_init_highmem_zone()  argument
300 	zone->glob = glob;  in ttm_mem_init_highmem_zone()
301 	glob->zone_highmem = zone;  in ttm_mem_init_highmem_zone()
303 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s",  in ttm_mem_init_highmem_zone()
309 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_highmem_zone()
313 static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob,  in ttm_mem_init_dma32_zone()  argument
348 	zone->glob = glob;  in ttm_mem_init_dma32_zone()
349 	glob->zone_dma32 = zone;  in ttm_mem_init_dma32_zone()
351 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);  in ttm_mem_init_dma32_zone()
356 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_dma32_zone()
361 int ttm_mem_global_init(struct ttm_mem_global *glob)  in ttm_mem_global_init()  argument
368 	spin_lock_init(&glob->lock);  in ttm_mem_global_init()
369 	glob->swap_queue = create_singlethread_workqueue("ttm_swap");  in ttm_mem_global_init()
370 	INIT_WORK(&glob->work, ttm_shrink_work);  in ttm_mem_global_init()
372 		&glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting");  in ttm_mem_global_init()
374 		kobject_put(&glob->kobj);  in ttm_mem_global_init()
380 	ret = ttm_mem_init_kernel_zone(glob, &si);  in ttm_mem_global_init()
384 	ret = ttm_mem_init_highmem_zone(glob, &si);  in ttm_mem_global_init()
388 	ret = ttm_mem_init_dma32_zone(glob, &si);  in ttm_mem_global_init()
392 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_init()
393 		zone = glob->zones[i];  in ttm_mem_global_init()
397 	ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE));  in ttm_mem_global_init()
398 	ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE));  in ttm_mem_global_init()
401 	ttm_mem_global_release(glob);  in ttm_mem_global_init()
406 void ttm_mem_global_release(struct ttm_mem_global *glob)  in ttm_mem_global_release()  argument
415 	flush_workqueue(glob->swap_queue);  in ttm_mem_global_release()
416 	destroy_workqueue(glob->swap_queue);  in ttm_mem_global_release()
417 	glob->swap_queue = NULL;  in ttm_mem_global_release()
418 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_release()
419 		zone = glob->zones[i];  in ttm_mem_global_release()
423 	kobject_del(&glob->kobj);  in ttm_mem_global_release()
424 	kobject_put(&glob->kobj);  in ttm_mem_global_release()
428 static void ttm_check_swapping(struct ttm_mem_global *glob)  in ttm_check_swapping()  argument
434 	spin_lock(&glob->lock);  in ttm_check_swapping()
435 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_check_swapping()
436 		zone = glob->zones[i];  in ttm_check_swapping()
443 	spin_unlock(&glob->lock);  in ttm_check_swapping()
446 		(void)queue_work(glob->swap_queue, &glob->work);  in ttm_check_swapping()
450 static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,  in ttm_mem_global_free_zone()  argument
457 	spin_lock(&glob->lock);  in ttm_mem_global_free_zone()
458 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_free_zone()
459 		zone = glob->zones[i];  in ttm_mem_global_free_zone()
464 	spin_unlock(&glob->lock);  in ttm_mem_global_free_zone()
467 void ttm_mem_global_free(struct ttm_mem_global *glob,  in ttm_mem_global_free()  argument
470 	return ttm_mem_global_free_zone(glob, NULL, amount);  in ttm_mem_global_free()
474 static int ttm_mem_global_reserve(struct ttm_mem_global *glob,  in ttm_mem_global_reserve()  argument
483 	spin_lock(&glob->lock);  in ttm_mem_global_reserve()
484 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_reserve()
485 		zone = glob->zones[i];  in ttm_mem_global_reserve()
497 		for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_reserve()
498 			zone = glob->zones[i];  in ttm_mem_global_reserve()
507 	spin_unlock(&glob->lock);  in ttm_mem_global_reserve()
508 	ttm_check_swapping(glob);  in ttm_mem_global_reserve()
514 static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,  in ttm_mem_global_alloc_zone()  argument
521 	while (unlikely(ttm_mem_global_reserve(glob,  in ttm_mem_global_alloc_zone()
529 		ttm_shrink(glob, false, memory + (memory >> 2) + 16);  in ttm_mem_global_alloc_zone()
535 int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,  in ttm_mem_global_alloc()  argument
543 	return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait,  in ttm_mem_global_alloc()
548 int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,  in ttm_mem_global_alloc_page()  argument
561 	if (PageHighMem(page) && glob->zone_highmem != NULL)  in ttm_mem_global_alloc_page()
562 		zone = glob->zone_highmem;  in ttm_mem_global_alloc_page()
564 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)  in ttm_mem_global_alloc_page()
565 		zone = glob->zone_kernel;  in ttm_mem_global_alloc_page()
567 	return ttm_mem_global_alloc_zone(glob, zone, PAGE_SIZE, no_wait,  in ttm_mem_global_alloc_page()
571 void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page)  in ttm_mem_global_free_page()  argument
576 	if (PageHighMem(page) && glob->zone_highmem != NULL)  in ttm_mem_global_free_page()
577 		zone = glob->zone_highmem;  in ttm_mem_global_free_page()
579 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)  in ttm_mem_global_free_page()
580 		zone = glob->zone_kernel;  in ttm_mem_global_free_page()
582 	ttm_mem_global_free_zone(glob, zone, PAGE_SIZE);  in ttm_mem_global_free_page()