285 static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,  in ttm_pool_update_free_locked()  argument
288 	pool->npages -= freed_pages;  in ttm_pool_update_free_locked()
289 	pool->nfrees += freed_pages;  in ttm_pool_update_free_locked()
302 static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free,  in ttm_page_pool_free()  argument
326 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_page_pool_free()
328 	list_for_each_entry_reverse(p, &pool->list, lru) {  in ttm_page_pool_free()
336 			__list_del(p->lru.prev, &pool->list);  in ttm_page_pool_free()
338 			ttm_pool_update_free_locked(pool, freed_pages);  in ttm_page_pool_free()
343 			spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_page_pool_free()
371 		__list_del(&p->lru, &pool->list);  in ttm_page_pool_free()
373 		ttm_pool_update_free_locked(pool, freed_pages);  in ttm_page_pool_free()
377 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_page_pool_free()
401 	struct ttm_page_pool *pool;  in ttm_pool_shrink_scan()  local
413 		pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];  in ttm_pool_shrink_scan()
415 		shrink_pages = ttm_page_pool_free(pool, nr_free, true);  in ttm_pool_shrink_scan()
573 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,  in ttm_page_pool_fill_locked()  argument
585 	if (pool->fill_lock)  in ttm_page_pool_fill_locked()
588 	pool->fill_lock = true;  in ttm_page_pool_fill_locked()
593 		&& count > pool->npages) {  in ttm_page_pool_fill_locked()
601 		spin_unlock_irqrestore(&pool->lock, *irq_flags);  in ttm_page_pool_fill_locked()
604 		r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,  in ttm_page_pool_fill_locked()
606 		spin_lock_irqsave(&pool->lock, *irq_flags);  in ttm_page_pool_fill_locked()
609 			list_splice(&new_pages, &pool->list);  in ttm_page_pool_fill_locked()
610 			++pool->nrefills;  in ttm_page_pool_fill_locked()
611 			pool->npages += alloc_size;  in ttm_page_pool_fill_locked()
613 			pr_err("Failed to fill pool (%p)\n", pool);  in ttm_page_pool_fill_locked()
615 			list_for_each_entry(p, &pool->list, lru) {  in ttm_page_pool_fill_locked()
618 			list_splice(&new_pages, &pool->list);  in ttm_page_pool_fill_locked()
619 			pool->npages += cpages;  in ttm_page_pool_fill_locked()
623 	pool->fill_lock = false;  in ttm_page_pool_fill_locked()
631 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,  in ttm_page_pool_get_pages()  argument
641 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_page_pool_get_pages()
642 	ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);  in ttm_page_pool_get_pages()
644 	if (count >= pool->npages) {  in ttm_page_pool_get_pages()
646 		list_splice_init(&pool->list, pages);  in ttm_page_pool_get_pages()
647 		count -= pool->npages;  in ttm_page_pool_get_pages()
648 		pool->npages = 0;  in ttm_page_pool_get_pages()
653 	if (count <= pool->npages/2) {  in ttm_page_pool_get_pages()
655 		list_for_each(p, &pool->list) {  in ttm_page_pool_get_pages()
660 		i = pool->npages + 1;  in ttm_page_pool_get_pages()
661 		list_for_each_prev(p, &pool->list) {  in ttm_page_pool_get_pages()
667 	list_cut_position(pages, &pool->list, p);  in ttm_page_pool_get_pages()
668 	pool->npages -= count;  in ttm_page_pool_get_pages()
671 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_page_pool_get_pages()
680 	struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);  in ttm_put_pages()  local
683 	if (pool == NULL) {  in ttm_put_pages()
696 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_put_pages()
701 			list_add_tail(&pages[i]->lru, &pool->list);  in ttm_put_pages()
703 			pool->npages++;  in ttm_put_pages()
708 	if (pool->npages > _manager->options.max_size) {  in ttm_put_pages()
709 		npages = pool->npages - _manager->options.max_size;  in ttm_put_pages()
715 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_put_pages()
717 		ttm_page_pool_free(pool, npages, false);  in ttm_put_pages()
727 	struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);  in ttm_get_pages()  local
739 	if (pool == NULL) {  in ttm_get_pages()
759 	gfp_flags |= pool->gfp_flags;  in ttm_get_pages()
763 	npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);  in ttm_get_pages()
801 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags,  in ttm_page_pool_init_locked()  argument
804 	spin_lock_init(&pool->lock);  in ttm_page_pool_init_locked()
805 	pool->fill_lock = false;  in ttm_page_pool_init_locked()
806 	INIT_LIST_HEAD(&pool->list);  in ttm_page_pool_init_locked()
807 	pool->npages = pool->nfrees = 0;  in ttm_page_pool_init_locked()
808 	pool->gfp_flags = flags;  in ttm_page_pool_init_locked()
809 	pool->name = name;  in ttm_page_pool_init_locked()