63 static int space_init(struct entry_space *es, unsigned nr_entries)  in space_init()  argument
66 		es->begin = es->end = NULL;  in space_init()
70 	es->begin = vzalloc(sizeof(struct entry) * nr_entries);  in space_init()
71 	if (!es->begin)  in space_init()
74 	es->end = es->begin + nr_entries;  in space_init()
78 static void space_exit(struct entry_space *es)  in space_exit()  argument
80 	vfree(es->begin);  in space_exit()
83 static struct entry *__get_entry(struct entry_space *es, unsigned block)  in __get_entry()  argument
87 	e = es->begin + block;  in __get_entry()
88 	BUG_ON(e >= es->end);  in __get_entry()
93 static unsigned to_index(struct entry_space *es, struct entry *e)  in to_index()  argument
95 	BUG_ON(e < es->begin || e >= es->end);  in to_index()
96 	return e - es->begin;  in to_index()
99 static struct entry *to_entry(struct entry_space *es, unsigned block)  in to_entry()  argument
104 	return __get_entry(es, block);  in to_entry()
120 static struct entry *l_head(struct entry_space *es, struct ilist *l)  in l_head()  argument
122 	return to_entry(es, l->head);  in l_head()
125 static struct entry *l_tail(struct entry_space *es, struct ilist *l)  in l_tail()  argument
127 	return to_entry(es, l->tail);  in l_tail()
130 static struct entry *l_next(struct entry_space *es, struct entry *e)  in l_next()  argument
132 	return to_entry(es, e->next);  in l_next()
135 static struct entry *l_prev(struct entry_space *es, struct entry *e)  in l_prev()  argument
137 	return to_entry(es, e->prev);  in l_prev()
145 static void l_add_head(struct entry_space *es, struct ilist *l, struct entry *e)  in l_add_head()  argument
147 	struct entry *head = l_head(es, l);  in l_add_head()
153 		head->prev = l->head = to_index(es, e);  in l_add_head()
155 		l->head = l->tail = to_index(es, e);  in l_add_head()
161 static void l_add_tail(struct entry_space *es, struct ilist *l, struct entry *e)  in l_add_tail()  argument
163 	struct entry *tail = l_tail(es, l);  in l_add_tail()
169 		tail->next = l->tail = to_index(es, e);  in l_add_tail()
171 		l->head = l->tail = to_index(es, e);  in l_add_tail()
177 static void l_add_before(struct entry_space *es, struct ilist *l,  in l_add_before()  argument
180 	struct entry *prev = l_prev(es, old);  in l_add_before()
183 		l_add_head(es, l, e);  in l_add_before()
187 		e->next = to_index(es, old);  in l_add_before()
188 		prev->next = old->prev = to_index(es, e);  in l_add_before()
195 static void l_del(struct entry_space *es, struct ilist *l, struct entry *e)  in l_del()  argument
197 	struct entry *prev = l_prev(es, e);  in l_del()
198 	struct entry *next = l_next(es, e);  in l_del()
214 static struct entry *l_pop_tail(struct entry_space *es, struct ilist *l)  in l_pop_tail()  argument
218 	for (e = l_tail(es, l); e; e = l_prev(es, e))  in l_pop_tail()
220 			l_del(es, l, e);  in l_pop_tail()
238 	struct entry_space *es;  member
254 static void q_init(struct queue *q, struct entry_space *es, unsigned nr_levels)  in q_init()  argument
258 	q->es = es;  in q_init()
285 	l_add_tail(q->es, q->qs + e->level, e);  in q_push()
293 	l_add_before(q->es, q->qs + e->level, old, e);  in q_push_before()
298 	l_del(q->es, q->qs + e->level, e);  in q_del()
314 		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e)) {  in q_peek()
361 		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e))  in __redist_pop_from()
363 				l_del(q->es, q->qs + e->level, e);  in __redist_pop_from()
434 			l_add_tail(q->es, l, e);  in q_redistribute()
442 			e = l_pop_tail(q->es, l);  in q_redistribute()
449 			l_add_head(q->es, l_above, e);  in q_redistribute()
463 		for (de = l_head(q->es, q->qs + new_level); de; de = l_next(q->es, de)) {  in q_requeue_before()
554 	struct entry_space *es;  member
563 static int h_init(struct hash_table *ht, struct entry_space *es, unsigned nr_entries)  in h_init()  argument
567 	ht->es = es;  in h_init()
588 	return to_entry(ht->es, ht->buckets[bucket]);  in h_head()
593 	return to_entry(ht->es, e->hash_next);  in h_next()
599 	ht->buckets[bucket] = to_index(ht->es, e);  in __h_insert()
671 	struct entry_space *es;  member
678 static void init_allocator(struct entry_alloc *ea, struct entry_space *es,  in init_allocator()  argument
683 	ea->es = es;  in init_allocator()
689 		l_add_tail(ea->es, &ea->free, __get_entry(ea->es, i));  in init_allocator()
712 	e = l_pop_tail(ea->es, &ea->free);  in alloc_entry()
724 	struct entry *e = __get_entry(ea->es, ea->begin + i);  in alloc_particular_entry()
728 	l_del(ea->es, &ea->free, e);  in alloc_particular_entry()
742 	l_add_tail(ea->es, &ea->free, e);  in free_entry()
752 	return to_index(ea->es, e) - ea->begin;  in get_index()
757 	return __get_entry(ea->es, ea->begin + index);  in get_entry()
784 	struct entry_space es;  member
1289 	space_exit(&mq->es);  in smq_destroy()
1386 		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e)) {  in smq_save_hints()
1630 	if (space_init(&mq->es, total_sentinels + mq->nr_hotspot_blocks + from_cblock(cache_size))) {  in smq_create()
1635 	init_allocator(&mq->writeback_sentinel_alloc, &mq->es, 0, nr_sentinels_per_queue);  in smq_create()
1639 	init_allocator(&mq->demote_sentinel_alloc, &mq->es, nr_sentinels_per_queue, total_sentinels);  in smq_create()
1643 	init_allocator(&mq->hotspot_alloc, &mq->es, total_sentinels,  in smq_create()
1646 	init_allocator(&mq->cache_alloc, &mq->es,  in smq_create()
1670 	q_init(&mq->hotspot, &mq->es, NR_HOTSPOT_LEVELS);  in smq_create()
1675 	q_init(&mq->clean, &mq->es, NR_CACHE_LEVELS);  in smq_create()
1676 	q_init(&mq->dirty, &mq->es, NR_CACHE_LEVELS);  in smq_create()
1681 	if (h_init(&mq->table, &mq->es, from_cblock(cache_size)))  in smq_create()
1684 	if (h_init(&mq->hotspot_table, &mq->es, mq->nr_hotspot_blocks))  in smq_create()
1702 	space_exit(&mq->es);  in smq_create()