38 	spinlock_t lock;  member
55 static unsigned __find_holder(struct block_lock *lock,  in __find_holder()  argument
61 		if (lock->holders[i] == task)  in __find_holder()
69 static void __add_holder(struct block_lock *lock, struct task_struct *task)  in __add_holder()  argument
71 	unsigned h = __find_holder(lock, NULL);  in __add_holder()
77 	lock->holders[h] = task;  in __add_holder()
80 	t = lock->traces + h;  in __add_holder()
83 	t->entries = lock->entries[h];  in __add_holder()
90 static void __del_holder(struct block_lock *lock, struct task_struct *task)  in __del_holder()  argument
92 	unsigned h = __find_holder(lock, task);  in __del_holder()
93 	lock->holders[h] = NULL;  in __del_holder()
97 static int __check_holder(struct block_lock *lock)  in __check_holder()  argument
106 		if (lock->holders[i] == current) {  in __check_holder()
110 			print_stack_trace(lock->traces + i, 4);  in __check_holder()
155 static void __wake_many(struct block_lock *lock)  in __wake_many()  argument
159 	BUG_ON(lock->count < 0);  in __wake_many()
160 	list_for_each_entry_safe(w, tmp, &lock->waiters, list) {  in __wake_many()
161 		if (lock->count >= MAX_HOLDERS)  in __wake_many()
165 			if (lock->count > 0)  in __wake_many()
168 			lock->count = -1;  in __wake_many()
169 			__add_holder(lock, w->task);  in __wake_many()
174 		lock->count++;  in __wake_many()
175 		__add_holder(lock, w->task);  in __wake_many()
180 static void bl_init(struct block_lock *lock)  in bl_init()  argument
184 	spin_lock_init(&lock->lock);  in bl_init()
185 	lock->count = 0;  in bl_init()
186 	INIT_LIST_HEAD(&lock->waiters);  in bl_init()
188 		lock->holders[i] = NULL;  in bl_init()
191 static int __available_for_read(struct block_lock *lock)  in __available_for_read()  argument
193 	return lock->count >= 0 &&  in __available_for_read()
194 		lock->count < MAX_HOLDERS &&  in __available_for_read()
195 		list_empty(&lock->waiters);  in __available_for_read()
198 static int bl_down_read(struct block_lock *lock)  in bl_down_read()  argument
203 	spin_lock(&lock->lock);  in bl_down_read()
204 	r = __check_holder(lock);  in bl_down_read()
206 		spin_unlock(&lock->lock);  in bl_down_read()
210 	if (__available_for_read(lock)) {  in bl_down_read()
211 		lock->count++;  in bl_down_read()
212 		__add_holder(lock, current);  in bl_down_read()
213 		spin_unlock(&lock->lock);  in bl_down_read()
221 	list_add_tail(&w.list, &lock->waiters);  in bl_down_read()
222 	spin_unlock(&lock->lock);  in bl_down_read()
229 static int bl_down_read_nonblock(struct block_lock *lock)  in bl_down_read_nonblock()  argument
233 	spin_lock(&lock->lock);  in bl_down_read_nonblock()
234 	r = __check_holder(lock);  in bl_down_read_nonblock()
238 	if (__available_for_read(lock)) {  in bl_down_read_nonblock()
239 		lock->count++;  in bl_down_read_nonblock()
240 		__add_holder(lock, current);  in bl_down_read_nonblock()
246 	spin_unlock(&lock->lock);  in bl_down_read_nonblock()
250 static void bl_up_read(struct block_lock *lock)  in bl_up_read()  argument
252 	spin_lock(&lock->lock);  in bl_up_read()
253 	BUG_ON(lock->count <= 0);  in bl_up_read()
254 	__del_holder(lock, current);  in bl_up_read()
255 	--lock->count;  in bl_up_read()
256 	if (!list_empty(&lock->waiters))  in bl_up_read()
257 		__wake_many(lock);  in bl_up_read()
258 	spin_unlock(&lock->lock);  in bl_up_read()
261 static int bl_down_write(struct block_lock *lock)  in bl_down_write()  argument
266 	spin_lock(&lock->lock);  in bl_down_write()
267 	r = __check_holder(lock);  in bl_down_write()
269 		spin_unlock(&lock->lock);  in bl_down_write()
273 	if (lock->count == 0 && list_empty(&lock->waiters)) {  in bl_down_write()
274 		lock->count = -1;  in bl_down_write()
275 		__add_holder(lock, current);  in bl_down_write()
276 		spin_unlock(&lock->lock);  in bl_down_write()
288 	list_add(&w.list, &lock->waiters);  in bl_down_write()
289 	spin_unlock(&lock->lock);  in bl_down_write()
297 static void bl_up_write(struct block_lock *lock)  in bl_up_write()  argument
299 	spin_lock(&lock->lock);  in bl_up_write()
300 	__del_holder(lock, current);  in bl_up_write()
301 	lock->count = 0;  in bl_up_write()
302 	if (!list_empty(&lock->waiters))  in bl_up_write()
303 		__wake_many(lock);  in bl_up_write()
304 	spin_unlock(&lock->lock);  in bl_up_write()
342 	struct block_lock lock;  member
350 	bl_init(&aux->lock);  in dm_block_manager_alloc_callback()
461 	r = bl_down_read(&aux->lock);  in dm_bm_read_lock()
472 		bl_up_read(&aux->lock);  in dm_bm_read_lock()
497 	r = bl_down_write(&aux->lock);  in dm_bm_write_lock()
508 		bl_up_write(&aux->lock);  in dm_bm_write_lock()
532 	r = bl_down_read_nonblock(&aux->lock);  in dm_bm_read_try_lock()
542 		bl_up_read(&aux->lock);  in dm_bm_read_try_lock()
568 	r = bl_down_write(&aux->lock);  in dm_bm_write_lock_zero()
588 		bl_up_write(&aux->lock);  in dm_bm_unlock()
590 		bl_up_read(&aux->lock);  in dm_bm_unlock()