1/*
2 * Queue read/write lock
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 * GNU General Public License for more details.
13 *
14 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
15 *
16 * Authors: Waiman Long <waiman.long@hp.com>
17 */
18#include <linux/smp.h>
19#include <linux/bug.h>
20#include <linux/cpumask.h>
21#include <linux/percpu.h>
22#include <linux/hardirq.h>
23#include <asm/qrwlock.h>
24
25/**
26 * rspin_until_writer_unlock - inc reader count & spin until writer is gone
27 * @lock  : Pointer to queue rwlock structure
28 * @writer: Current queue rwlock writer status byte
29 *
30 * In interrupt context or at the head of the queue, the reader will just
31 * increment the reader count & wait until the writer releases the lock.
32 */
33static __always_inline void
34rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts)
35{
36	while ((cnts & _QW_WMASK) == _QW_LOCKED) {
37		cpu_relax_lowlatency();
38		cnts = smp_load_acquire((u32 *)&lock->cnts);
39	}
40}
41
42/**
43 * queue_read_lock_slowpath - acquire read lock of a queue rwlock
44 * @lock: Pointer to queue rwlock structure
45 */
46void queue_read_lock_slowpath(struct qrwlock *lock)
47{
48	u32 cnts;
49
50	/*
51	 * Readers come here when they cannot get the lock without waiting
52	 */
53	if (unlikely(in_interrupt())) {
54		/*
55		 * Readers in interrupt context will spin until the lock is
56		 * available without waiting in the queue.
57		 */
58		cnts = smp_load_acquire((u32 *)&lock->cnts);
59		rspin_until_writer_unlock(lock, cnts);
60		return;
61	}
62	atomic_sub(_QR_BIAS, &lock->cnts);
63
64	/*
65	 * Put the reader into the wait queue
66	 */
67	arch_spin_lock(&lock->lock);
68
69	/*
70	 * At the head of the wait queue now, wait until the writer state
71	 * goes to 0 and then try to increment the reader count and get
72	 * the lock. It is possible that an incoming writer may steal the
73	 * lock in the interim, so it is necessary to check the writer byte
74	 * to make sure that the write lock isn't taken.
75	 */
76	while (atomic_read(&lock->cnts) & _QW_WMASK)
77		cpu_relax_lowlatency();
78
79	cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS;
80	rspin_until_writer_unlock(lock, cnts);
81
82	/*
83	 * Signal the next one in queue to become queue head
84	 */
85	arch_spin_unlock(&lock->lock);
86}
87EXPORT_SYMBOL(queue_read_lock_slowpath);
88
89/**
90 * queue_write_lock_slowpath - acquire write lock of a queue rwlock
91 * @lock : Pointer to queue rwlock structure
92 */
93void queue_write_lock_slowpath(struct qrwlock *lock)
94{
95	u32 cnts;
96
97	/* Put the writer into the wait queue */
98	arch_spin_lock(&lock->lock);
99
100	/* Try to acquire the lock directly if no reader is present */
101	if (!atomic_read(&lock->cnts) &&
102	    (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0))
103		goto unlock;
104
105	/*
106	 * Set the waiting flag to notify readers that a writer is pending,
107	 * or wait for a previous writer to go away.
108	 */
109	for (;;) {
110		cnts = atomic_read(&lock->cnts);
111		if (!(cnts & _QW_WMASK) &&
112		    (atomic_cmpxchg(&lock->cnts, cnts,
113				    cnts | _QW_WAITING) == cnts))
114			break;
115
116		cpu_relax_lowlatency();
117	}
118
119	/* When no more readers, set the locked flag */
120	for (;;) {
121		cnts = atomic_read(&lock->cnts);
122		if ((cnts == _QW_WAITING) &&
123		    (atomic_cmpxchg(&lock->cnts, _QW_WAITING,
124				    _QW_LOCKED) == _QW_WAITING))
125			break;
126
127		cpu_relax_lowlatency();
128	}
129unlock:
130	arch_spin_unlock(&lock->lock);
131}
132EXPORT_SYMBOL(queue_write_lock_slowpath);
133