1#include <linux/export.h>
2#include <linux/lockref.h>
3
4#if USE_CMPXCHG_LOCKREF
5
6/*
7 * Allow weakly-ordered memory architectures to provide barrier-less
8 * cmpxchg semantics for lockref updates.
9 */
10#ifndef cmpxchg64_relaxed
11# define cmpxchg64_relaxed cmpxchg64
12#endif
13
14/*
15 * Note that the "cmpxchg()" reloads the "old" value for the
16 * failure case.
17 */
18#define CMPXCHG_LOOP(CODE, SUCCESS) do {					\
19	struct lockref old;							\
20	BUILD_BUG_ON(sizeof(old) != 8);						\
21	old.lock_count = READ_ONCE(lockref->lock_count);			\
22	while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {  	\
23		struct lockref new = old, prev = old;				\
24		CODE								\
25		old.lock_count = cmpxchg64_relaxed(&lockref->lock_count,	\
26						   old.lock_count,		\
27						   new.lock_count);		\
28		if (likely(old.lock_count == prev.lock_count)) {		\
29			SUCCESS;						\
30		}								\
31		cpu_relax_lowlatency();						\
32	}									\
33} while (0)
34
35#else
36
37#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
38
39#endif
40
41/**
42 * lockref_get - Increments reference count unconditionally
43 * @lockref: pointer to lockref structure
44 *
45 * This operation is only valid if you already hold a reference
46 * to the object, so you know the count cannot be zero.
47 */
48void lockref_get(struct lockref *lockref)
49{
50	CMPXCHG_LOOP(
51		new.count++;
52	,
53		return;
54	);
55
56	spin_lock(&lockref->lock);
57	lockref->count++;
58	spin_unlock(&lockref->lock);
59}
60EXPORT_SYMBOL(lockref_get);
61
62/**
63 * lockref_get_not_zero - Increments count unless the count is 0 or dead
64 * @lockref: pointer to lockref structure
65 * Return: 1 if count updated successfully or 0 if count was zero
66 */
67int lockref_get_not_zero(struct lockref *lockref)
68{
69	int retval;
70
71	CMPXCHG_LOOP(
72		new.count++;
73		if (old.count <= 0)
74			return 0;
75	,
76		return 1;
77	);
78
79	spin_lock(&lockref->lock);
80	retval = 0;
81	if (lockref->count > 0) {
82		lockref->count++;
83		retval = 1;
84	}
85	spin_unlock(&lockref->lock);
86	return retval;
87}
88EXPORT_SYMBOL(lockref_get_not_zero);
89
90/**
91 * lockref_get_or_lock - Increments count unless the count is 0 or dead
92 * @lockref: pointer to lockref structure
93 * Return: 1 if count updated successfully or 0 if count was zero
94 * and we got the lock instead.
95 */
96int lockref_get_or_lock(struct lockref *lockref)
97{
98	CMPXCHG_LOOP(
99		new.count++;
100		if (old.count <= 0)
101			break;
102	,
103		return 1;
104	);
105
106	spin_lock(&lockref->lock);
107	if (lockref->count <= 0)
108		return 0;
109	lockref->count++;
110	spin_unlock(&lockref->lock);
111	return 1;
112}
113EXPORT_SYMBOL(lockref_get_or_lock);
114
115/**
116 * lockref_put_return - Decrement reference count if possible
117 * @lockref: pointer to lockref structure
118 *
119 * Decrement the reference count and return the new value.
120 * If the lockref was dead or locked, return an error.
121 */
122int lockref_put_return(struct lockref *lockref)
123{
124	CMPXCHG_LOOP(
125		new.count--;
126		if (old.count <= 0)
127			return -1;
128	,
129		return new.count;
130	);
131	return -1;
132}
133EXPORT_SYMBOL(lockref_put_return);
134
135/**
136 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
137 * @lockref: pointer to lockref structure
138 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
139 */
140int lockref_put_or_lock(struct lockref *lockref)
141{
142	CMPXCHG_LOOP(
143		new.count--;
144		if (old.count <= 1)
145			break;
146	,
147		return 1;
148	);
149
150	spin_lock(&lockref->lock);
151	if (lockref->count <= 1)
152		return 0;
153	lockref->count--;
154	spin_unlock(&lockref->lock);
155	return 1;
156}
157EXPORT_SYMBOL(lockref_put_or_lock);
158
159/**
160 * lockref_mark_dead - mark lockref dead
161 * @lockref: pointer to lockref structure
162 */
163void lockref_mark_dead(struct lockref *lockref)
164{
165	assert_spin_locked(&lockref->lock);
166	lockref->count = -128;
167}
168EXPORT_SYMBOL(lockref_mark_dead);
169
170/**
171 * lockref_get_not_dead - Increments count unless the ref is dead
172 * @lockref: pointer to lockref structure
173 * Return: 1 if count updated successfully or 0 if lockref was dead
174 */
175int lockref_get_not_dead(struct lockref *lockref)
176{
177	int retval;
178
179	CMPXCHG_LOOP(
180		new.count++;
181		if (old.count < 0)
182			return 0;
183	,
184		return 1;
185	);
186
187	spin_lock(&lockref->lock);
188	retval = 0;
189	if (lockref->count >= 0) {
190		lockref->count++;
191		retval = 1;
192	}
193	spin_unlock(&lockref->lock);
194	return retval;
195}
196EXPORT_SYMBOL(lockref_get_not_dead);
197