1#include <linux/export.h> 2#include <linux/lockref.h> 3 4#if USE_CMPXCHG_LOCKREF 5 6/* 7 * Allow weakly-ordered memory architectures to provide barrier-less 8 * cmpxchg semantics for lockref updates. 9 */ 10#ifndef cmpxchg64_relaxed 11# define cmpxchg64_relaxed cmpxchg64 12#endif 13 14/* 15 * Note that the "cmpxchg()" reloads the "old" value for the 16 * failure case. 17 */ 18#define CMPXCHG_LOOP(CODE, SUCCESS) do { \ 19 struct lockref old; \ 20 BUILD_BUG_ON(sizeof(old) != 8); \ 21 old.lock_count = READ_ONCE(lockref->lock_count); \ 22 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \ 23 struct lockref new = old, prev = old; \ 24 CODE \ 25 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \ 26 old.lock_count, \ 27 new.lock_count); \ 28 if (likely(old.lock_count == prev.lock_count)) { \ 29 SUCCESS; \ 30 } \ 31 cpu_relax_lowlatency(); \ 32 } \ 33} while (0) 34 35#else 36 37#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0) 38 39#endif 40 41/** 42 * lockref_get - Increments reference count unconditionally 43 * @lockref: pointer to lockref structure 44 * 45 * This operation is only valid if you already hold a reference 46 * to the object, so you know the count cannot be zero. 47 */ 48void lockref_get(struct lockref *lockref) 49{ 50 CMPXCHG_LOOP( 51 new.count++; 52 , 53 return; 54 ); 55 56 spin_lock(&lockref->lock); 57 lockref->count++; 58 spin_unlock(&lockref->lock); 59} 60EXPORT_SYMBOL(lockref_get); 61 62/** 63 * lockref_get_not_zero - Increments count unless the count is 0 or dead 64 * @lockref: pointer to lockref structure 65 * Return: 1 if count updated successfully or 0 if count was zero 66 */ 67int lockref_get_not_zero(struct lockref *lockref) 68{ 69 int retval; 70 71 CMPXCHG_LOOP( 72 new.count++; 73 if (old.count <= 0) 74 return 0; 75 , 76 return 1; 77 ); 78 79 spin_lock(&lockref->lock); 80 retval = 0; 81 if (lockref->count > 0) { 82 lockref->count++; 83 retval = 1; 84 } 85 spin_unlock(&lockref->lock); 86 return retval; 87} 88EXPORT_SYMBOL(lockref_get_not_zero); 89 90/** 91 * lockref_get_or_lock - Increments count unless the count is 0 or dead 92 * @lockref: pointer to lockref structure 93 * Return: 1 if count updated successfully or 0 if count was zero 94 * and we got the lock instead. 95 */ 96int lockref_get_or_lock(struct lockref *lockref) 97{ 98 CMPXCHG_LOOP( 99 new.count++; 100 if (old.count <= 0) 101 break; 102 , 103 return 1; 104 ); 105 106 spin_lock(&lockref->lock); 107 if (lockref->count <= 0) 108 return 0; 109 lockref->count++; 110 spin_unlock(&lockref->lock); 111 return 1; 112} 113EXPORT_SYMBOL(lockref_get_or_lock); 114 115/** 116 * lockref_put_return - Decrement reference count if possible 117 * @lockref: pointer to lockref structure 118 * 119 * Decrement the reference count and return the new value. 120 * If the lockref was dead or locked, return an error. 121 */ 122int lockref_put_return(struct lockref *lockref) 123{ 124 CMPXCHG_LOOP( 125 new.count--; 126 if (old.count <= 0) 127 return -1; 128 , 129 return new.count; 130 ); 131 return -1; 132} 133EXPORT_SYMBOL(lockref_put_return); 134 135/** 136 * lockref_put_or_lock - decrements count unless count <= 1 before decrement 137 * @lockref: pointer to lockref structure 138 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken 139 */ 140int lockref_put_or_lock(struct lockref *lockref) 141{ 142 CMPXCHG_LOOP( 143 new.count--; 144 if (old.count <= 1) 145 break; 146 , 147 return 1; 148 ); 149 150 spin_lock(&lockref->lock); 151 if (lockref->count <= 1) 152 return 0; 153 lockref->count--; 154 spin_unlock(&lockref->lock); 155 return 1; 156} 157EXPORT_SYMBOL(lockref_put_or_lock); 158 159/** 160 * lockref_mark_dead - mark lockref dead 161 * @lockref: pointer to lockref structure 162 */ 163void lockref_mark_dead(struct lockref *lockref) 164{ 165 assert_spin_locked(&lockref->lock); 166 lockref->count = -128; 167} 168EXPORT_SYMBOL(lockref_mark_dead); 169 170/** 171 * lockref_get_not_dead - Increments count unless the ref is dead 172 * @lockref: pointer to lockref structure 173 * Return: 1 if count updated successfully or 0 if lockref was dead 174 */ 175int lockref_get_not_dead(struct lockref *lockref) 176{ 177 int retval; 178 179 CMPXCHG_LOOP( 180 new.count++; 181 if (old.count < 0) 182 return 0; 183 , 184 return 1; 185 ); 186 187 spin_lock(&lockref->lock); 188 retval = 0; 189 if (lockref->count >= 0) { 190 lockref->count++; 191 retval = 1; 192 } 193 spin_unlock(&lockref->lock); 194 return retval; 195} 196EXPORT_SYMBOL(lockref_get_not_dead); 197