root/arch/arc/include/asm/spinlock.h

INCLUDED FROM

DEFINITIONS

1. arch_spin_lock
2. arch_spin_trylock
3. arch_spin_unlock
4. arch_read_lock
5. arch_read_trylock
6. arch_write_lock
7. arch_write_trylock
8. arch_read_unlock
9. arch_write_unlock
10. arch_spin_lock
11. arch_spin_trylock
12. arch_spin_unlock
13. arch_read_trylock
14. arch_write_trylock
15. arch_read_lock
16. arch_write_lock
17. arch_read_unlock
18. arch_write_unlock

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   4  */
   5 
   6 #ifndef __ASM_SPINLOCK_H
   7 #define __ASM_SPINLOCK_H
   8 
   9 #include <asm/spinlock_types.h>
  10 #include <asm/processor.h>
  11 #include <asm/barrier.h>
  12 
  13 #define arch_spin_is_locked(x)  ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)
  14 
  15 #ifdef CONFIG_ARC_HAS_LLSC
  16 
  17 static inline void arch_spin_lock(arch_spinlock_t *lock)
  18 {
  19         unsigned int val;
  20 
  21         __asm__ __volatile__(
  22         "1:     llock   %[val], [%[slock]]      \n"
  23         "       breq    %[val], %[LOCKED], 1b   \n"     /* spin while LOCKED */
  24         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
  25         "       bnz     1b                      \n"
  26         "                                       \n"
  27         : [val]         "=&r"   (val)
  28         : [slock]       "r"     (&(lock->slock)),
  29           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
  30         : "memory", "cc");
  31 
  32         /*
  33          * ACQUIRE barrier to ensure load/store after taking the lock
  34          * don't "bleed-up" out of the critical section (leak-in is allowed)
  35          * http://www.spinics.net/lists/kernel/msg2010409.html
  36          *
  37          * ARCv2 only has load-load, store-store and all-all barrier
  38          * thus need the full all-all barrier
  39          */
  40         smp_mb();
  41 }
  42 
  43 /* 1 - lock taken successfully */
  44 static inline int arch_spin_trylock(arch_spinlock_t *lock)
  45 {
  46         unsigned int val, got_it = 0;
  47 
  48         __asm__ __volatile__(
  49         "1:     llock   %[val], [%[slock]]      \n"
  50         "       breq    %[val], %[LOCKED], 4f   \n"     /* already LOCKED, just bail */
  51         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
  52         "       bnz     1b                      \n"
  53         "       mov     %[got_it], 1            \n"
  54         "4:                                     \n"
  55         "                                       \n"
  56         : [val]         "=&r"   (val),
  57           [got_it]      "+&r"   (got_it)
  58         : [slock]       "r"     (&(lock->slock)),
  59           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
  60         : "memory", "cc");
  61 
  62         smp_mb();
  63 
  64         return got_it;
  65 }
  66 
  67 static inline void arch_spin_unlock(arch_spinlock_t *lock)
  68 {
  69         smp_mb();
  70 
  71         WRITE_ONCE(lock->slock, __ARCH_SPIN_LOCK_UNLOCKED__);
  72 }
  73 
  74 /*
  75  * Read-write spinlocks, allowing multiple readers but only one writer.
  76  * Unfair locking as Writers could be starved indefinitely by Reader(s)
  77  */
  78 
  79 static inline void arch_read_lock(arch_rwlock_t *rw)
  80 {
  81         unsigned int val;
  82 
  83         /*
  84          * zero means writer holds the lock exclusively, deny Reader.
  85          * Otherwise grant lock to first/subseq reader
  86          *
  87          *      if (rw->counter > 0) {
  88          *              rw->counter--;
  89          *              ret = 1;
  90          *      }
  91          */
  92 
  93         __asm__ __volatile__(
  94         "1:     llock   %[val], [%[rwlock]]     \n"
  95         "       brls    %[val], %[WR_LOCKED], 1b\n"     /* <= 0: spin while write locked */
  96         "       sub     %[val], %[val], 1       \n"     /* reader lock */
  97         "       scond   %[val], [%[rwlock]]     \n"
  98         "       bnz     1b                      \n"
  99         "                                       \n"
 100         : [val]         "=&r"   (val)
 101         : [rwlock]      "r"     (&(rw->counter)),
 102           [WR_LOCKED]   "ir"    (0)
 103         : "memory", "cc");
 104 
 105         smp_mb();
 106 }
 107 
 108 /* 1 - lock taken successfully */
 109 static inline int arch_read_trylock(arch_rwlock_t *rw)
 110 {
 111         unsigned int val, got_it = 0;
 112 
 113         __asm__ __volatile__(
 114         "1:     llock   %[val], [%[rwlock]]     \n"
 115         "       brls    %[val], %[WR_LOCKED], 4f\n"     /* <= 0: already write locked, bail */
 116         "       sub     %[val], %[val], 1       \n"     /* counter-- */
 117         "       scond   %[val], [%[rwlock]]     \n"
 118         "       bnz     1b                      \n"     /* retry if collided with someone */
 119         "       mov     %[got_it], 1            \n"
 120         "                                       \n"
 121         "4: ; --- done ---                      \n"
 122 
 123         : [val]         "=&r"   (val),
 124           [got_it]      "+&r"   (got_it)
 125         : [rwlock]      "r"     (&(rw->counter)),
 126           [WR_LOCKED]   "ir"    (0)
 127         : "memory", "cc");
 128 
 129         smp_mb();
 130 
 131         return got_it;
 132 }
 133 
 134 static inline void arch_write_lock(arch_rwlock_t *rw)
 135 {
 136         unsigned int val;
 137 
 138         /*
 139          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 140          * deny writer. Otherwise if unlocked grant to writer
 141          * Hence the claim that Linux rwlocks are unfair to writers.
 142          * (can be starved for an indefinite time by readers).
 143          *
 144          *      if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 145          *              rw->counter = 0;
 146          *              ret = 1;
 147          *      }
 148          */
 149 
 150         __asm__ __volatile__(
 151         "1:     llock   %[val], [%[rwlock]]     \n"
 152         "       brne    %[val], %[UNLOCKED], 1b \n"     /* while !UNLOCKED spin */
 153         "       mov     %[val], %[WR_LOCKED]    \n"
 154         "       scond   %[val], [%[rwlock]]     \n"
 155         "       bnz     1b                      \n"
 156         "                                       \n"
 157         : [val]         "=&r"   (val)
 158         : [rwlock]      "r"     (&(rw->counter)),
 159           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 160           [WR_LOCKED]   "ir"    (0)
 161         : "memory", "cc");
 162 
 163         smp_mb();
 164 }
 165 
 166 /* 1 - lock taken successfully */
 167 static inline int arch_write_trylock(arch_rwlock_t *rw)
 168 {
 169         unsigned int val, got_it = 0;
 170 
 171         __asm__ __volatile__(
 172         "1:     llock   %[val], [%[rwlock]]     \n"
 173         "       brne    %[val], %[UNLOCKED], 4f \n"     /* !UNLOCKED, bail */
 174         "       mov     %[val], %[WR_LOCKED]    \n"
 175         "       scond   %[val], [%[rwlock]]     \n"
 176         "       bnz     1b                      \n"     /* retry if collided with someone */
 177         "       mov     %[got_it], 1            \n"
 178         "                                       \n"
 179         "4: ; --- done ---                      \n"
 180 
 181         : [val]         "=&r"   (val),
 182           [got_it]      "+&r"   (got_it)
 183         : [rwlock]      "r"     (&(rw->counter)),
 184           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 185           [WR_LOCKED]   "ir"    (0)
 186         : "memory", "cc");
 187 
 188         smp_mb();
 189 
 190         return got_it;
 191 }
 192 
 193 static inline void arch_read_unlock(arch_rwlock_t *rw)
 194 {
 195         unsigned int val;
 196 
 197         smp_mb();
 198 
 199         /*
 200          * rw->counter++;
 201          */
 202         __asm__ __volatile__(
 203         "1:     llock   %[val], [%[rwlock]]     \n"
 204         "       add     %[val], %[val], 1       \n"
 205         "       scond   %[val], [%[rwlock]]     \n"
 206         "       bnz     1b                      \n"
 207         "                                       \n"
 208         : [val]         "=&r"   (val)
 209         : [rwlock]      "r"     (&(rw->counter))
 210         : "memory", "cc");
 211 }
 212 
 213 static inline void arch_write_unlock(arch_rwlock_t *rw)
 214 {
 215         smp_mb();
 216 
 217         WRITE_ONCE(rw->counter, __ARCH_RW_LOCK_UNLOCKED__);
 218 }
 219 
 220 #else   /* !CONFIG_ARC_HAS_LLSC */
 221 
 222 static inline void arch_spin_lock(arch_spinlock_t *lock)
 223 {
 224         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
 225 
 226         /*
 227          * Per lkmm, smp_mb() is only required after _lock (and before_unlock)
 228          * for ACQ and REL semantics respectively. However EX based spinlocks
 229          * need the extra smp_mb to workaround a hardware quirk.
 230          */
 231         smp_mb();
 232 
 233         __asm__ __volatile__(
 234         "1:     ex  %0, [%1]            \n"
 235 #ifdef CONFIG_EZNPS_MTM_EXT
 236         "       .word %3                \n"
 237 #endif
 238         "       breq  %0, %2, 1b        \n"
 239         : "+&r" (val)
 240         : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__)
 241 #ifdef CONFIG_EZNPS_MTM_EXT
 242         , "i"(CTOP_INST_SCHD_RW)
 243 #endif
 244         : "memory");
 245 
 246         smp_mb();
 247 }
 248 
 249 /* 1 - lock taken successfully */
 250 static inline int arch_spin_trylock(arch_spinlock_t *lock)
 251 {
 252         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
 253 
 254         smp_mb();
 255 
 256         __asm__ __volatile__(
 257         "1:     ex  %0, [%1]            \n"
 258         : "+r" (val)
 259         : "r"(&(lock->slock))
 260         : "memory");
 261 
 262         smp_mb();
 263 
 264         return (val == __ARCH_SPIN_LOCK_UNLOCKED__);
 265 }
 266 
 267 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 268 {
 269         unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__;
 270 
 271         /*
 272          * RELEASE barrier: given the instructions avail on ARCv2, full barrier
 273          * is the only option
 274          */
 275         smp_mb();
 276 
 277         /*
 278          * EX is not really required here, a simple STore of 0 suffices.
 279          * However this causes tasklist livelocks in SystemC based SMP virtual
 280          * platforms where the systemc core scheduler uses EX as a cue for
 281          * moving to next core. Do a git log of this file for details
 282          */
 283         __asm__ __volatile__(
 284         "       ex  %0, [%1]            \n"
 285         : "+r" (val)
 286         : "r"(&(lock->slock))
 287         : "memory");
 288 
 289         /*
 290          * see pairing version/comment in arch_spin_lock above
 291          */
 292         smp_mb();
 293 }
 294 
 295 /*
 296  * Read-write spinlocks, allowing multiple readers but only one writer.
 297  * Unfair locking as Writers could be starved indefinitely by Reader(s)
 298  *
 299  * The spinlock itself is contained in @counter and access to it is
 300  * serialized with @lock_mutex.
 301  */
 302 
 303 /* 1 - lock taken successfully */
 304 static inline int arch_read_trylock(arch_rwlock_t *rw)
 305 {
 306         int ret = 0;
 307         unsigned long flags;
 308 
 309         local_irq_save(flags);
 310         arch_spin_lock(&(rw->lock_mutex));
 311 
 312         /*
 313          * zero means writer holds the lock exclusively, deny Reader.
 314          * Otherwise grant lock to first/subseq reader
 315          */
 316         if (rw->counter > 0) {
 317                 rw->counter--;
 318                 ret = 1;
 319         }
 320 
 321         arch_spin_unlock(&(rw->lock_mutex));
 322         local_irq_restore(flags);
 323 
 324         return ret;
 325 }
 326 
 327 /* 1 - lock taken successfully */
 328 static inline int arch_write_trylock(arch_rwlock_t *rw)
 329 {
 330         int ret = 0;
 331         unsigned long flags;
 332 
 333         local_irq_save(flags);
 334         arch_spin_lock(&(rw->lock_mutex));
 335 
 336         /*
 337          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 338          * deny writer. Otherwise if unlocked grant to writer
 339          * Hence the claim that Linux rwlocks are unfair to writers.
 340          * (can be starved for an indefinite time by readers).
 341          */
 342         if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 343                 rw->counter = 0;
 344                 ret = 1;
 345         }
 346         arch_spin_unlock(&(rw->lock_mutex));
 347         local_irq_restore(flags);
 348 
 349         return ret;
 350 }
 351 
 352 static inline void arch_read_lock(arch_rwlock_t *rw)
 353 {
 354         while (!arch_read_trylock(rw))
 355                 cpu_relax();
 356 }
 357 
 358 static inline void arch_write_lock(arch_rwlock_t *rw)
 359 {
 360         while (!arch_write_trylock(rw))
 361                 cpu_relax();
 362 }
 363 
 364 static inline void arch_read_unlock(arch_rwlock_t *rw)
 365 {
 366         unsigned long flags;
 367 
 368         local_irq_save(flags);
 369         arch_spin_lock(&(rw->lock_mutex));
 370         rw->counter++;
 371         arch_spin_unlock(&(rw->lock_mutex));
 372         local_irq_restore(flags);
 373 }
 374 
 375 static inline void arch_write_unlock(arch_rwlock_t *rw)
 376 {
 377         unsigned long flags;
 378 
 379         local_irq_save(flags);
 380         arch_spin_lock(&(rw->lock_mutex));
 381         rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
 382         arch_spin_unlock(&(rw->lock_mutex));
 383         local_irq_restore(flags);
 384 }
 385 
 386 #endif
 387 
 388 #endif /* __ASM_SPINLOCK_H */