1/* 2 * Copyright 2010 Tilera Corporation. All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation, version 2. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11 * NON INFRINGEMENT. See the GNU General Public License for 12 * more details. 13 */ 14 15#include <linux/spinlock.h> 16#include <linux/module.h> 17#include <asm/processor.h> 18#include <arch/spr_def.h> 19 20#include "spinlock_common.h" 21 22void arch_spin_lock(arch_spinlock_t *lock) 23{ 24 int my_ticket; 25 int iterations = 0; 26 int delta; 27 28 while ((my_ticket = __insn_tns((void *)&lock->next_ticket)) & 1) 29 delay_backoff(iterations++); 30 31 /* Increment the next ticket number, implicitly releasing tns lock. */ 32 lock->next_ticket = my_ticket + TICKET_QUANTUM; 33 34 /* Wait until it's our turn. */ 35 while ((delta = my_ticket - lock->current_ticket) != 0) 36 relax((128 / CYCLES_PER_RELAX_LOOP) * delta); 37} 38EXPORT_SYMBOL(arch_spin_lock); 39 40int arch_spin_trylock(arch_spinlock_t *lock) 41{ 42 /* 43 * Grab a ticket; no need to retry if it's busy, we'll just 44 * treat that the same as "locked", since someone else 45 * will lock it momentarily anyway. 46 */ 47 int my_ticket = __insn_tns((void *)&lock->next_ticket); 48 49 if (my_ticket == lock->current_ticket) { 50 /* Not currently locked, so lock it by keeping this ticket. */ 51 lock->next_ticket = my_ticket + TICKET_QUANTUM; 52 /* Success! */ 53 return 1; 54 } 55 56 if (!(my_ticket & 1)) { 57 /* Release next_ticket. */ 58 lock->next_ticket = my_ticket; 59 } 60 61 return 0; 62} 63EXPORT_SYMBOL(arch_spin_trylock); 64 65void arch_spin_unlock_wait(arch_spinlock_t *lock) 66{ 67 u32 iterations = 0; 68 int curr = READ_ONCE(lock->current_ticket); 69 int next = READ_ONCE(lock->next_ticket); 70 71 /* Return immediately if unlocked. */ 72 if (next == curr) 73 return; 74 75 /* Wait until the current locker has released the lock. */ 76 do { 77 delay_backoff(iterations++); 78 } while (READ_ONCE(lock->current_ticket) == curr); 79} 80EXPORT_SYMBOL(arch_spin_unlock_wait); 81 82/* 83 * The low byte is always reserved to be the marker for a "tns" operation 84 * since the low bit is set to "1" by a tns. The next seven bits are 85 * zeroes. The next byte holds the "next" writer value, i.e. the ticket 86 * available for the next task that wants to write. The third byte holds 87 * the current writer value, i.e. the writer who holds the current ticket. 88 * If current == next == 0, there are no interested writers. 89 */ 90#define WR_NEXT_SHIFT _WR_NEXT_SHIFT 91#define WR_CURR_SHIFT _WR_CURR_SHIFT 92#define WR_WIDTH _WR_WIDTH 93#define WR_MASK ((1 << WR_WIDTH) - 1) 94 95/* 96 * The last eight bits hold the active reader count. This has to be 97 * zero before a writer can start to write. 98 */ 99#define RD_COUNT_SHIFT _RD_COUNT_SHIFT 100#define RD_COUNT_WIDTH _RD_COUNT_WIDTH 101#define RD_COUNT_MASK ((1 << RD_COUNT_WIDTH) - 1) 102 103 104/* 105 * We can get the read lock if everything but the reader bits (which 106 * are in the high part of the word) is zero, i.e. no active or 107 * waiting writers, no tns. 108 * 109 * We guard the tns/store-back with an interrupt critical section to 110 * preserve the semantic that the same read lock can be acquired in an 111 * interrupt context. 112 */ 113int arch_read_trylock(arch_rwlock_t *rwlock) 114{ 115 u32 val; 116 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1); 117 val = __insn_tns((int *)&rwlock->lock); 118 if (likely((val << _RD_COUNT_WIDTH) == 0)) { 119 val += 1 << RD_COUNT_SHIFT; 120 rwlock->lock = val; 121 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); 122 BUG_ON(val == 0); /* we don't expect wraparound */ 123 return 1; 124 } 125 if ((val & 1) == 0) 126 rwlock->lock = val; 127 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); 128 return 0; 129} 130EXPORT_SYMBOL(arch_read_trylock); 131 132/* 133 * Spin doing arch_read_trylock() until we acquire the lock. 134 * ISSUE: This approach can permanently starve readers. A reader who sees 135 * a writer could instead take a ticket lock (just like a writer would), 136 * and atomically enter read mode (with 1 reader) when it gets the ticket. 137 * This way both readers and writers would always make forward progress 138 * in a finite time. 139 */ 140void arch_read_lock(arch_rwlock_t *rwlock) 141{ 142 u32 iterations = 0; 143 while (unlikely(!arch_read_trylock(rwlock))) 144 delay_backoff(iterations++); 145} 146EXPORT_SYMBOL(arch_read_lock); 147 148void arch_read_unlock(arch_rwlock_t *rwlock) 149{ 150 u32 val, iterations = 0; 151 152 mb(); /* guarantee anything modified under the lock is visible */ 153 for (;;) { 154 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1); 155 val = __insn_tns((int *)&rwlock->lock); 156 if (likely((val & 1) == 0)) { 157 rwlock->lock = val - (1 << _RD_COUNT_SHIFT); 158 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); 159 break; 160 } 161 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); 162 delay_backoff(iterations++); 163 } 164} 165EXPORT_SYMBOL(arch_read_unlock); 166 167/* 168 * We don't need an interrupt critical section here (unlike for 169 * arch_read_lock) since we should never use a bare write lock where 170 * it could be interrupted by code that could try to re-acquire it. 171 */ 172void arch_write_lock(arch_rwlock_t *rwlock) 173{ 174 /* 175 * The trailing underscore on this variable (and curr_ below) 176 * reminds us that the high bits are garbage; we mask them out 177 * when we compare them. 178 */ 179 u32 my_ticket_; 180 u32 iterations = 0; 181 u32 val = __insn_tns((int *)&rwlock->lock); 182 183 if (likely(val == 0)) { 184 rwlock->lock = 1 << _WR_NEXT_SHIFT; 185 return; 186 } 187 188 /* 189 * Wait until there are no readers, then bump up the next 190 * field and capture the ticket value. 191 */ 192 for (;;) { 193 if (!(val & 1)) { 194 if ((val >> RD_COUNT_SHIFT) == 0) 195 break; 196 rwlock->lock = val; 197 } 198 delay_backoff(iterations++); 199 val = __insn_tns((int *)&rwlock->lock); 200 } 201 202 /* Take out the next ticket and extract my ticket value. */ 203 rwlock->lock = __insn_addb(val, 1 << WR_NEXT_SHIFT); 204 my_ticket_ = val >> WR_NEXT_SHIFT; 205 206 /* Wait until the "current" field matches our ticket. */ 207 for (;;) { 208 u32 curr_ = val >> WR_CURR_SHIFT; 209 u32 delta = ((my_ticket_ - curr_) & WR_MASK); 210 if (likely(delta == 0)) 211 break; 212 213 /* Delay based on how many lock-holders are still out there. */ 214 relax((256 / CYCLES_PER_RELAX_LOOP) * delta); 215 216 /* 217 * Get a non-tns value to check; we don't need to tns 218 * it ourselves. Since we're not tns'ing, we retry 219 * more rapidly to get a valid value. 220 */ 221 while ((val = rwlock->lock) & 1) 222 relax(4); 223 } 224} 225EXPORT_SYMBOL(arch_write_lock); 226 227int arch_write_trylock(arch_rwlock_t *rwlock) 228{ 229 u32 val = __insn_tns((int *)&rwlock->lock); 230 231 /* 232 * If a tns is in progress, or there's a waiting or active locker, 233 * or active readers, we can't take the lock, so give up. 234 */ 235 if (unlikely(val != 0)) { 236 if (!(val & 1)) 237 rwlock->lock = val; 238 return 0; 239 } 240 241 /* Set the "next" field to mark it locked. */ 242 rwlock->lock = 1 << _WR_NEXT_SHIFT; 243 return 1; 244} 245EXPORT_SYMBOL(arch_write_trylock); 246 247void arch_write_unlock(arch_rwlock_t *rwlock) 248{ 249 u32 val, eq, mask; 250 251 mb(); /* guarantee anything modified under the lock is visible */ 252 val = __insn_tns((int *)&rwlock->lock); 253 if (likely(val == (1 << _WR_NEXT_SHIFT))) { 254 rwlock->lock = 0; 255 return; 256 } 257 while (unlikely(val & 1)) { 258 /* Limited backoff since we are the highest-priority task. */ 259 relax(4); 260 val = __insn_tns((int *)&rwlock->lock); 261 } 262 mask = 1 << WR_CURR_SHIFT; 263 val = __insn_addb(val, mask); 264 eq = __insn_seqb(val, val << (WR_CURR_SHIFT - WR_NEXT_SHIFT)); 265 val = __insn_mz(eq & mask, val); 266 rwlock->lock = val; 267} 268EXPORT_SYMBOL(arch_write_unlock); 269