include/asm-ppc64/spinlock.h

   1 #ifndef __ASM_SPINLOCK_H
   2 #define __ASM_SPINLOCK_H
   3
   4 /*
   5  * Simple spin lock operations.
   6  *
   7  * Copyright (C) 2001 Paul Mackerras <paulus@au.ibm.com>, IBM
   8  * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
   9  * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com>, IBM
  10  *   Rework to support virtual processors
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version
  15  * 2 of the License, or (at your option) any later version.
  16  */
  17
  18 #include <asm/memory.h>
  19 #include <asm/hvcall.h>
  20
  21 /*
  22  * The following define is being used to select basic or shared processor
  23  * locking when running on an RPA platform.  As we do more performance
  24  * tuning, I would expect this selection mechanism to change.  Dave E.
  25  */
  26 /* #define SPLPAR_LOCKS */
  27
  28 typedef struct {
  29         volatile unsigned long lock;
  30 } spinlock_t;
  31
  32 #ifdef __KERNEL__
  33 #define SPIN_LOCK_UNLOCKED      (spinlock_t) { 0 }
  34
  35 #define spin_is_locked(x)       ((x)->lock != 0)
  36
  37 static __inline__ int spin_trylock(spinlock_t *lock)
  38 {
  39         unsigned long tmp;
  40
  41         __asm__ __volatile__(
  42 "1:     ldarx           %0,0,%1         # spin_trylock\n\
  43         cmpdi           0,%0,0\n\
  44         li              %0,0\n\
  45         bne-            2f\n\
  46         li              %0,1\n\
  47         stdcx.          13,0,%1\n\
  48         bne-            1b\n\
  49         isync\n\
  50 2:"     : "=&r"(tmp)
  51         : "r"(&lock->lock)
  52         : "cr0", "memory");
  53
  54         return tmp;
  55 }
  56
  57 /*
  58  * Spin lock states:
  59  *   0        : Unlocked
  60  *   Negative : Locked.  Value is paca pointer (0xc...0) of holder
  61  */
  62 #ifdef CONFIG_PPC_ISERIES
  63 static __inline__ void spin_lock(spinlock_t *lock)
  64 {
  65         unsigned long tmp, tmp2;
  66
  67         __asm__ __volatile__(
  68         "b              2f              # spin_lock\n\
  69 1:"
  70         HMT_LOW
  71 "       ldx             %0,0,%2         # load the lock value\n\
  72         cmpdi           0,%0,0          # if not locked, try to acquire\n\
  73         beq-            2f\n\
  74         lwz             5,0x280(%0)     # load yield counter\n\
  75         andi.           %1,5,1          # if even then spin\n\
  76         beq             1b\n\
  77         lwsync                          # if odd, give up cycles\n\
  78         ldx             %1,0,%2         # reverify the lock holder\n\
  79         cmpd            %0,%1\n\
  80         bne             1b              # new holder so restart\n\
  81         li              3,0x25          # yield hcall 0x8-12 \n\
  82         rotrdi          3,3,1           #   put the bits in the right spot\n\
  83         lhz             4,0x18(%0)      # processor number\n\
  84         sldi            4,4,32          #   move into top half of word\n\
  85         or              5,5,4           # r5 has yield cnt - or it in\n\
  86         li              4,2             # yield to processor\n\
  87         li              0,-1            # indicate an hcall\n\
  88         sc                              # do the hcall \n\
  89         b               1b\n\
  90 2: \n"
  91         HMT_MEDIUM
  92 "       ldarx           %0,0,%2\n\
  93         cmpdi           0,%0,0\n\
  94         bne-            1b\n\
  95         stdcx.          13,0,%2\n\
  96         bne-            2b\n\
  97         isync"
  98         : "=&r"(tmp), "=&r"(tmp2)
  99         : "r"(&lock->lock)
 100         : "r0", "r3", "r4", "r5", "ctr", "cr0", "cr1", "cr2", "cr3", "cr4",
 101           "xer", "memory");
 102 }
 103 #else
 104 #ifdef SPLPAR_LOCKS
 105 static __inline__ void spin_lock(spinlock_t *lock)
 106 {
 107         unsigned long tmp, tmp2;
 108
 109         __asm__ __volatile__(
 110         "b              2f              # spin_lock\n\
 111 1:"
 112         HMT_LOW
 113 "       ldx             %0,0,%2         # load the lock value\n\
 114         cmpdi           0,%0,0          # if not locked, try to acquire\n\
 115         beq-            2f\n\
 116         lwz             5,0x280(%0)     # load dispatch counter\n\
 117         andi.           %1,5,1          # if even then spin\n\
 118         beq             1b\n\
 119         lwsync                          # if odd, give up cycles\n\
 120         ldx             %1,0,%2         # reverify the lock holder\n\
 121         cmpd            %0,%1\n\
 122         bne             1b              # new holder so restart\n\
 123         li              3,0xE4          # give up the cycles H_CONFER\n\
 124         lhz             4,0x18(%0)      # processor number\n\
 125                                         # r5 has dispatch cnt already\n"
 126         HVSC
 127 "        b               1b\n\
 128 2: \n"
 129         HMT_MEDIUM
 130 "       ldarx           %0,0,%2\n\
 131         cmpdi           0,%0,0\n\
 132         bne-            1b\n\
 133         stdcx.          13,0,%2\n\
 134         bne-            2b\n\
 135         isync"
 136         : "=&r"(tmp), "=&r"(tmp2)
 137         : "r"(&lock->lock)
 138         : "r3", "r4", "r5", "cr0", "cr1", "ctr", "xer", "memory");
 139 }
 140 #else
 141 static __inline__ void spin_lock(spinlock_t *lock)
 142 {
 143         unsigned long tmp;
 144
 145         __asm__ __volatile__(
 146        "b               2f              # spin_lock\n\
 147 1:"
 148         HMT_LOW
 149 "       ldx             %0,0,%1         # load the lock value\n\
 150         cmpdi           0,%0,0          # if not locked, try to acquire\n\
 151         bne+            1b\n\
 152 2: \n"
 153         HMT_MEDIUM
 154 "       ldarx           %0,0,%1\n\
 155         cmpdi           0,%0,0\n\
 156         bne-            1b\n\
 157         stdcx.          13,0,%1\n\
 158         bne-            2b\n\
 159         isync"
 160         : "=&r"(tmp)
 161         : "r"(&lock->lock)
 162         : "cr0", "memory");
 163 }
 164 #endif
 165 #endif
 166
 167 static __inline__ void spin_unlock(spinlock_t *lock)
 168 {
 169         __asm__ __volatile__("lwsync    # spin_unlock": : :"memory");
 170         lock->lock = 0;
 171 }
 172
 173 /*
 174  * Read-write spinlocks, allowing multiple readers
 175  * but only one writer.
 176  *
 177  * NOTE! it is quite common to have readers in interrupts
 178  * but no interrupt writers. For those circumstances we
 179  * can "mix" irq-safe locks - any writer needs to get a
 180  * irq-safe write-lock, but readers can get non-irqsafe
 181  * read-locks.
 182  *
 183  * Write lock states:
 184  *   0        : Unlocked
 185  *   Positive : Reader count
 186  *   Negative : Writer locked.  Value is paca pointer (0xc...0) of holder
 187  *
 188  * If lock is not held, try to acquire.
 189  * If lock is held by a writer, yield cycles to the holder.
 190  * If lock is help by reader(s), spin.
 191  */
 192 typedef struct {
 193         volatile signed long lock;
 194 } rwlock_t;
 195
 196 #define RW_LOCK_UNLOCKED (rwlock_t) { 0 }
 197
 198 static __inline__ int read_trylock(rwlock_t *rw)
 199 {
 200         unsigned long tmp;
 201         unsigned int ret;
 202
 203         __asm__ __volatile__(
 204 "1:     ldarx           %0,0,%2         # read_trylock\n\
 205         li              %1,0\n\
 206         addic.          %0,%0,1\n\
 207         ble-            2f\n\
 208         stdcx.          %0,0,%2\n\
 209         bne-            1b\n\
 210         li              %1,1\n\
 211         isync\n\
 212 2:"     : "=&r"(tmp), "=&r"(ret)
 213         : "r"(&rw->lock)
 214         : "cr0", "memory");
 215
 216         return ret;
 217 }
 218
 219 #ifdef CONFIG_PPC_ISERIES
 220 static __inline__ void read_lock(rwlock_t *rw)
 221 {
 222         unsigned long tmp, tmp2;
 223
 224         __asm__ __volatile__(
 225         "b              2f              # read_lock\n\
 226 1:"
 227         HMT_LOW
 228 "       ldx             %0,0,%2\n\
 229         cmpdi           0,%0,0\n\
 230         bge-            2f\n\
 231         lwz             5,0x280(%0)     # load yield counter\n\
 232         andi.           %1,5,1          # if even then spin\n\
 233         beq             1b\n\
 234         lwsync                          # if odd, give up cycles\n\
 235         ldx             %1,0,%2         # reverify the lock holder\n\
 236         cmpd            %0,%1\n\
 237         bne             1b              # new holder so restart\n\
 238         li              3,0x25          # yield hcall 0x8-12 \n\
 239         rotrdi          3,3,1           #   put the bits in the right spot\n\
 240         lhz             4,0x18(%0)      # processor number\n\
 241         sldi            4,4,32          #   move into top half of word\n\
 242         or              5,5,4           # r5 has yield cnt - or it in\n\
 243         li              4,2             # yield to processor\n\
 244         li              0,-1            # indicate an hcall\n\
 245         sc                              # do the hcall \n\
 246 2: \n"
 247         HMT_MEDIUM
 248 "       ldarx           %0,0,%2\n\
 249         addic.          %0,%0,1\n\
 250         ble-            1b\n\
 251         stdcx.          %0,0,%2\n\
 252         bne-            2b\n\
 253         isync"
 254         : "=&r"(tmp), "=&r"(tmp2)
 255         : "r"(&rw->lock)
 256         : "r0", "r3", "r4", "r5", "ctr", "cr0", "cr1", "cr2", "cr3", "cr4",
 257           "xer", "memory");
 258 }
 259 #else
 260 #ifdef SPLPAR_LOCKS
 261 static __inline__ void read_lock(rwlock_t *rw)
 262 {
 263         unsigned long tmp, tmp2;
 264
 265         __asm__ __volatile__(
 266         "b              2f              # read_lock\n\
 267 1:"
 268         HMT_LOW
 269 "       ldx             %0,0,%2\n\
 270         cmpdi           0,%0,0\n\
 271         bge-            2f\n\
 272         lwz             5,0x280(%0)     # load dispatch counter\n\
 273         andi.           %1,5,1          # if even then spin\n\
 274         beq             1b\n\
 275         lwsync                          # if odd, give up cycles\n\
 276         ldx             %1,0,%2         # reverify the lock holder\n\
 277         cmpd            %0,%1\n\
 278         bne             1b              # new holder so restart\n\
 279         li              3,0xE4          # give up the cycles H_CONFER\n\
 280         lhz             4,0x18(%0)      # processor number\n\
 281                                         # r5 has dispatch cnt already\n"
 282         HVSC
 283 "2: \n"
 284         HMT_MEDIUM
 285 "       ldarx           %0,0,%2\n\
 286         addic.          %0,%0,1\n\
 287         ble-            1b\n\
 288         stdcx.          %0,0,%2\n\
 289         bne-            2b\n\
 290         isync"
 291         : "=&r"(tmp), "=&r"(tmp2)
 292         : "r"(&rw->lock)
 293         : "r3", "r4", "r5", "cr0", "cr1", "ctr", "xer", "memory");
 294 }
 295 #else
 296 static __inline__ void read_lock(rwlock_t *rw)
 297 {
 298         unsigned long tmp;
 299
 300         __asm__ __volatile__(
 301         "b              2f              # read_lock\n\
 302 1:"
 303         HMT_LOW
 304 "       ldx             %0,0,%1\n\
 305         cmpdi           0,%0,0\n\
 306         blt+            1b\n\
 307 2: \n"
 308         HMT_MEDIUM
 309 "       ldarx           %0,0,%1\n\
 310         addic.          %0,%0,1\n\
 311         ble-            1b\n\
 312         stdcx.          %0,0,%1\n\
 313         bne-            2b\n\
 314         isync"
 315         : "=&r"(tmp)
 316         : "r"(&rw->lock)
 317         : "cr0", "memory");
 318 }
 319 #endif
 320 #endif
 321
 322 static __inline__ void read_unlock(rwlock_t *rw)
 323 {
 324         unsigned long tmp;
 325
 326         __asm__ __volatile__(
 327         "eieio                          # read_unlock\n\
 328 1:      ldarx           %0,0,%1\n\
 329         addic           %0,%0,-1\n\
 330         stdcx.          %0,0,%1\n\
 331         bne-            1b"
 332         : "=&r"(tmp)
 333         : "r"(&rw->lock)
 334         : "cr0", "memory");
 335 }
 336
 337 static __inline__ int write_trylock(rwlock_t *rw)
 338 {
 339         unsigned long tmp;
 340         unsigned long ret;
 341
 342         __asm__ __volatile__(
 343 "1:     ldarx           %0,0,%2         # write_trylock\n\
 344         cmpdi           0,%0,0\n\
 345         li              %1,0\n\
 346         bne-            2f\n\
 347         stdcx.          13,0,%2\n\
 348         bne-            1b\n\
 349         li              %1,1\n\
 350         isync\n\
 351 2:"     : "=&r"(tmp), "=&r"(ret)
 352         : "r"(&rw->lock)
 353         : "cr0", "memory");
 354
 355         return ret;
 356 }
 357
 358 #ifdef CONFIG_PPC_ISERIES
 359 static __inline__ void write_lock(rwlock_t *rw)
 360 {
 361         unsigned long tmp, tmp2;
 362
 363         __asm__ __volatile__(
 364         "b              2f              # spin_lock\n\
 365 1:"
 366         HMT_LOW
 367 "       ldx             %0,0,%2         # load the lock value\n\
 368         cmpdi           0,%0,0          # if not locked(0), try to acquire\n\
 369         beq-            2f\n\
 370         bgt             1b              # negative(0xc..)->cycles to holder\n"
 371 "3:     lwz             5,0x280(%0)     # load yield counter\n\
 372         andi.           %1,5,1          # if even then spin\n\
 373         beq             1b\n\
 374         lwsync                          # if odd, give up cycles\n\
 375         ldx             %1,0,%2         # reverify the lock holder\n\
 376         cmpd            %0,%1\n\
 377         bne             1b              # new holder so restart\n\
 378         lhz             4,0x18(%0)      # processor number\n\
 379         sldi            4,4,32          #   move into top half of word\n\
 380         or              5,5,4           # r5 has yield cnt - or it in\n\
 381         li              3,0x25          # yield hcall 0x8-12 \n\
 382         rotrdi          3,3,1           #   put the bits in the right spot\n\
 383         li              4,2             # yield to processor\n\
 384         li              0,-1            # indicate an hcall\n\
 385         sc                              # do the hcall \n\
 386 2: \n"
 387         HMT_MEDIUM
 388 "       ldarx           %0,0,%2\n\
 389         cmpdi           0,%0,0\n\
 390         bne-            1b\n\
 391         stdcx.          13,0,%2\n\
 392         bne-            2b\n\
 393         isync"
 394         : "=&r"(tmp), "=&r"(tmp2)
 395         : "r"(&rw->lock)
 396         : "r0", "r3", "r4", "r5", "ctr", "cr0", "cr1", "cr2", "cr3", "cr4",
 397           "xer", "memory");
 398 }
 399 #else
 400 #ifdef SPLPAR_LOCKS
 401 static __inline__ void write_lock(rwlock_t *rw)
 402 {
 403         unsigned long tmp, tmp2;
 404
 405         __asm__ __volatile__(
 406         "b              2f              # spin_lock\n\
 407 1:"
 408         HMT_LOW
 409 "       ldx             %0,0,%2         # load the lock value\n\
 410         li              3,0xE4          # give up the cycles H_CONFER\n\
 411         cmpdi           0,%0,0          # if not locked(0), try to acquire\n\
 412         beq-            2f\n\
 413         blt             3f              # negative(0xc..)->confer to holder\n\
 414         b               1b\n"
 415 "3:      lwz             5,0x280(%0)     # load dispatch counter\n\
 416         andi.           %1,5,1          # if even then spin\n\
 417         beq             1b\n\
 418         lwsync                          # if odd, give up cycles\n\
 419         ldx             %1,0,%2         # reverify the lock holder\n\
 420         cmpd            %0,%1\n\
 421         bne             1b              # new holder so restart\n\
 422         lhz             4,0x18(%0)      # processor number\n\
 423                                         # r5 has dispatch cnt already\n"
 424         HVSC
 425 "        b               1b\n\
 426 2: \n"
 427         HMT_MEDIUM
 428 "       ldarx           %0,0,%2\n\
 429         cmpdi           0,%0,0\n\
 430         bne-            1b\n\
 431         stdcx.          13,0,%2\n\
 432         bne-            2b\n\
 433         isync"
 434         : "=&r"(tmp), "=&r"(tmp2)
 435         : "r"(&rw->lock)
 436         : "r3", "r4", "r5", "cr0", "cr1", "ctr", "xer", "memory");
 437 }
 438 #else
 439 static __inline__ void write_lock(rwlock_t *rw)
 440 {
 441         unsigned long tmp;
 442
 443         __asm__ __volatile__(
 444         "b              2f              # spin_lock\n\
 445 1:"
 446         HMT_LOW
 447 "       ldx             %0,0,%1         # load the lock value\n\
 448         cmpdi           0,%0,0          # if not locked(0), try to acquire\n\
 449         bne+            1b\n\
 450 2: \n"
 451         HMT_MEDIUM
 452 "       ldarx           %0,0,%1\n\
 453         cmpdi           0,%0,0\n\
 454         bne-            1b\n\
 455         stdcx.          13,0,%1\n\
 456         bne-            2b\n\
 457         isync"
 458         : "=&r"(tmp)
 459         : "r"(&rw->lock)
 460         : "cr0", "memory");
 461 }
 462 #endif
 463 #endif
 464
 465 static __inline__ void write_unlock(rwlock_t *rw)
 466 {
 467         __asm__ __volatile__("lwsync            # write_unlock": : :"memory");
 468         rw->lock = 0;
 469 }
 470
 471 static __inline__ int is_read_locked(rwlock_t *rw)
 472 {
 473         return rw->lock > 0;
 474 }
 475
 476 static __inline__ int is_write_locked(rwlock_t *rw)
 477 {
 478         return rw->lock < 0;
 479 }
 480
 481 #define spin_lock_init(x)      do { *(x) = SPIN_LOCK_UNLOCKED; } while(0)
 482 #define spin_unlock_wait(x)    do { barrier(); } while(spin_is_locked(x))
 483
 484 #define rwlock_init(x)         do { *(x) = RW_LOCK_UNLOCKED; } while(0)
 485
 486 #endif /* __KERNEL__ */
 487 #endif /* __ASM_SPINLOCK_H */