and changed files

[powerpc.git] / include / asm-mips / atomic.h

diff --git a/include/asm-mips/atomic.h b/include/asm-mips/atomic.h
index 7978d8e..62daa74 100644 (file)
--- a/include/asm-mips/atomic.h
+++ b/include/asm-mips/atomic.h
@@ -15,8 +15,10 @@
 #define _ASM_ATOMIC_H
 
 #include <linux/irqflags.h>
+#include <asm/barrier.h>
 #include <asm/cpu-features.h>
 #include <asm/war.h>
+#include <asm/system.h>
 
 typedef struct { volatile int counter; } atomic_t;
 
@@ -68,16 +70,19 @@ static __inline__ void atomic_add(int i, atomic_t * v)
                "1:     ll      %0, %1          # atomic_add            \n"
                "       addu    %0, %2                                  \n"
                "       sc      %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                v->counter += i;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 }
 
@@ -110,16 +115,19 @@ static __inline__ void atomic_sub(int i, atomic_t * v)
                "1:     ll      %0, %1          # atomic_sub            \n"
                "       subu    %0, %2                                  \n"
                "       sc      %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                v->counter -= i;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 }
 
@@ -130,6 +138,8 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -140,7 +150,6 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
                "       sc      %0, %2                                  \n"
                "       beqzl   %0, 1b                                  \n"
                "       addu    %0, %1, %3                              \n"
-               "       sync                                            \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -153,9 +162,11 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
                "1:     ll      %1, %2          # atomic_add_return     \n"
                "       addu    %0, %1, %3                              \n"
                "       sc      %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       addu    %0, %1, %3                              \n"
-               "       sync                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -163,13 +174,15 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result += i;
                v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -177,6 +190,8 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -187,7 +202,6 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
                "       sc      %0, %2                                  \n"
                "       beqzl   %0, 1b                                  \n"
                "       subu    %0, %1, %3                              \n"
-               "       sync                                            \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -200,9 +214,11 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
                "1:     ll      %1, %2          # atomic_sub_return     \n"
                "       subu    %0, %1, %3                              \n"
                "       sc      %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       subu    %0, %1, %3                              \n"
-               "       sync                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -210,13 +226,15 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result -= i;
                v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -232,6 +250,8 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -245,7 +265,6 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
                "       beqzl   %0, 1b                                  \n"
                "        subu   %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
-               "       sync                                            \n"
                "1:                                                     \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
@@ -261,11 +280,13 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
                "       bltz    %0, 1f                                  \n"
                "       sc      %0, %2                                  \n"
                "       .set    noreorder                               \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "        subu   %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
-               "       sync                                            \n"
                "1:                                                     \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -273,19 +294,21 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result -= i;
                if (result >= 0)
                        v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
-#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
-#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
+#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
+#define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
 
 /**
  * atomic_add_unless - add unless the number is a given value
@@ -296,14 +319,20 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
  * Atomically adds @a to @v, so long as it was not @u.
  * Returns non-zero if @v was not @u, and zero otherwise.
  */
-#define atomic_add_unless(v, a, u)                             \
-({                                                             \
-       int c, old;                                             \
-       c = atomic_read(v);                                     \
-       while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
-               c = old;                                        \
-       c != (u);                                               \
-})
+static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
+{
+       int c, old;
+       c = atomic_read(v);
+       for (;;) {
+               if (unlikely(c == (u)))
+                       break;
+               old = atomic_cmpxchg((v), c, c + (a));
+               if (likely(old == c))
+                       break;
+               c = old;
+       }
+       return c != (u);
+}
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
 
 #define atomic_dec_return(v) atomic_sub_return(1,(v))
@@ -375,7 +404,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
 
 #ifdef CONFIG_64BIT
 
-typedef struct { volatile __s64 counter; } atomic64_t;
+typedef struct { volatile long counter; } atomic64_t;
 
 #define ATOMIC64_INIT(i)    { (i) }
 
@@ -422,16 +451,19 @@ static __inline__ void atomic64_add(long i, atomic64_t * v)
                "1:     lld     %0, %1          # atomic64_add          \n"
                "       addu    %0, %2                                  \n"
                "       scd     %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                v->counter += i;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 }
 
@@ -464,16 +496,19 @@ static __inline__ void atomic64_sub(long i, atomic64_t * v)
                "1:     lld     %0, %1          # atomic64_sub          \n"
                "       subu    %0, %2                                  \n"
                "       scd     %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                v->counter -= i;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 }
 
@@ -484,6 +519,8 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -494,7 +531,6 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
                "       scd     %0, %2                                  \n"
                "       beqzl   %0, 1b                                  \n"
                "       addu    %0, %1, %3                              \n"
-               "       sync                                            \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -507,9 +543,11 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
                "1:     lld     %1, %2          # atomic64_add_return   \n"
                "       addu    %0, %1, %3                              \n"
                "       scd     %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       addu    %0, %1, %3                              \n"
-               "       sync                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -517,13 +555,15 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result += i;
                v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -531,6 +571,8 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -541,7 +583,6 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
                "       scd     %0, %2                                  \n"
                "       beqzl   %0, 1b                                  \n"
                "       subu    %0, %1, %3                              \n"
-               "       sync                                            \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -554,9 +595,11 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
                "1:     lld     %1, %2          # atomic64_sub_return   \n"
                "       subu    %0, %1, %3                              \n"
                "       scd     %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       subu    %0, %1, %3                              \n"
-               "       sync                                            \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -564,13 +607,15 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result -= i;
                v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
@@ -586,6 +631,8 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
 {
        unsigned long result;
 
+       smp_mb();
+
        if (cpu_has_llsc && R10000_LLSC_WAR) {
                unsigned long temp;
 
@@ -599,7 +646,6 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
                "       beqzl   %0, 1b                                  \n"
                "        dsubu  %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
-               "       sync                                            \n"
                "1:                                                     \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
@@ -615,11 +661,13 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
                "       bltz    %0, 1f                                  \n"
                "       scd     %0, %2                                  \n"
                "       .set    noreorder                               \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "        dsubu  %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
-               "       sync                                            \n"
                "1:                                                     \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -627,17 +675,49 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result -= i;
                if (result >= 0)
                        v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
+       smp_mb();
+
        return result;
 }
 
+#define atomic64_cmpxchg(v, o, n) \
+       (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n)))
+#define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
+
+/**
+ * atomic64_add_unless - add unless the number is a given value
+ * @v: pointer of type atomic64_t
+ * @a: the amount to add to v...
+ * @u: ...unless v is equal to u.
+ *
+ * Atomically adds @a to @v, so long as it was not @u.
+ * Returns non-zero if @v was not @u, and zero otherwise.
+ */
+static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
+{
+       long c, old;
+       c = atomic64_read(v);
+       for (;;) {
+               if (unlikely(c == (u)))
+                       break;
+               old = atomic64_cmpxchg((v), c, c + (a));
+               if (likely(old == c))
+                       break;
+               c = old;
+       }
+       return c != (u);
+}
+
+#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
+
 #define atomic64_dec_return(v) atomic64_sub_return(1,(v))
 #define atomic64_inc_return(v) atomic64_add_return(1,(v))