atomic.h: add atomic64 cmpxchg, xchg and add_unless to mips

[powerpc.git] / include / asm-mips / atomic.h

diff --git a/include/asm-mips/atomic.h b/include/asm-mips/atomic.h
index c1a2409..6423ffa 100644 (file)
--- a/include/asm-mips/atomic.h
+++ b/include/asm-mips/atomic.h
@@ -69,16 +69,19 @@ static __inline__ void atomic_add(int i, atomic_t * v)
                "1:     ll      %0, %1          # atomic_add            \n"
                "       addu    %0, %2                                  \n"
                "       sc      %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                v->counter += i;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 }
 
@@ -111,16 +114,19 @@ static __inline__ void atomic_sub(int i, atomic_t * v)
                "1:     ll      %0, %1          # atomic_sub            \n"
                "       subu    %0, %2                                  \n"
                "       sc      %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                v->counter -= i;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 }
 
@@ -155,8 +161,11 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
                "1:     ll      %1, %2          # atomic_add_return     \n"
                "       addu    %0, %1, %3                              \n"
                "       sc      %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       addu    %0, %1, %3                              \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -164,11 +173,11 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result += i;
                v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
        smp_mb();
@@ -204,8 +213,11 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
                "1:     ll      %1, %2          # atomic_sub_return     \n"
                "       subu    %0, %1, %3                              \n"
                "       sc      %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       subu    %0, %1, %3                              \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -213,11 +225,11 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result -= i;
                v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
        smp_mb();
@@ -267,10 +279,13 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
                "       bltz    %0, 1f                                  \n"
                "       sc      %0, %2                                  \n"
                "       .set    noreorder                               \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "        subu   %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
                "1:                                                     \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -278,12 +293,12 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result -= i;
                if (result >= 0)
                        v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
        smp_mb();
@@ -291,8 +306,8 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
        return result;
 }
 
-#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
-#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
+#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
+#define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
 
 /**
  * atomic_add_unless - add unless the number is a given value
@@ -305,7 +320,7 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
  */
 #define atomic_add_unless(v, a, u)                             \
 ({                                                             \
-       int c, old;                                             \
+       __typeof__((v)->counter) c, old;                        \
        c = atomic_read(v);                                     \
        while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
                c = old;                                        \
@@ -429,16 +444,19 @@ static __inline__ void atomic64_add(long i, atomic64_t * v)
                "1:     lld     %0, %1          # atomic64_add          \n"
                "       addu    %0, %2                                  \n"
                "       scd     %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                v->counter += i;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 }
 
@@ -471,16 +489,19 @@ static __inline__ void atomic64_sub(long i, atomic64_t * v)
                "1:     lld     %0, %1          # atomic64_sub          \n"
                "       subu    %0, %2                                  \n"
                "       scd     %0, %1                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter));
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                v->counter -= i;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 }
 
@@ -515,8 +536,11 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
                "1:     lld     %1, %2          # atomic64_add_return   \n"
                "       addu    %0, %1, %3                              \n"
                "       scd     %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       addu    %0, %1, %3                              \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -524,11 +548,11 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result += i;
                v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
        smp_mb();
@@ -564,8 +588,11 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
                "1:     lld     %1, %2          # atomic64_sub_return   \n"
                "       subu    %0, %1, %3                              \n"
                "       scd     %0, %2                                  \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "       subu    %0, %1, %3                              \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -573,11 +600,11 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result -= i;
                v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
        smp_mb();
@@ -627,10 +654,13 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
                "       bltz    %0, 1f                                  \n"
                "       scd     %0, %2                                  \n"
                "       .set    noreorder                               \n"
-               "       beqz    %0, 1b                                  \n"
+               "       beqz    %0, 2f                                  \n"
                "        dsubu  %0, %1, %3                              \n"
                "       .set    reorder                                 \n"
                "1:                                                     \n"
+               "       .subsection 2                                   \n"
+               "2:     b       1b                                      \n"
+               "       .previous                                       \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (v->counter)
                : "Ir" (i), "m" (v->counter)
@@ -638,12 +668,12 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
        } else {
                unsigned long flags;
 
-               local_irq_save(flags);
+               raw_local_irq_save(flags);
                result = v->counter;
                result -= i;
                if (result >= 0)
                        v->counter = result;
-               local_irq_restore(flags);
+               raw_local_irq_restore(flags);
        }
 
        smp_mb();
@@ -651,6 +681,29 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
        return result;
 }
 
+#define atomic64_cmpxchg(v, o, n) \
+       (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n)))
+#define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
+
+/**
+ * atomic64_add_unless - add unless the number is a given value
+ * @v: pointer of type atomic64_t
+ * @a: the amount to add to v...
+ * @u: ...unless v is equal to u.
+ *
+ * Atomically adds @a to @v, so long as it was not @u.
+ * Returns non-zero if @v was not @u, and zero otherwise.
+ */
+#define atomic64_add_unless(v, a, u)                           \
+({                                                             \
+       __typeof__((v)->counter) c, old;                        \
+       c = atomic_read(v);                                     \
+       while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \
+               c = old;                                        \
+       c != (u);                                               \
+})
+#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
+
 #define atomic64_dec_return(v) atomic64_sub_return(1,(v))
 #define atomic64_inc_return(v) atomic64_add_return(1,(v))