Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[linux] / arch / x86 / include / asm / refcount.h

diff --git a/arch/x86/include/asm/refcount.h b/arch/x86/include/asm/refcount.h
index 19b9052..a8b5e1e 100644 (file)
--- a/arch/x86/include/asm/refcount.h
+++ b/arch/x86/include/asm/refcount.h
@@ -4,6 +4,41 @@
  * x86-specific implementation of refcount_t. Based on PAX_REFCOUNT from
  * PaX/grsecurity.
  */
+
+#ifdef __ASSEMBLY__
+
+#include <asm/asm.h>
+#include <asm/bug.h>
+
+.macro REFCOUNT_EXCEPTION counter:req
+       .pushsection .text..refcount
+111:   lea \counter, %_ASM_CX
+112:   ud2
+       ASM_UNREACHABLE
+       .popsection
+113:   _ASM_EXTABLE_REFCOUNT(112b, 113b)
+.endm
+
+/* Trigger refcount exception if refcount result is negative. */
+.macro REFCOUNT_CHECK_LT_ZERO counter:req
+       js 111f
+       REFCOUNT_EXCEPTION counter="\counter"
+.endm
+
+/* Trigger refcount exception if refcount result is zero or negative. */
+.macro REFCOUNT_CHECK_LE_ZERO counter:req
+       jz 111f
+       REFCOUNT_CHECK_LT_ZERO counter="\counter"
+.endm
+
+/* Trigger refcount exception unconditionally. */
+.macro REFCOUNT_ERROR counter:req
+       jmp 111f
+       REFCOUNT_EXCEPTION counter="\counter"
+.endm
+
+#else /* __ASSEMBLY__ */
+
 #include <linux/refcount.h>
 #include <asm/bug.h>
 
@@ -15,34 +50,11 @@
  * central refcount exception. The fixup address for the exception points
  * back to the regular execution flow in .text.
  */
-#define _REFCOUNT_EXCEPTION                            \
-       ".pushsection .text..refcount\n"                \
-       "111:\tlea %[counter], %%" _ASM_CX "\n"         \
-       "112:\t" ASM_UD2 "\n"                           \
-       ASM_UNREACHABLE                                 \
-       ".popsection\n"                                 \
-       "113:\n"                                        \
-       _ASM_EXTABLE_REFCOUNT(112b, 113b)
-
-/* Trigger refcount exception if refcount result is negative. */
-#define REFCOUNT_CHECK_LT_ZERO                         \
-       "js 111f\n\t"                                   \
-       _REFCOUNT_EXCEPTION
-
-/* Trigger refcount exception if refcount result is zero or negative. */
-#define REFCOUNT_CHECK_LE_ZERO                         \
-       "jz 111f\n\t"                                   \
-       REFCOUNT_CHECK_LT_ZERO
-
-/* Trigger refcount exception unconditionally. */
-#define REFCOUNT_ERROR                                 \
-       "jmp 111f\n\t"                                  \
-       _REFCOUNT_EXCEPTION
 
 static __always_inline void refcount_add(unsigned int i, refcount_t *r)
 {
        asm volatile(LOCK_PREFIX "addl %1,%0\n\t"
-               REFCOUNT_CHECK_LT_ZERO
+               "REFCOUNT_CHECK_LT_ZERO counter=\"%[counter]\""
                : [counter] "+m" (r->refs.counter)
                : "ir" (i)
                : "cc", "cx");
@@ -51,7 +63,7 @@ static __always_inline void refcount_add(unsigned int i, refcount_t *r)
 static __always_inline void refcount_inc(refcount_t *r)
 {
        asm volatile(LOCK_PREFIX "incl %0\n\t"
-               REFCOUNT_CHECK_LT_ZERO
+               "REFCOUNT_CHECK_LT_ZERO counter=\"%[counter]\""
                : [counter] "+m" (r->refs.counter)
                : : "cc", "cx");
 }
@@ -59,7 +71,7 @@ static __always_inline void refcount_inc(refcount_t *r)
 static __always_inline void refcount_dec(refcount_t *r)
 {
        asm volatile(LOCK_PREFIX "decl %0\n\t"
-               REFCOUNT_CHECK_LE_ZERO
+               "REFCOUNT_CHECK_LE_ZERO counter=\"%[counter]\""
                : [counter] "+m" (r->refs.counter)
                : : "cc", "cx");
 }
@@ -67,14 +79,17 @@ static __always_inline void refcount_dec(refcount_t *r)
 static __always_inline __must_check
 bool refcount_sub_and_test(unsigned int i, refcount_t *r)
 {
-       GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl", REFCOUNT_CHECK_LT_ZERO,
-                                 r->refs.counter, "er", i, "%0", e, "cx");
+
+       return GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl",
+                                        "REFCOUNT_CHECK_LT_ZERO counter=\"%[var]\"",
+                                        r->refs.counter, e, "er", i, "cx");
 }
 
 static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r)
 {
-       GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl", REFCOUNT_CHECK_LT_ZERO,
-                                r->refs.counter, "%0", e, "cx");
+       return GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl",
+                                       "REFCOUNT_CHECK_LT_ZERO counter=\"%[var]\"",
+                                       r->refs.counter, e, "cx");
 }
 
 static __always_inline __must_check
@@ -91,7 +106,7 @@ bool refcount_add_not_zero(unsigned int i, refcount_t *r)
 
                /* Did we try to increment from/to an undesirable state? */
                if (unlikely(c < 0 || c == INT_MAX || result < c)) {
-                       asm volatile(REFCOUNT_ERROR
+                       asm volatile("REFCOUNT_ERROR counter=\"%[counter]\""
                                     : : [counter] "m" (r->refs.counter)
                                     : "cc", "cx");
                        break;
@@ -107,4 +122,6 @@ static __always_inline __must_check bool refcount_inc_not_zero(refcount_t *r)
        return refcount_add_not_zero(1, r);
 }
 
+#endif /* __ASSEMBLY__ */
+
 #endif