SLUB: move tracking definitions and check_valid_pointer() away from debug code

[powerpc.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 07492a8..b9e0536 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -87,6 +87,21 @@
  *                     the fast path.
  */
 
+static inline int SlabDebug(struct page *page)
+{
+       return PageError(page);
+}
+
+static inline void SetSlabDebug(struct page *page)
+{
+       SetPageError(page);
+}
+
+static inline void ClearSlabDebug(struct page *page)
+{
+       ClearPageError(page);
+}
+
 /*
  * Issues still to be resolved:
  *
@@ -180,6 +195,18 @@ static enum {
 static DECLARE_RWSEM(slub_lock);
 LIST_HEAD(slab_caches);
 
+/*
+ * Tracking user of a slab.
+ */
+struct track {
+       void *addr;             /* Called from address */
+       int cpu;                /* Was running on cpu */
+       int pid;                /* Pid context */
+       unsigned long when;     /* When did the operation occur */
+};
+
+enum track_item { TRACK_ALLOC, TRACK_FREE };
+
 #ifdef CONFIG_SYSFS
 static int sysfs_slab_add(struct kmem_cache *);
 static int sysfs_slab_alias(struct kmem_cache *, const char *);
@@ -208,6 +235,23 @@ static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
 #endif
 }
 
+static inline int check_valid_pointer(struct kmem_cache *s,
+                               struct page *page, const void *object)
+{
+       void *base;
+
+       if (!object)
+               return 1;
+
+       base = page_address(page);
+       if (object < base || object >= base + s->objects * s->size ||
+               (object - base) % s->size) {
+               return 0;
+       }
+
+       return 1;
+}
+
 /*
  * Slow version of get and set free pointer.
  *
@@ -275,18 +319,6 @@ static void print_section(char *text, u8 *addr, unsigned int length)
        }
 }
 
-/*
- * Tracking user of a slab.
- */
-struct track {
-       void *addr;             /* Called from address */
-       int cpu;                /* Was running on cpu */
-       int pid;                /* Pid context */
-       unsigned long when;     /* When did the operation occur */
-};
-
-enum track_item { TRACK_ALLOC, TRACK_FREE };
-
 static struct track *get_track(struct kmem_cache *s, void *object,
        enum track_item alloc)
 {
@@ -421,23 +453,6 @@ static int check_bytes(u8 *start, unsigned int value, unsigned int bytes)
        return 1;
 }
 
-static inline int check_valid_pointer(struct kmem_cache *s,
-                               struct page *page, const void *object)
-{
-       void *base;
-
-       if (!object)
-               return 1;
-
-       base = page_address(page);
-       if (object < base || object >= base + s->objects * s->size ||
-               (object - base) % s->size) {
-               return 0;
-       }
-
-       return 1;
-}
-
 /*
  * Object layout:
  *
@@ -790,6 +805,22 @@ fail:
        return 0;
 }
 
+static void trace(struct kmem_cache *s, struct page *page, void *object, int alloc)
+{
+       if (s->flags & SLAB_TRACE) {
+               printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
+                       s->name,
+                       alloc ? "alloc" : "free",
+                       object, page->inuse,
+                       page->freelist);
+
+               if (!alloc)
+                       print_section("Object", (void *)object, s->objsize);
+
+               dump_stack();
+       }
+}
+
 /*
  * Slab allocation and freeing
  */
@@ -823,7 +854,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 static void setup_object(struct kmem_cache *s, struct page *page,
                                void *object)
 {
-       if (PageError(page)) {
+       if (SlabDebug(page)) {
                init_object(s, object, 0);
                init_tracking(s, object);
        }
@@ -858,7 +889,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
        page->flags |= 1 << PG_slab;
        if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON |
                        SLAB_STORE_USER | SLAB_TRACE))
-               page->flags |= 1 << PG_error;
+               SetSlabDebug(page);
 
        start = page_address(page);
        end = start + s->objects * s->size;
@@ -887,7 +918,7 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
 {
        int pages = 1 << s->order;
 
-       if (unlikely(PageError(page) || s->dtor)) {
+       if (unlikely(SlabDebug(page) || s->dtor)) {
                void *p;
 
                slab_pad_check(s, page);
@@ -934,7 +965,8 @@ static void discard_slab(struct kmem_cache *s, struct page *page)
 
        atomic_long_dec(&n->nr_slabs);
        reset_page_mapcount(page);
-       page->flags &= ~(1 << PG_slab | 1 << PG_error);
+       ClearSlabDebug(page);
+       __ClearPageSlab(page);
        free_slab(s, page);
 }
 
@@ -1109,7 +1141,7 @@ static void putback_slab(struct kmem_cache *s, struct page *page)
 
                if (page->freelist)
                        add_partial(n, page);
-               else if (PageError(page) && (s->flags & SLAB_STORE_USER))
+               else if (SlabDebug(page) && (s->flags & SLAB_STORE_USER))
                        add_full(n, page);
                slab_unlock(page);
 
@@ -1193,7 +1225,7 @@ static void flush_all(struct kmem_cache *s)
  * per cpu array in the kmem_cache struct.
  *
  * Fastpath is not possible if we need to get a new slab or have
- * debugging enabled (which means all slabs are marked with PageError)
+ * debugging enabled (which means all slabs are marked with SlabDebug)
  */
 static void *slab_alloc(struct kmem_cache *s,
                                gfp_t gfpflags, int node, void *addr)
@@ -1216,7 +1248,7 @@ redo:
        object = page->freelist;
        if (unlikely(!object))
                goto another_slab;
-       if (unlikely(PageError(page)))
+       if (unlikely(SlabDebug(page)))
                goto debug;
 
 have_object:
@@ -1273,12 +1305,7 @@ debug:
                goto another_slab;
        if (s->flags & SLAB_STORE_USER)
                set_track(s, object, TRACK_ALLOC, addr);
-       if (s->flags & SLAB_TRACE) {
-               printk(KERN_INFO "TRACE %s alloc 0x%p inuse=%d fp=0x%p\n",
-                       s->name, object, page->inuse,
-                       page->freelist);
-               dump_stack();
-       }
+       trace(s, page, object, 1);
        init_object(s, object, 1);
        goto have_object;
 }
@@ -1314,7 +1341,7 @@ static void slab_free(struct kmem_cache *s, struct page *page,
        local_irq_save(flags);
        slab_lock(page);
 
-       if (unlikely(PageError(page)))
+       if (unlikely(SlabDebug(page)))
                goto debug;
 checks_ok:
        prior = object[page->offset] = page->freelist;
@@ -1363,13 +1390,7 @@ debug:
                remove_full(s, page);
        if (s->flags & SLAB_STORE_USER)
                set_track(s, x, TRACK_FREE, addr);
-       if (s->flags & SLAB_TRACE) {
-               printk(KERN_INFO "TRACE %s free 0x%p inuse=%d fp=0x%p\n",
-                       s->name, object, page->inuse,
-                       page->freelist);
-               print_section("Object", (void *)object, s->objsize);
-               dump_stack();
-       }
+       trace(s, page, object, 0);
        init_object(s, object, 0);
        goto checks_ok;
 }
@@ -2571,12 +2592,12 @@ static void validate_slab_slab(struct kmem_cache *s, struct page *page)
                        s->name, page);
 
        if (s->flags & DEBUG_DEFAULT_FLAGS) {
-               if (!PageError(page))
-                       printk(KERN_ERR "SLUB %s: PageError not set "
+               if (!SlabDebug(page))
+                       printk(KERN_ERR "SLUB %s: SlabDebug not set "
                                "on slab 0x%p\n", s->name, page);
        } else {
-               if (PageError(page))
-                       printk(KERN_ERR "SLUB %s: PageError set on "
+               if (SlabDebug(page))
+                       printk(KERN_ERR "SLUB %s: SlabDebug set on "
                                "slab 0x%p\n", s->name, page);
        }
 }