unsigned int buffer_size;
u32 reciprocal_buffer_size;
/* 3) touched by every alloc & free from the backend */
- struct kmem_list3 *nodelists[MAX_NUMNODES];
unsigned int flags; /* constant flags */
unsigned int num; /* # of objs per slab */
int obj_offset;
int obj_size;
#endif
+ /*
+ * We put nodelists[] at the end of kmem_cache, because we want to size
+ * this array to nr_node_ids slots instead of MAX_NUMNODES
+ * (see kmem_cache_init())
+ * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
+ * is statically defined, so we reserve the max number of nodes.
+ */
+ struct kmem_list3 *nodelists[MAX_NUMNODES];
+ /*
+ * Do not add fields after nodelists[]
+ */
};
#define CFLGS_OFF_SLAB (0x80000000UL)
.shared = 1,
.buffer_size = sizeof(struct kmem_cache),
.name = "kmem_cache",
-#if DEBUG
- .obj_size = sizeof(struct kmem_cache),
-#endif
};
#define BAD_ALIEN_MAGIC 0x01020304ul
static struct array_cache **alloc_alien_cache(int node, int limit)
{
struct array_cache **ac_ptr;
- int memsize = sizeof(void *) * MAX_NUMNODES;
+ int memsize = sizeof(void *) * nr_node_ids;
int i;
if (limit > 1)
* Make sure we are not freeing a object from another node to the array
* cache on this cpu.
*/
- if (likely(slabp->nodeid == node) || unlikely(!use_alien_caches))
+ if (likely(slabp->nodeid == node))
return 0;
l3 = cachep->nodelists[node];
*/
list_for_each_entry(cachep, &cache_chain, next) {
struct array_cache *nc;
- struct array_cache *shared;
+ struct array_cache *shared = NULL;
struct array_cache **alien = NULL;
nc = alloc_arraycache(node, cachep->limit,
cachep->batchcount);
if (!nc)
goto bad;
- shared = alloc_arraycache(node,
+ if (cachep->shared) {
+ shared = alloc_arraycache(node,
cachep->shared * cachep->batchcount,
0xbaadf00d);
- if (!shared)
- goto bad;
-
+ if (!shared)
+ goto bad;
+ }
if (use_alien_caches) {
alien = alloc_alien_cache(node, cachep->limit);
if (!alien)
shared = l3->shared;
if (shared) {
- free_block(cachep, l3->shared->entry,
- l3->shared->avail, node);
+ free_block(cachep, shared->entry,
+ shared->avail, node);
l3->shared = NULL;
}
int order;
int node;
+ if (num_possible_nodes() == 1)
+ use_alien_caches = 0;
+
for (i = 0; i < NUM_INIT_LISTS; i++) {
kmem_list3_init(&initkmem_list3[i]);
if (i < MAX_NUMNODES)
cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE];
+ /*
+ * struct kmem_cache size depends on nr_node_ids, which
+ * can be less than MAX_NUMNODES.
+ */
+ cache_cache.buffer_size = offsetof(struct kmem_cache, nodelists) +
+ nr_node_ids * sizeof(struct kmem_list3 *);
+#if DEBUG
+ cache_cache.obj_size = cache_cache.buffer_size;
+#endif
cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
cache_line_size());
cache_cache.reciprocal_buffer_size =
/* Print header */
if (lines == 0) {
printk(KERN_ERR
- "Slab corruption: start=%p, len=%d\n",
- realobj, size);
+ "Slab corruption: %s start=%p, len=%d\n",
+ cachep->name, realobj, size);
print_objinfo(cachep, objp, 0);
}
/* Hexdump the affected line */
slabp = list_entry(entry, struct slab, list);
check_slabp(cachep, slabp);
check_spinlock_acquired(cachep);
+
+ /*
+ * The slab was either on partial or free list so
+ * there must be at least one object available for
+ * allocation.
+ */
+ BUG_ON(slabp->inuse < 0 || slabp->inuse >= cachep->num);
+
while (slabp->inuse < cachep->num && batchcount--) {
STATS_INC_ALLOCED(cachep);
STATS_INC_ACTIVE(cachep);
check_irq_off();
objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0));
- if (cache_free_alien(cachep, objp))
+ if (use_alien_caches && cache_free_alien(cachep, objp))
return;
if (likely(ac->avail < ac->limit)) {
EXPORT_SYMBOL(__kmalloc);
#endif
+/**
+ * krealloc - reallocate memory. The contents will remain unchanged.
+ *
+ * @p: object to reallocate memory for.
+ * @new_size: how many bytes of memory are required.
+ * @flags: the type of memory to allocate.
+ *
+ * The contents of the object pointed to are preserved up to the
+ * lesser of the new and old sizes. If @p is %NULL, krealloc()
+ * behaves exactly like kmalloc(). If @size is 0 and @p is not a
+ * %NULL pointer, the object pointed to is freed.
+ */
+void *krealloc(const void *p, size_t new_size, gfp_t flags)
+{
+ struct kmem_cache *cache, *new_cache;
+ void *ret;
+
+ if (unlikely(!p))
+ return kmalloc_track_caller(new_size, flags);
+
+ if (unlikely(!new_size)) {
+ kfree(p);
+ return NULL;
+ }
+
+ cache = virt_to_cache(p);
+ new_cache = __find_general_cachep(new_size, flags);
+
+ /*
+ * If new size fits in the current cache, bail out.
+ */
+ if (likely(cache == new_cache))
+ return (void *)p;
+
+ /*
+ * We are on the slow-path here so do not use __cache_alloc
+ * because it bloats kernel text.
+ */
+ ret = kmalloc_track_caller(new_size, flags);
+ if (ret) {
+ memcpy(ret, p, min(new_size, ksize(p)));
+ kfree(p);
+ }
+ return ret;
+}
+EXPORT_SYMBOL(krealloc);
+
/**
* kmem_cache_free - Deallocate an object
* @cachep: The cache the allocation was from.
goto fail;
}
- new_shared = alloc_arraycache(node,
+ new_shared = NULL;
+ if (cachep->shared) {
+ new_shared = alloc_arraycache(node,
cachep->shared*cachep->batchcount,
0xbaadf00d);
- if (!new_shared) {
- free_alien_cache(new_alien);
- goto fail;
+ if (!new_shared) {
+ free_alien_cache(new_alien);
+ goto fail;
+ }
}
l3 = cachep->nodelists[node];
* to a larger limit. Thus disabled by default.
*/
shared = 0;
-#ifdef CONFIG_SMP
- if (cachep->buffer_size <= PAGE_SIZE)
+ if (cachep->buffer_size <= PAGE_SIZE && num_possible_cpus() > 1)
shared = 8;
-#endif
#if DEBUG
/*
/**
* cache_reap - Reclaim memory from caches.
- * @unused: unused parameter
+ * @w: work descriptor
*
* Called from workqueue/eventd every few seconds.
* Purpose:
* allocated with either kmalloc() or kmem_cache_alloc(). The object
* must not be freed during the duration of the call.
*/
-unsigned int ksize(const void *objp)
+size_t ksize(const void *objp)
{
if (unlikely(objp == NULL))
return 0;
projects / powerpc.git / blobdiff