*
* SMP synchronization:
* constructors and destructors are called without any locking.
- * Several members in kmem_cache_t and slab_t never change, they
+ * Several members in kmem_cache_t and struct slab never change, they
* are accessed without any locking.
* The per-cpu arrays are never accessed from the wrong cpu, no locking,
* and local interrupts are disabled so slab code is preempt-safe.
static unsigned long offslab_limit;
/*
- * slab_t
+ * struct slab
*
* Manages the objs in a slab. Placed either at the beginning of mem allocated
* for a slab, or allocated from an general cache.
* Slabs are chained into three list: fully used, partial, fully free slabs.
*/
-typedef struct slab_s {
+struct slab {
struct list_head list;
unsigned long colouroff;
void *s_mem; /* including colour offset */
unsigned int inuse; /* num of objs active in slab */
kmem_bufctl_t free;
-} slab_t;
-
-#define slab_bufctl(slabp) \
- ((kmem_bufctl_t *)(((slab_t*)slabp)+1))
+};
/*
- * cpucache_t
+ * struct array_cache
*
* Per cpu structures
* Purpose:
* - LIFO ordering, to hand out cache-warm objects from _alloc
+ * - reduce the number of linked list operations
* - reduce spinlock operations
*
* The limit is stored in the per-cpu structure to reduce the data cache
* footprint.
- * On NUMA systems, 2 per-cpu structures exist: one for the current
- * node, one for wrong node free calls.
- * Memory from the wrong node is never returned by alloc, it's returned
- * to the home node as soon as the cpu cache is filled
*
*/
-typedef struct cpucache_s {
+struct array_cache {
unsigned int avail;
unsigned int limit;
unsigned int batchcount;
unsigned int touched;
-} cpucache_t;
+};
/* bootstrap: The caches do not work without cpuarrays anymore,
* but the cpuarrays are allocated from the generic caches...
*/
#define BOOT_CPUCACHE_ENTRIES 1
-struct cpucache_int {
- cpucache_t cache;
+struct arraycache_init {
+ struct array_cache cache;
void * entries[BOOT_CPUCACHE_ENTRIES];
};
-#define cc_entry(cpucache) \
- ((void **)(((cpucache_t*)(cpucache))+1))
-#define cc_data(cachep) \
- ((cachep)->cpudata[smp_processor_id()])
-/*
- * NUMA: check if 'ptr' points into the current node,
- * use the alternate cpudata cache if wrong
- */
-#define cc_data_ptr(cachep, ptr) \
- cc_data(cachep)
-
/*
* The slab lists of all objects.
* Hopefully reduce the internal fragmentation
struct kmem_cache_s {
/* 1) per-cpu data, touched during every alloc/free */
- cpucache_t *cpudata[NR_CPUS];
- /* NUMA: cpucache_t *cpudata_othernode[NR_CPUS]; */
+ struct array_cache *array[NR_CPUS];
unsigned int batchcount;
unsigned int limit;
/* 2) touched by every alloc & free from the backend */
#define SET_PAGE_CACHE(pg,x) ((pg)->list.next = (struct list_head *)(x))
#define GET_PAGE_CACHE(pg) ((kmem_cache_t *)(pg)->list.next)
#define SET_PAGE_SLAB(pg,x) ((pg)->list.prev = (struct list_head *)(x))
-#define GET_PAGE_SLAB(pg) ((slab_t *)(pg)->list.prev)
+#define GET_PAGE_SLAB(pg) ((struct slab *)(pg)->list.prev)
/* Size description struct for general caches. */
-typedef struct cache_sizes {
+struct cache_sizes {
size_t cs_size;
kmem_cache_t *cs_cachep;
kmem_cache_t *cs_dmacachep;
-} cache_sizes_t;
+};
/* These are the default caches for kmalloc. Custom caches can have other sizes. */
-static cache_sizes_t cache_sizes[] = {
+static struct cache_sizes malloc_sizes[] = {
#if PAGE_SIZE == 4096
{ 32, NULL, NULL},
#endif
{ 64, NULL, NULL},
+#if L1_CACHE_BYTES < 64
{ 96, NULL, NULL},
+#endif
{ 128, NULL, NULL},
+#if L1_CACHE_BYTES < 128
{ 192, NULL, NULL},
+#endif
{ 256, NULL, NULL},
{ 512, NULL, NULL},
{ 1024, NULL, NULL},
CN("size-32"),
#endif
CN("size-64"),
+#if L1_CACHE_BYTES < 64
CN("size-96"),
+#endif
CN("size-128"),
+#if L1_CACHE_BYTES < 128
CN("size-192"),
+#endif
CN("size-256"),
CN("size-512"),
CN("size-1024"),
};
#undef CN
-struct cpucache_int cpuarray_cache __initdata = { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
-struct cpucache_int cpuarray_generic __initdata = { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
+struct arraycache_init initarray_cache __initdata = { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
+struct arraycache_init initarray_generic __initdata = { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
/* internal cache of cache description objs */
static kmem_cache_t cache_cache = {
.lists = LIST3_INIT(cache_cache.lists),
- .cpudata = { [0] = &cpuarray_cache.cache },
+ .array = { [0] = &initarray_cache.cache },
.batchcount = 1,
.limit = BOOT_CPUCACHE_ENTRIES,
.objsize = sizeof(kmem_cache_t),
size_t base = 0;
if (!(flags & CFLGS_OFF_SLAB)) {
- base = sizeof(slab_t);
+ base = sizeof(struct slab);
extra = sizeof(kmem_bufctl_t);
}
i = 0;
int cpu = (int)hcpu;
if (action == CPU_ONLINE) {
struct list_head *p;
- cpucache_t *nc;
+ struct array_cache *nc;
down(&cache_chain_sem);
int memsize;
kmem_cache_t* cachep = list_entry(p, kmem_cache_t, next);
- memsize = sizeof(void*)*cachep->limit+sizeof(cpucache_t);
+ memsize = sizeof(void*)*cachep->limit+sizeof(struct array_cache);
nc = kmalloc(memsize, GFP_KERNEL);
if (!nc)
goto bad;
nc->touched = 0;
spin_lock_irq(&cachep->spinlock);
- cachep->cpudata[cpu] = nc;
+ cachep->array[cpu] = nc;
cachep->free_limit = (1+num_online_cpus())*cachep->batchcount
+ cachep->num;
spin_unlock_irq(&cachep->spinlock);
static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 };
+static inline void ** ac_entry(struct array_cache *ac)
+{
+ return (void**)(ac+1);
+}
+
+static inline struct array_cache *ac_data(kmem_cache_t *cachep)
+{
+ return cachep->array[smp_processor_id()];
+}
+
/* Initialisation - setup the `cache' cache. */
void __init kmem_cache_init(void)
{
cache_cache.colour_next = 0;
/* Register a cpu startup notifier callback
- * that initializes cc_data for all new cpus
+ * that initializes ac_data for all new cpus
*/
register_cpu_notifier(&cpucache_notifier);
}
*/
void __init kmem_cache_sizes_init(void)
{
- cache_sizes_t *sizes = cache_sizes;
+ struct cache_sizes *sizes = malloc_sizes;
/*
* Fragmentation resistance on low memory - only use bigger
* page orders on machines with more than 32MB of memory.
* Note for systems short on memory removing the alignment will
* allow tighter packing of the smaller caches. */
if (!(sizes->cs_cachep =
- kmem_cache_create(cache_names[sizes-cache_sizes].name,
+ kmem_cache_create(cache_names[sizes-malloc_sizes].name,
sizes->cs_size,
0, SLAB_HWCACHE_ALIGN, NULL, NULL))) {
BUG();
/* Inc off-slab bufctl limit until the ceiling is hit. */
if (!(OFF_SLAB(sizes->cs_cachep))) {
- offslab_limit = sizes->cs_size-sizeof(slab_t);
+ offslab_limit = sizes->cs_size-sizeof(struct slab);
offslab_limit /= sizeof(kmem_bufctl_t);
}
sizes->cs_dmacachep = kmem_cache_create(
- cache_names[sizes-cache_sizes].name_dma,
+ cache_names[sizes-malloc_sizes].name_dma,
sizes->cs_size, 0,
SLAB_CACHE_DMA|SLAB_HWCACHE_ALIGN, NULL, NULL);
if (!sizes->cs_dmacachep)
{
void * ptr;
- ptr = kmalloc(sizeof(struct cpucache_int), GFP_KERNEL);
+ ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
local_irq_disable();
- BUG_ON(cc_data(&cache_cache) != &cpuarray_cache.cache);
- memcpy(ptr, cc_data(&cache_cache), sizeof(struct cpucache_int));
- cc_data(&cache_cache) = ptr;
+ BUG_ON(ac_data(&cache_cache) != &initarray_cache.cache);
+ memcpy(ptr, ac_data(&cache_cache), sizeof(struct arraycache_init));
+ cache_cache.array[smp_processor_id()] = ptr;
local_irq_enable();
- ptr = kmalloc(sizeof(struct cpucache_int), GFP_KERNEL);
+ ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
local_irq_disable();
- BUG_ON(cc_data(cache_sizes[0].cs_cachep) != &cpuarray_generic.cache);
- memcpy(ptr, cc_data(cache_sizes[0].cs_cachep),
- sizeof(struct cpucache_int));
- cc_data(cache_sizes[0].cs_cachep) = ptr;
+ BUG_ON(ac_data(malloc_sizes[0].cs_cachep) != &initarray_generic.cache);
+ memcpy(ptr, ac_data(malloc_sizes[0].cs_cachep),
+ sizeof(struct arraycache_init));
+ malloc_sizes[0].cs_cachep->array[smp_processor_id()] = ptr;
local_irq_enable();
}
}
* Before calling the slab must have been unlinked from the cache.
* The cache-lock is not held/needed.
*/
-static void slab_destroy (kmem_cache_t *cachep, slab_t *slabp)
+static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp)
{
#if DEBUG
int i;
cachep = NULL;
goto opps;
}
- slab_size = L1_CACHE_ALIGN(cachep->num*sizeof(kmem_bufctl_t)+sizeof(slab_t));
+ slab_size = L1_CACHE_ALIGN(cachep->num*sizeof(kmem_bufctl_t)+sizeof(struct slab));
/*
* If the slab has been placed off-slab, and we have enough space then
* the cache that's used by kmalloc(24), otherwise
* the creation of further caches will BUG().
*/
- cc_data(cachep) = &cpuarray_generic.cache;
+ cachep->array[smp_processor_id()] = &initarray_generic.cache;
g_cpucache_up = PARTIAL;
} else {
- cc_data(cachep) = kmalloc(sizeof(struct cpucache_int),GFP_KERNEL);
+ cachep->array[smp_processor_id()] = kmalloc(sizeof(struct arraycache_init),GFP_KERNEL);
}
- BUG_ON(!cc_data(cachep));
- cc_data(cachep)->avail = 0;
- cc_data(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
- cc_data(cachep)->batchcount = 1;
- cc_data(cachep)->touched = 0;
+ BUG_ON(!ac_data(cachep));
+ ac_data(cachep)->avail = 0;
+ ac_data(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
+ ac_data(cachep)->batchcount = 1;
+ ac_data(cachep)->touched = 0;
cachep->batchcount = 1;
cachep->limit = BOOT_CPUCACHE_ENTRIES;
cachep->free_limit = (1+num_online_cpus())*cachep->batchcount
static void do_drain(void *arg)
{
kmem_cache_t *cachep = (kmem_cache_t*)arg;
- cpucache_t *cc;
+ struct array_cache *ac;
check_irq_off();
- cc = cc_data(cachep);
- free_block(cachep, &cc_entry(cc)[0], cc->avail);
- cc->avail = 0;
+ ac = ac_data(cachep);
+ free_block(cachep, &ac_entry(ac)[0], ac->avail);
+ ac->avail = 0;
}
static void drain_cpu_caches(kmem_cache_t *cachep)
/* NUMA shrink all list3s */
static int __cache_shrink(kmem_cache_t *cachep)
{
- slab_t *slabp;
+ struct slab *slabp;
int ret;
drain_cpu_caches(cachep);
if (p == &cachep->lists.slabs_free)
break;
- slabp = list_entry(cachep->lists.slabs_free.prev, slab_t, list);
+ slabp = list_entry(cachep->lists.slabs_free.prev, struct slab, list);
#if DEBUG
if (slabp->inuse)
BUG();
{
int i;
for (i = 0; i < NR_CPUS; i++)
- kfree(cachep->cpudata[i]);
+ kfree(cachep->array[i]);
/* NUMA: free the list3 structures */
}
kmem_cache_free(&cache_cache, cachep);
}
/* Get the memory for a slab management obj. */
-static inline slab_t * alloc_slabmgmt (kmem_cache_t *cachep,
+static inline struct slab* alloc_slabmgmt (kmem_cache_t *cachep,
void *objp, int colour_off, int local_flags)
{
- slab_t *slabp;
+ struct slab *slabp;
if (OFF_SLAB(cachep)) {
/* Slab management obj is off-slab. */
} else {
slabp = objp+colour_off;
colour_off += L1_CACHE_ALIGN(cachep->num *
- sizeof(kmem_bufctl_t) + sizeof(slab_t));
+ sizeof(kmem_bufctl_t) + sizeof(struct slab));
}
slabp->inuse = 0;
slabp->colouroff = colour_off;
return slabp;
}
-static inline void cache_init_objs (kmem_cache_t * cachep,
- slab_t * slabp, unsigned long ctor_flags)
+static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp)
+{
+ return (kmem_bufctl_t *)(slabp+1);
+}
+
+static void cache_init_objs (kmem_cache_t * cachep,
+ struct slab * slabp, unsigned long ctor_flags)
{
int i;
*/
static int cache_grow (kmem_cache_t * cachep, int flags)
{
- slab_t *slabp;
+ struct slab *slabp;
struct page *page;
void *objp;
size_t offset;
#if DEBUG
struct page *page;
unsigned int objnr;
- slab_t *slabp;
+ struct slab *slabp;
kfree_debugcheck(objp);
page = virt_to_page(objp);
return objp;
}
-static inline void check_slabp(kmem_cache_t *cachep, slab_t *slabp)
+static inline void check_slabp(kmem_cache_t *cachep, struct slab *slabp)
{
#if DEBUG
int i;
}
static inline void * cache_alloc_one_tail (kmem_cache_t *cachep,
- slab_t *slabp)
+ struct slab *slabp)
{
void *objp;
return objp;
}
-static inline void cache_alloc_listfixup(struct kmem_list3 *l3, slab_t *slabp)
+static inline void cache_alloc_listfixup(struct kmem_list3 *l3, struct slab *slabp)
{
list_del(&slabp->list);
if (slabp->free == BUFCTL_END) {
{
int batchcount;
struct kmem_list3 *l3;
- cpucache_t *cc;
+ struct array_cache *ac;
check_irq_off();
- cc = cc_data(cachep);
+ ac = ac_data(cachep);
retry:
- batchcount = cc->batchcount;
- if (!cc->touched && batchcount > BATCHREFILL_LIMIT) {
+ batchcount = ac->batchcount;
+ if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
/* if there was little recent activity on this
* cache, then perform only a partial refill.
* Otherwise we could generate refill bouncing.
}
l3 = list3_data(cachep);
- BUG_ON(cc->avail > 0);
+ BUG_ON(ac->avail > 0);
spin_lock(&cachep->spinlock);
while (batchcount > 0) {
struct list_head *entry;
- slab_t *slabp;
+ struct slab *slabp;
/* Get slab alloc is to come from. */
entry = l3->slabs_partial.next;
if (entry == &l3->slabs_partial) {
goto must_grow;
}
- slabp = list_entry(entry, slab_t, list);
+ slabp = list_entry(entry, struct slab, list);
check_slabp(cachep, slabp);
while (slabp->inuse < cachep->num && batchcount--)
- cc_entry(cc)[cc->avail++] =
+ ac_entry(ac)[ac->avail++] =
cache_alloc_one_tail(cachep, slabp);
check_slabp(cachep, slabp);
cache_alloc_listfixup(l3, slabp);
}
must_grow:
- l3->free_objects -= cc->avail;
+ l3->free_objects -= ac->avail;
spin_unlock(&cachep->spinlock);
- if (unlikely(!cc->avail)) {
+ if (unlikely(!ac->avail)) {
int x;
x = cache_grow(cachep, flags);
- // cache_grow can reenable interrupts, then cc could change.
- cc = cc_data(cachep);
- if (!x && cc->avail == 0) // no objects in sight? abort
+ // cache_grow can reenable interrupts, then ac could change.
+ ac = ac_data(cachep);
+ if (!x && ac->avail == 0) // no objects in sight? abort
return NULL;
- if (!cc->avail) // objects refilled by interrupt?
+ if (!ac->avail) // objects refilled by interrupt?
goto retry;
}
- cc->touched = 1;
- return cc_entry(cc)[--cc->avail];
+ ac->touched = 1;
+ return ac_entry(ac)[--ac->avail];
}
static inline void cache_alloc_debugcheck_before(kmem_cache_t *cachep, int flags)
{
unsigned long save_flags;
void* objp;
- cpucache_t *cc;
+ struct array_cache *ac;
cache_alloc_debugcheck_before(cachep, flags);
local_irq_save(save_flags);
- cc = cc_data(cachep);
- if (likely(cc->avail)) {
+ ac = ac_data(cachep);
+ if (likely(ac->avail)) {
STATS_INC_ALLOCHIT(cachep);
- cc->touched = 1;
- objp = cc_entry(cc)[--cc->avail];
+ ac->touched = 1;
+ objp = ac_entry(ac)[--ac->avail];
} else {
STATS_INC_ALLOCMISS(cachep);
objp = cache_alloc_refill(cachep, flags);
* the l3 structure
*/
-static inline void __free_block (kmem_cache_t* cachep, void** objpp, int len)
+static inline void
+__free_block(kmem_cache_t *cachep, void **objpp, int nr_objects)
{
+ int i;
+
check_irq_off();
spin_lock(&cachep->spinlock);
+
/* NUMA: move add into loop */
- cachep->lists.free_objects += len;
+ cachep->lists.free_objects += nr_objects;
- for ( ; len > 0; len--, objpp++) {
- slab_t* slabp;
- void *objp = *objpp;
+ for (i = 0; i < nr_objects; i++) {
+ void *objp = objpp[i];
+ struct slab *slabp;
+ unsigned int objnr;
slabp = GET_PAGE_SLAB(virt_to_page(objp));
list_del(&slabp->list);
- {
- unsigned int objnr = (objp-slabp->s_mem)/cachep->objsize;
-
- slab_bufctl(slabp)[objnr] = slabp->free;
- slabp->free = objnr;
- }
+ objnr = (objp - slabp->s_mem) / cachep->objsize;
+ slab_bufctl(slabp)[objnr] = slabp->free;
+ slabp->free = objnr;
STATS_DEC_ACTIVE(cachep);
-
+ slabp->inuse--;
+
/* fixup slab chains */
- if (unlikely(!--slabp->inuse)) {
+ if (slabp->inuse == 0) {
if (cachep->lists.free_objects > cachep->free_limit) {
cachep->lists.free_objects -= cachep->num;
slab_destroy(cachep, slabp);
} else {
list_add(&slabp->list,
- &list3_data_ptr(cachep, objp)->slabs_free);
+ &list3_data_ptr(cachep, objp)->slabs_free);
}
} else {
/* Unconditionally move a slab to the end of the
* partial list on free - maximum time for the
* other objects to be freed, too.
*/
- list_add_tail(&slabp->list, &list3_data_ptr(cachep, objp)->slabs_partial);
+ list_add_tail(&slabp->list,
+ &list3_data_ptr(cachep, objp)->slabs_partial);
}
}
spin_unlock(&cachep->spinlock);
__free_block(cachep, objpp, len);
}
-static void cache_flusharray (kmem_cache_t* cachep, cpucache_t *cc)
+static void cache_flusharray (kmem_cache_t* cachep, struct array_cache *ac)
{
int batchcount;
- batchcount = cc->batchcount;
+ batchcount = ac->batchcount;
#if DEBUG
- BUG_ON(!batchcount || batchcount > cc->avail);
+ BUG_ON(!batchcount || batchcount > ac->avail);
#endif
check_irq_off();
- __free_block(cachep, &cc_entry(cc)[0], batchcount);
+ __free_block(cachep, &ac_entry(ac)[0], batchcount);
#if STATS
{
spin_lock(&cachep->spinlock);
p = list3_data(cachep)->slabs_free.next;
while (p != &(list3_data(cachep)->slabs_free)) {
- slab_t *slabp;
+ struct slab *slabp;
- slabp = list_entry(p, slab_t, list);
+ slabp = list_entry(p, struct slab, list);
BUG_ON(slabp->inuse);
i++;
spin_unlock(&cachep->spinlock);
}
#endif
- cc->avail -= batchcount;
- memmove(&cc_entry(cc)[0], &cc_entry(cc)[batchcount],
- sizeof(void*)*cc->avail);
+ ac->avail -= batchcount;
+ memmove(&ac_entry(ac)[0], &ac_entry(ac)[batchcount],
+ sizeof(void*)*ac->avail);
}
/*
*/
static inline void __cache_free (kmem_cache_t *cachep, void* objp)
{
- cpucache_t *cc = cc_data_ptr(cachep, objp);
+ struct array_cache *ac = ac_data(cachep);
check_irq_off();
objp = cache_free_debugcheck(cachep, objp);
- if (likely(cc->avail < cc->limit)) {
+ if (likely(ac->avail < ac->limit)) {
STATS_INC_FREEHIT(cachep);
- cc_entry(cc)[cc->avail++] = objp;
+ ac_entry(ac)[ac->avail++] = objp;
return;
} else {
STATS_INC_FREEMISS(cachep);
- cache_flusharray(cachep, cc);
- cc_entry(cc)[cc->avail++] = objp;
+ cache_flusharray(cachep, ac);
+ ac_entry(ac)[ac->avail++] = objp;
}
}
*/
void * kmalloc (size_t size, int flags)
{
- cache_sizes_t *csizep = cache_sizes;
+ struct cache_sizes *csizep = malloc_sizes;
for (; csizep->cs_size; csizep++) {
if (size > csizep->cs_size)
kmem_cache_t * kmem_find_general_cachep (size_t size, int gfpflags)
{
- cache_sizes_t *csizep = cache_sizes;
+ struct cache_sizes *csizep = malloc_sizes;
/* This function could be moved to the header file, and
* made inline so consumers can quickly determine what
struct ccupdate_struct {
kmem_cache_t *cachep;
- cpucache_t *new[NR_CPUS];
+ struct array_cache *new[NR_CPUS];
};
static void do_ccupdate_local(void *info)
{
struct ccupdate_struct *new = (struct ccupdate_struct *)info;
- cpucache_t *old;
+ struct array_cache *old;
check_irq_off();
- old = cc_data(new->cachep);
+ old = ac_data(new->cachep);
- cc_data(new->cachep) = new->new[smp_processor_id()];
+ new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()];
new->new[smp_processor_id()] = old;
}
memset(&new.new,0,sizeof(new.new));
for (i = 0; i < NR_CPUS; i++) {
- cpucache_t* ccnew;
+ struct array_cache *ccnew;
ccnew = kmalloc(sizeof(void*)*limit+
- sizeof(cpucache_t), GFP_KERNEL);
+ sizeof(struct array_cache), GFP_KERNEL);
if (!ccnew) {
for (i--; i >= 0; i--) kfree(new.new[i]);
return -ENOMEM;
spin_unlock_irq(&cachep->spinlock);
for (i = 0; i < NR_CPUS; i++) {
- cpucache_t* ccold = new.new[i];
+ struct array_cache *ccold = new.new[i];
if (!ccold)
continue;
local_irq_disable();
- free_block(cachep, cc_entry(ccold), ccold->avail);
+ free_block(cachep, ac_entry(ccold), ccold->avail);
local_irq_enable();
kfree(ccold);
}
kmem_cache_t *searchp;
struct list_head* p;
int tofree;
- cpucache_t *cc;
- slab_t *slabp;
+ struct array_cache *ac;
+ struct slab *slabp;
searchp = list_entry(walk, kmem_cache_t, next);
check_irq_on();
local_irq_disable();
- cc = cc_data(searchp);
- if (cc->touched) {
- cc->touched = 0;
- } else if (cc->avail) {
- tofree = (cc->limit+4)/5;
- if (tofree > cc->avail) {
- tofree = (cc->avail+1)/2;
+ ac = ac_data(searchp);
+ if (ac->touched) {
+ ac->touched = 0;
+ } else if (ac->avail) {
+ tofree = (ac->limit+4)/5;
+ if (tofree > ac->avail) {
+ tofree = (ac->avail+1)/2;
}
- free_block(searchp, cc_entry(cc), tofree);
- cc->avail -= tofree;
- memmove(&cc_entry(cc)[0], &cc_entry(cc)[tofree],
- sizeof(void*)*cc->avail);
+ free_block(searchp, ac_entry(ac), tofree);
+ ac->avail -= tofree;
+ memmove(&ac_entry(ac)[0], &ac_entry(ac)[tofree],
+ sizeof(void*)*ac->avail);
}
if(time_after(searchp->lists.next_reap, jiffies))
goto next_irqon;
if (p == &(list3_data(searchp)->slabs_free))
break;
- slabp = list_entry(p, slab_t, list);
+ slabp = list_entry(p, struct slab, list);
BUG_ON(slabp->inuse);
list_del(&slabp->list);
STATS_INC_REAPED(searchp);
{
kmem_cache_t *cachep = p;
struct list_head *q;
- slab_t *slabp;
+ struct slab *slabp;
unsigned long active_objs;
unsigned long num_objs;
unsigned long active_slabs = 0;
active_objs = 0;
num_slabs = 0;
list_for_each(q,&cachep->lists.slabs_full) {
- slabp = list_entry(q, slab_t, list);
+ slabp = list_entry(q, struct slab, list);
if (slabp->inuse != cachep->num)
BUG();
active_objs += cachep->num;
active_slabs++;
}
list_for_each(q,&cachep->lists.slabs_partial) {
- slabp = list_entry(q, slab_t, list);
+ slabp = list_entry(q, struct slab, list);
BUG_ON(slabp->inuse == cachep->num || !slabp->inuse);
active_objs += slabp->inuse;
active_slabs++;
}
list_for_each(q,&cachep->lists.slabs_free) {
- slabp = list_entry(q, slab_t, list);
+ slabp = list_entry(q, struct slab, list);
if (slabp->inuse)
BUG();
num_slabs++;
projects / history.git / commitdiff