470 lines
11 KiB
C
470 lines
11 KiB
C
|
|
#include <types.h>
|
|
#include <core.h>
|
|
#include <spinlock.h>
|
|
#include <link.h>
|
|
#include <mm.h>
|
|
#include <slab.h>
|
|
|
|
|
|
extern zone_t z_core;
|
|
|
|
|
|
static LIST_INITIALIZE(slab_cache_list);
|
|
|
|
static slab_cache_t *slab_cache;
|
|
|
|
static slab_cache_t slab_cache_cache;
|
|
|
|
static slab_t *slab_create();
|
|
|
|
static slab_cache_t * slab_cache_alloc();
|
|
|
|
|
|
/**
|
|
* Allocate frames for slab space and initialize
|
|
*
|
|
*/
|
|
static slab_t * slab_space_alloc(slab_cache_t *cache, int flags)
|
|
{
|
|
void *data;
|
|
slab_t *slab;
|
|
size_t fsize;
|
|
unsigned int i;
|
|
u32_t p;
|
|
|
|
data = (void*)PA2KA(core_alloc(cache->order));
|
|
if (!data) {
|
|
return NULL;
|
|
}
|
|
slab = (slab_t*)slab_create();
|
|
if (!slab) {
|
|
core_free(KA2PA(data));
|
|
return NULL;
|
|
}
|
|
|
|
/* Fill in slab structures */
|
|
for (i = 0; i < ((u32_t) 1 << cache->order); i++)
|
|
frame_set_parent(ADDR2PFN(KA2PA(data)) + i, slab);
|
|
|
|
slab->start = data;
|
|
slab->available = cache->objects;
|
|
slab->nextavail = (void*)data;
|
|
slab->cache = cache;
|
|
|
|
for (i = 0, p = (u32_t)slab->start; i < cache->objects; i++)
|
|
{
|
|
*(addr_t *)p = p+cache->size;
|
|
p = p+cache->size;
|
|
};
|
|
atomic_inc(&cache->allocated_slabs);
|
|
return slab;
|
|
}
|
|
|
|
/**
|
|
* Take new object from slab or create new if needed
|
|
*
|
|
* @return Object address or null
|
|
*/
|
|
static void * slab_obj_create(slab_cache_t *cache, int flags)
|
|
{
|
|
slab_t *slab;
|
|
void *obj;
|
|
|
|
spinlock_lock(&cache->slablock);
|
|
|
|
if (list_empty(&cache->partial_slabs)) {
|
|
/* Allow recursion and reclaiming
|
|
* - this should work, as the slab control structures
|
|
* are small and do not need to allocate with anything
|
|
* other than frame_alloc when they are allocating,
|
|
* that's why we should get recursion at most 1-level deep
|
|
*/
|
|
slab = slab_space_alloc(cache, flags);
|
|
if (!slab)
|
|
{
|
|
spinlock_unlock(&cache->slablock);
|
|
return NULL;
|
|
}
|
|
} else {
|
|
slab = list_get_instance(cache->partial_slabs.next, slab_t, link);
|
|
list_remove(&slab->link);
|
|
}
|
|
|
|
obj = slab->nextavail;
|
|
slab->nextavail = *(void**)obj;
|
|
slab->available--;
|
|
|
|
if (!slab->available)
|
|
list_prepend(&slab->link, &cache->full_slabs);
|
|
else
|
|
list_prepend(&slab->link, &cache->partial_slabs);
|
|
|
|
spinlock_unlock(&cache->slablock);
|
|
|
|
// if (cache->constructor && cache->constructor(obj, flags)) {
|
|
/* Bad, bad, construction failed */
|
|
// slab_obj_destroy(cache, obj, slab);
|
|
// return NULL;
|
|
// }
|
|
return obj;
|
|
}
|
|
|
|
|
|
/** Map object to slab structure */
|
|
static slab_t * obj2slab(void *obj)
|
|
{
|
|
return (slab_t *) frame_get_parent(ADDR2PFN(KA2PA(obj)));
|
|
}
|
|
|
|
|
|
/** Allocate new object from cache - if no flags given, always returns
|
|
memory */
|
|
void* __fastcall slab_alloc(slab_cache_t *cache, int flags)
|
|
{
|
|
eflags_t efl;
|
|
void *result = NULL;
|
|
|
|
/* Disable interrupts to avoid deadlocks with interrupt handlers */
|
|
efl = safe_cli();
|
|
|
|
// if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) {
|
|
// result = magazine_obj_get(cache);
|
|
// }
|
|
// if (!result)
|
|
result = slab_obj_create(cache, flags);
|
|
|
|
safe_sti(efl);
|
|
|
|
// if (result)
|
|
// atomic_inc(&cache->allocated_objs);
|
|
|
|
return result;
|
|
}
|
|
|
|
|
|
|
|
/**************************************/
|
|
/* Slab cache functions */
|
|
|
|
/** Return number of objects that fit in certain cache size */
|
|
static unsigned int comp_objects(slab_cache_t *cache)
|
|
{
|
|
if (cache->flags & SLAB_CACHE_SLINSIDE)
|
|
return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size;
|
|
else
|
|
return (PAGE_SIZE << cache->order) / cache->size;
|
|
}
|
|
|
|
/** Return wasted space in slab */
|
|
static unsigned int badness(slab_cache_t *cache)
|
|
{
|
|
unsigned int objects;
|
|
unsigned int ssize;
|
|
size_t val;
|
|
|
|
objects = comp_objects(cache);
|
|
ssize = PAGE_SIZE << cache->order;
|
|
if (cache->flags & SLAB_CACHE_SLINSIDE)
|
|
ssize -= sizeof(slab_t);
|
|
val = ssize - objects * cache->size;
|
|
return val;
|
|
|
|
}
|
|
|
|
|
|
/** Initialize allocated memory as a slab cache */
|
|
static void
|
|
_slab_cache_create(slab_cache_t *cache,
|
|
size_t size,
|
|
size_t align,
|
|
int (*constructor)(void *obj, int kmflag),
|
|
int (*destructor)(void *obj),
|
|
int flags)
|
|
{
|
|
int pages;
|
|
// ipl_t ipl;
|
|
|
|
// memsetb((uintptr_t)cache, sizeof(*cache), 0);
|
|
// cache->name = name;
|
|
|
|
//if (align < sizeof(unative_t))
|
|
// align = sizeof(unative_t);
|
|
// size = ALIGN_UP(size, align);
|
|
|
|
cache->size = size;
|
|
|
|
// cache->constructor = constructor;
|
|
// cache->destructor = destructor;
|
|
cache->flags = flags;
|
|
|
|
list_initialize(&cache->full_slabs);
|
|
list_initialize(&cache->partial_slabs);
|
|
list_initialize(&cache->magazines);
|
|
// spinlock_initialize(&cache->slablock, "slab_lock");
|
|
// spinlock_initialize(&cache->maglock, "slab_maglock");
|
|
// if (! (cache->flags & SLAB_CACHE_NOMAGAZINE))
|
|
// make_magcache(cache);
|
|
|
|
/* Compute slab sizes, object counts in slabs etc. */
|
|
|
|
/* Minimum slab order */
|
|
pages = SIZE2FRAMES(cache->size);
|
|
/* We need the 2^order >= pages */
|
|
if (pages == 1)
|
|
cache->order = 0;
|
|
else
|
|
cache->order = fnzb(pages-1)+1;
|
|
|
|
while (badness(cache) > SLAB_MAX_BADNESS(cache)) {
|
|
cache->order += 1;
|
|
}
|
|
cache->objects = comp_objects(cache);
|
|
|
|
/* Add cache to cache list */
|
|
// ipl = interrupts_disable();
|
|
// spinlock_lock(&slab_cache_lock);
|
|
|
|
list_append(&cache->link, &slab_cache_list);
|
|
|
|
// spinlock_unlock(&slab_cache_lock);
|
|
// interrupts_restore(ipl);
|
|
}
|
|
|
|
/** Create slab cache */
|
|
slab_cache_t * slab_cache_create(
|
|
size_t size,
|
|
size_t align,
|
|
int (*constructor)(void *obj, int kmflag),
|
|
int (*destructor)(void *obj),
|
|
int flags)
|
|
{
|
|
slab_cache_t *cache;
|
|
|
|
cache = (slab_cache_t*)slab_cache_alloc();
|
|
_slab_cache_create(cache, size, align, constructor, destructor, flags);
|
|
return cache;
|
|
}
|
|
|
|
/**
|
|
* Deallocate space associated with slab
|
|
*
|
|
* @return number of freed frames
|
|
*/
|
|
static count_t slab_space_free(slab_cache_t *cache, slab_t *slab)
|
|
{
|
|
frame_free(KA2PA(slab->start));
|
|
if (! (cache->flags & SLAB_CACHE_SLINSIDE))
|
|
slab_free(slab_cache, slab);
|
|
|
|
// atomic_dec(&cache->allocated_slabs);
|
|
|
|
return 1 << cache->order;
|
|
}
|
|
|
|
/**
|
|
* Return object to slab and call a destructor
|
|
*
|
|
* @param slab If the caller knows directly slab of the object, otherwise NULL
|
|
*
|
|
* @return Number of freed pages
|
|
*/
|
|
static count_t slab_obj_destroy(slab_cache_t *cache, void *obj,
|
|
slab_t *slab)
|
|
{
|
|
int freed = 0;
|
|
|
|
if (!slab)
|
|
slab = obj2slab(obj);
|
|
|
|
// ASSERT(slab->cache == cache);
|
|
|
|
// if (cache->destructor)
|
|
// freed = cache->destructor(obj);
|
|
|
|
// spinlock_lock(&cache->slablock);
|
|
// ASSERT(slab->available < cache->objects);
|
|
|
|
*(void**)obj = slab->nextavail;
|
|
slab->nextavail = obj;
|
|
slab->available++;
|
|
|
|
/* Move it to correct list */
|
|
if (slab->available == cache->objects) {
|
|
/* Free associated memory */
|
|
list_remove(&slab->link);
|
|
// spinlock_unlock(&cache->slablock);
|
|
|
|
return freed + slab_space_free(cache, slab);
|
|
|
|
} else if (slab->available == 1) {
|
|
/* It was in full, move to partial */
|
|
list_remove(&slab->link);
|
|
list_prepend(&slab->link, &cache->partial_slabs);
|
|
}
|
|
// spinlock_unlock(&cache->slablock);
|
|
return freed;
|
|
}
|
|
|
|
|
|
|
|
/** Return object to cache, use slab if known */
|
|
static void _slab_free(slab_cache_t *cache, void *obj, slab_t *slab)
|
|
{
|
|
eflags_t efl;
|
|
|
|
efl = safe_cli();
|
|
|
|
// if ((cache->flags & SLAB_CACHE_NOMAGAZINE) \
|
|
// || magazine_obj_put(cache, obj)) {
|
|
|
|
slab_obj_destroy(cache, obj, slab);
|
|
|
|
// }
|
|
safe_sti(efl);
|
|
atomic_dec(&cache->allocated_objs);
|
|
}
|
|
|
|
/** Return slab object to cache */
|
|
void __fastcall slab_free(slab_cache_t *cache, void *obj)
|
|
{
|
|
_slab_free(cache, obj, NULL);
|
|
}
|
|
|
|
static slab_t *slab_create()
|
|
{
|
|
slab_t *slab;
|
|
void *obj;
|
|
u32_t p;
|
|
|
|
// spinlock_lock(&cache->slablock);
|
|
|
|
if (list_empty(&slab_cache->partial_slabs)) {
|
|
/* Allow recursion and reclaiming
|
|
* - this should work, as the slab control structures
|
|
* are small and do not need to allocate with anything
|
|
* other than frame_alloc when they are allocating,
|
|
* that's why we should get recursion at most 1-level deep
|
|
*/
|
|
// spinlock_unlock(&cache->slablock);
|
|
// slab = slab_create();
|
|
|
|
void *data;
|
|
unsigned int i;
|
|
|
|
data = (void*)PA2KA(core_alloc(0));
|
|
if (!data) {
|
|
return NULL;
|
|
}
|
|
|
|
slab = (slab_t*)((u32_t)data + PAGE_SIZE - sizeof(slab_t));
|
|
|
|
/* Fill in slab structures */
|
|
frame_set_parent(ADDR2PFN(KA2PA(data)), slab);
|
|
|
|
slab->start = data;
|
|
slab->available = slab_cache->objects;
|
|
slab->nextavail = (void*)data;
|
|
slab->cache = slab_cache;
|
|
|
|
for (i = 0,p = (u32_t)slab->start;i < slab_cache->objects; i++)
|
|
{
|
|
*(int *)p = p+slab_cache->size;
|
|
p = p+slab_cache->size;
|
|
};
|
|
|
|
|
|
atomic_inc(&slab_cache->allocated_slabs);
|
|
// spinlock_lock(&cache->slablock);
|
|
} else {
|
|
slab = list_get_instance(slab_cache->partial_slabs.next, slab_t, link);
|
|
list_remove(&slab->link);
|
|
}
|
|
obj = slab->nextavail;
|
|
slab->nextavail = *((void**)obj);
|
|
slab->available--;
|
|
|
|
if (!slab->available)
|
|
list_prepend(&slab->link, &slab_cache->full_slabs);
|
|
else
|
|
list_prepend(&slab->link, &slab_cache->partial_slabs);
|
|
|
|
// spinlock_unlock(&cache->slablock);
|
|
|
|
return (slab_t*)obj;
|
|
}
|
|
|
|
static slab_cache_t * slab_cache_alloc()
|
|
{
|
|
slab_t *slab;
|
|
void *obj;
|
|
u32_t *p;
|
|
|
|
if (list_empty(&slab_cache_cache.partial_slabs)) {
|
|
/* Allow recursion and reclaiming
|
|
* - this should work, as the slab control structures
|
|
* are small and do not need to allocate with anything
|
|
* other than frame_alloc when they are allocating,
|
|
* that's why we should get recursion at most 1-level deep
|
|
*/
|
|
// spinlock_unlock(&cache->slablock);
|
|
// slab = slab_create();
|
|
|
|
void *data;
|
|
unsigned int i;
|
|
|
|
data = (void*)(PA2KA(core_alloc(0)));
|
|
if (!data) {
|
|
return NULL;
|
|
}
|
|
|
|
slab = (slab_t*)((u32_t)data + PAGE_SIZE - sizeof(slab_t));
|
|
|
|
/* Fill in slab structures */
|
|
frame_set_parent(ADDR2PFN(KA2PA(data)), slab);
|
|
|
|
slab->start = data;
|
|
slab->available = slab_cache_cache.objects;
|
|
slab->nextavail = (void*)data;
|
|
slab->cache = &slab_cache_cache;
|
|
|
|
for (i = 0,p = (u32_t*)slab->start;i < slab_cache_cache.objects; i++)
|
|
{
|
|
*p = (u32_t)p+slab_cache_cache.size;
|
|
p = (u32_t*)((u32_t)p+slab_cache_cache.size);
|
|
};
|
|
|
|
|
|
atomic_inc(&slab_cache_cache.allocated_slabs);
|
|
// spinlock_lock(&cache->slablock);
|
|
} else {
|
|
slab = list_get_instance(slab_cache_cache.partial_slabs.next, slab_t, link);
|
|
list_remove(&slab->link);
|
|
}
|
|
obj = slab->nextavail;
|
|
slab->nextavail = *((void**)obj);
|
|
slab->available--;
|
|
|
|
if (!slab->available)
|
|
list_prepend(&slab->link, &slab_cache_cache.full_slabs);
|
|
else
|
|
list_prepend(&slab->link, &slab_cache_cache.partial_slabs);
|
|
|
|
// spinlock_unlock(&cache->slablock);
|
|
|
|
return (slab_cache_t*)obj;
|
|
}
|
|
|
|
void slab_cache_init(void)
|
|
{
|
|
|
|
_slab_cache_create(&slab_cache_cache, sizeof(slab_cache_t),
|
|
sizeof(void *), NULL, NULL,
|
|
SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE);
|
|
|
|
/* Initialize external slab cache */
|
|
slab_cache = slab_cache_create(sizeof(slab_t),
|
|
0, NULL, NULL,SLAB_CACHE_MAGDEFERRED);
|
|
};
|
|
|