/*
* Copyright 2008-2010, Michael Lotz, mmlr@mlotz.ch.
* Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
* Distributed under the terms of the MIT License.
*
* Copyright 2001, Travis Geiselbrecht. All rights reserved.
* Distributed under the terms of the NewOS License.
*/
#include <arch/debug.h>
#include <debug.h>
#include <elf.h>
#include <heap.h>
#include <int.h>
#include <kernel.h>
#include <lock.h>
#include <string.h>
#include <team.h>
#include <thread.h>
#include <tracing.h>
#include <util/AutoLock.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
//#define TRACE_HEAP
#ifdef TRACE_HEAP
# define TRACE(x) dprintf x
#else
# define TRACE(x) ;
#endif
#if !USE_DEBUG_HEAP_FOR_MALLOC
# undef KERNEL_HEAP_LEAK_CHECK
#endif
#if KERNEL_HEAP_LEAK_CHECK
typedef struct heap_leak_check_info_s {
addr_t caller;
size_t size;
thread_id thread;
team_id team;
} heap_leak_check_info;
struct caller_info {
addr_t caller;
uint32 count;
uint32 size;
};
static const int32 kCallerInfoTableSize = 1024;
static caller_info sCallerInfoTable[kCallerInfoTableSize];
static int32 sCallerInfoCount = 0;
#endif // KERNEL_HEAP_LEAK_CHECK
typedef struct heap_page_s heap_page;
typedef struct heap_area_s {
area_id area;
addr_t base;
size_t size;
uint32 page_count;
uint32 free_page_count;
heap_page * free_pages;
heap_page * page_table;
heap_area_s * prev;
heap_area_s * next;
heap_area_s * all_next;
} heap_area;
#define MAX_BIN_COUNT 31 // depends on the size of the bin_index field
typedef struct heap_page_s {
heap_area * area;
uint16 index;
uint16 bin_index : 5;
uint16 free_count : 10;
uint16 in_use : 1;
heap_page_s * next;
heap_page_s * prev;
union {
uint16 empty_index;
uint16 allocation_id; // used for bin == bin_count allocations
};
addr_t * free_list;
} heap_page;
typedef struct heap_bin_s {
mutex lock;
uint32 element_size;
uint16 max_free_count;
heap_page * page_list; // sorted so that the desired page is always first
} heap_bin;
struct heap_allocator_s {
rw_lock area_lock;
mutex page_lock;
const char *name;
uint32 bin_count;
uint32 page_size;
uint32 total_pages;
uint32 total_free_pages;
uint32 empty_areas;
#if KERNEL_HEAP_LEAK_CHECK
addr_t (*get_caller)();
#endif
heap_bin * bins;
heap_area * areas; // sorted so that the desired area is always first
heap_area * all_areas; // all areas including full ones
};
static const uint32 kAreaAllocationMagic = 'AAMG';
typedef struct area_allocation_info_s {
area_id area;
void * base;
uint32 magic;
size_t size;
size_t allocation_size;
size_t allocation_alignment;
void * allocation_base;
} area_allocation_info;
struct DeferredFreeListEntry : SinglyLinkedListLinkImpl<DeferredFreeListEntry> {
};
typedef SinglyLinkedList<DeferredFreeListEntry> DeferredFreeList;
typedef SinglyLinkedList<DeferredDeletable> DeferredDeletableList;
#if USE_DEBUG_HEAP_FOR_MALLOC
#define VIP_HEAP_SIZE 1024 * 1024
// Heap class configuration
#define HEAP_CLASS_COUNT 3
static const heap_class sHeapClasses[HEAP_CLASS_COUNT] = {
{
"small", /* name */
50, /* initial percentage */
B_PAGE_SIZE / 8, /* max allocation size */
B_PAGE_SIZE, /* page size */
8, /* min bin size */
4, /* bin alignment */
8, /* min count per page */
16 /* max waste per page */
},
{
"medium", /* name */
30, /* initial percentage */
B_PAGE_SIZE * 2, /* max allocation size */
B_PAGE_SIZE * 8, /* page size */
B_PAGE_SIZE / 8, /* min bin size */
32, /* bin alignment */
4, /* min count per page */
64 /* max waste per page */
},
{
"large", /* name */
20, /* initial percentage */
HEAP_AREA_USE_THRESHOLD, /* max allocation size */
B_PAGE_SIZE * 16, /* page size */
B_PAGE_SIZE * 2, /* min bin size */
128, /* bin alignment */
1, /* min count per page */
256 /* max waste per page */
}
};
static uint32 sHeapCount;
static heap_allocator *sHeaps[HEAP_CLASS_COUNT * SMP_MAX_CPUS];
static uint32 *sLastGrowRequest[HEAP_CLASS_COUNT * SMP_MAX_CPUS];
static uint32 *sLastHandledGrowRequest[HEAP_CLASS_COUNT * SMP_MAX_CPUS];
static heap_allocator *sVIPHeap;
static heap_allocator *sGrowHeap = NULL;
static thread_id sHeapGrowThread = -1;
static sem_id sHeapGrowSem = -1;
static sem_id sHeapGrownNotify = -1;
static bool sAddGrowHeap = false;
#endif // USE_DEBUG_HEAP_FOR_MALLOC
static DeferredFreeList sDeferredFreeList;
static DeferredDeletableList sDeferredDeletableList;
static spinlock sDeferredFreeListLock;
// #pragma mark - Tracing
#if KERNEL_HEAP_TRACING
namespace KernelHeapTracing {
class Allocate : public AbstractTraceEntry {
public:
Allocate(addr_t address, size_t size)
: fAddress(address),
fSize(size)
{
Initialized();
}
virtual void AddDump(TraceOutput &out)
{
out.Print("heap allocate: 0x%08lx (%lu bytes)", fAddress, fSize);
}
private:
addr_t fAddress;
size_t fSize;
};
class Reallocate : public AbstractTraceEntry {
public:
Reallocate(addr_t oldAddress, addr_t newAddress, size_t newSize)
: fOldAddress(oldAddress),
fNewAddress(newAddress),
fNewSize(newSize)
{
Initialized();
};
virtual void AddDump(TraceOutput &out)
{
out.Print("heap reallocate: 0x%08lx -> 0x%08lx (%lu bytes)",
fOldAddress, fNewAddress, fNewSize);
}
private:
addr_t fOldAddress;
addr_t fNewAddress;
size_t fNewSize;
};
class Free : public AbstractTraceEntry {
public:
Free(addr_t address)
: fAddress(address)
{
Initialized();
};
virtual void AddDump(TraceOutput &out)
{
out.Print("heap free: 0x%08lx", fAddress);
}
private:
addr_t fAddress;
};
} // namespace KernelHeapTracing
# define T(x) if (!gKernelStartup) new(std::nothrow) KernelHeapTracing::x;
#else
# define T(x) ;
#endif
// #pragma mark - Debug functions
#if KERNEL_HEAP_LEAK_CHECK
static addr_t
get_caller()
{
// Find the first return address outside of the allocator code. Note, that
// this makes certain assumptions about how the code for the functions
// ends up in the kernel object.
addr_t returnAddresses[5];
int32 depth = arch_debug_get_stack_trace(returnAddresses, 5, 0, 1,
STACK_TRACE_KERNEL);
for (int32 i = 0; i < depth; i++) {
if (returnAddresses[i] < (addr_t)&get_caller
|| returnAddresses[i] > (addr_t)&malloc_referenced_release) {
return returnAddresses[i];
}
}
return 0;
}
#endif
static void
dump_page(heap_page *page)
{
uint32 count = 0;
for (addr_t *temp = page->free_list; temp != NULL; temp = (addr_t *)*temp)
count++;
kprintf("\t\tpage %p: bin_index: %u; free_count: %u; empty_index: %u; "
"free_list %p (%" B_PRIu32 " entr%s)\n", page, page->bin_index,
page->free_count, page->empty_index, page->free_list, count,
count == 1 ? "y" : "ies");
}
static void
dump_bin(heap_bin *bin)
{
uint32 count = 0;
for (heap_page *page = bin->page_list; page != NULL; page = page->next)
count++;
kprintf("\telement_size: %" B_PRIu32 "; max_free_count: %u; page_list %p "
"(%" B_PRIu32 " pages);\n", bin->element_size, bin->max_free_count,
bin->page_list, count);
for (heap_page *page = bin->page_list; page != NULL; page = page->next)
dump_page(page);
}
static void
dump_bin_list(heap_allocator *heap)
{
for (uint32 i = 0; i < heap->bin_count; i++)
dump_bin(&heap->bins[i]);
kprintf("\n");
}
static void
dump_allocator_areas(heap_allocator *heap)
{
heap_area *area = heap->all_areas;
while (area) {
kprintf("\tarea %p: area: %" B_PRId32 "; base: %p; size: %zu; page_count: "
"%" B_PRIu32 "; free_pages: %p (%" B_PRIu32 " entr%s)\n", area,
area->area, (void *)area->base, area->size, area->page_count,
area->free_pages, area->free_page_count,
area->free_page_count == 1 ? "y" : "ies");
area = area->all_next;
}
kprintf("\n");
}
static void
dump_allocator(heap_allocator *heap, bool areas, bool bins)
{
kprintf("allocator %p: name: %s; page_size: %" B_PRIu32 "; bin_count: "
"%" B_PRIu32 "; pages: %" B_PRIu32 "; free_pages: %" B_PRIu32 "; "
"empty_areas: %" B_PRIu32 "\n", heap, heap->name, heap->page_size,
heap->bin_count, heap->total_pages, heap->total_free_pages,
heap->empty_areas);
if (areas)
dump_allocator_areas(heap);
if (bins)
dump_bin_list(heap);
}
static int
dump_heap_list(int argc, char **argv)
{
#if USE_DEBUG_HEAP_FOR_MALLOC
if (argc == 2 && strcmp(argv[1], "grow") == 0) {
// only dump dedicated grow heap info
kprintf("dedicated grow heap:\n");
dump_allocator(sGrowHeap, true, true);
return 0;
}
#endif
bool stats = false;
int i = 1;
if (strcmp(argv[1], "stats") == 0) {
stats = true;
i++;
}
uint64 heapAddress = 0;
if (i < argc && !evaluate_debug_expression(argv[i], &heapAddress, true)) {
print_debugger_command_usage(argv[0]);
return 0;
}
if (heapAddress == 0) {
#if USE_DEBUG_HEAP_FOR_MALLOC
// dump default kernel heaps
for (uint32 i = 0; i < sHeapCount; i++)
dump_allocator(sHeaps[i], !stats, !stats);
#else
print_debugger_command_usage(argv[0]);
#endif
} else {
// dump specified heap
dump_allocator((heap_allocator*)(addr_t)heapAddress, !stats, !stats);
}
return 0;
}
#if !KERNEL_HEAP_LEAK_CHECK
static int
dump_allocations(int argc, char **argv)
{
uint64 heapAddress = 0;
bool statsOnly = false;
for (int32 i = 1; i < argc; i++) {
if (strcmp(argv[i], "stats") == 0)
statsOnly = true;
else if (!evaluate_debug_expression(argv[i], &heapAddress, true)) {
print_debugger_command_usage(argv[0]);
return 0;
}
}
size_t totalSize = 0;
uint32 totalCount = 0;
#if USE_DEBUG_HEAP_FOR_MALLOC
for (uint32 heapIndex = 0; heapIndex < sHeapCount; heapIndex++) {
heap_allocator *heap = sHeaps[heapIndex];
if (heapAddress != 0)
heap = (heap_allocator *)(addr_t)heapAddress;
#else
while (true) {
heap_allocator *heap = (heap_allocator *)(addr_t)heapAddress;
if (heap == NULL) {
print_debugger_command_usage(argv[0]);
return 0;
}
#endif
#if 0
}
#endif
// go through all the pages in all the areas
heap_area *area = heap->all_areas;
while (area) {
for (uint32 i = 0; i < area->page_count; i++) {
heap_page *page = &area->page_table[i];
if (!page->in_use)
continue;
addr_t base = area->base + i * heap->page_size;
if (page->bin_index < heap->bin_count) {
// page is used by a small allocation bin
uint32 elementCount = page->empty_index;
size_t elementSize
= heap->bins[page->bin_index].element_size;
for (uint32 j = 0; j < elementCount;
j++, base += elementSize) {
// walk the free list to see if this element is in use
bool elementInUse = true;
for (addr_t *temp = page->free_list; temp != NULL;
temp = (addr_t *)*temp) {
if ((addr_t)temp == base) {
elementInUse = false;
break;
}
}
if (!elementInUse)
continue;
if (!statsOnly) {
kprintf("address: 0x%p; size: %lu bytes\n",
(void *)base, elementSize);
}
totalSize += elementSize;
totalCount++;
}
} else {
// page is used by a big allocation, find the page count
uint32 pageCount = 1;
while (i + pageCount < area->page_count
&& area->page_table[i + pageCount].in_use
&& area->page_table[i + pageCount].bin_index
== heap->bin_count
&& area->page_table[i + pageCount].allocation_id
== page->allocation_id)
pageCount++;
size_t size = pageCount * heap->page_size;
if (!statsOnly) {
kprintf("address: %p; size: %lu bytes\n", (void *)base,
size);
}
totalSize += size;
totalCount++;
// skip the allocated pages
i += pageCount - 1;
}
}
area = area->all_next;
}
if (heapAddress != 0)
break;
}
kprintf("total allocations: %" B_PRIu32 "; total bytes: %zu\n", totalCount, totalSize);
return 0;
}
#else // !KERNEL_HEAP_LEAK_CHECK
static int
dump_allocations(int argc, char **argv)
{
team_id team = -1;
thread_id thread = -1;
addr_t caller = 0;
addr_t address = 0;
bool statsOnly = false;
for (int32 i = 1; i < argc; i++) {
if (strcmp(argv[i], "team") == 0)
team = parse_expression(argv[++i]);
else if (strcmp(argv[i], "thread") == 0)
thread = parse_expression(argv[++i]);
else if (strcmp(argv[i], "caller") == 0)
caller = parse_expression(argv[++i]);
else if (strcmp(argv[i], "address") == 0)
address = parse_expression(argv[++i]);
else if (strcmp(argv[i], "stats") == 0)
statsOnly = true;
else {
print_debugger_command_usage(argv[0]);
return 0;
}
}
size_t totalSize = 0;
uint32 totalCount = 0;
for (uint32 heapIndex = 0; heapIndex < sHeapCount; heapIndex++) {
heap_allocator *heap = sHeaps[heapIndex];
// go through all the pages in all the areas
heap_area *area = heap->all_areas;
while (area) {
heap_leak_check_info *info = NULL;
for (uint32 i = 0; i < area->page_count; i++) {
heap_page *page = &area->page_table[i];
if (!page->in_use)
continue;
addr_t base = area->base + i * heap->page_size;
if (page->bin_index < heap->bin_count) {
// page is used by a small allocation bin
uint32 elementCount = page->empty_index;
size_t elementSize
= heap->bins[page->bin_index].element_size;
for (uint32 j = 0; j < elementCount;
j++, base += elementSize) {
// walk the free list to see if this element is in use
bool elementInUse = true;
for (addr_t *temp = page->free_list; temp != NULL;
temp = (addr_t *)*temp) {
if ((addr_t)temp == base) {
elementInUse = false;
break;
}
}
if (!elementInUse)
continue;
info = (heap_leak_check_info *)(base + elementSize
- sizeof(heap_leak_check_info));
if ((team == -1 || info->team == team)
&& (thread == -1 || info->thread == thread)
&& (caller == 0 || info->caller == caller)
&& (address == 0 || base == address)) {
// interesting...
if (!statsOnly) {
kprintf("team: % 6ld; thread: % 6ld; "
"address: 0x%08lx; size: %lu bytes; "
"caller: %#lx\n", info->team, info->thread,
base, info->size, info->caller);
}
totalSize += info->size;
totalCount++;
}
}
} else {
// page is used by a big allocation, find the page count
uint32 pageCount = 1;
while (i + pageCount < area->page_count
&& area->page_table[i + pageCount].in_use
&& area->page_table[i + pageCount].bin_index
== heap->bin_count
&& area->page_table[i + pageCount].allocation_id
== page->allocation_id)
pageCount++;
info = (heap_leak_check_info *)(base + pageCount
* heap->page_size - sizeof(heap_leak_check_info));
if ((team == -1 || info->team == team)
&& (thread == -1 || info->thread == thread)
&& (caller == 0 || info->caller == caller)
&& (address == 0 || base == address)) {
// interesting...
if (!statsOnly) {
kprintf("team: % 6ld; thread: % 6ld;"
" address: 0x%08lx; size: %lu bytes;"
" caller: %#lx\n", info->team, info->thread,
base, info->size, info->caller);
}
totalSize += info->size;
totalCount++;
}
// skip the allocated pages
i += pageCount - 1;
}
}
area = area->all_next;
}
}
kprintf("total allocations: %lu; total bytes: %lu\n", totalCount,
totalSize);
return 0;
}
static caller_info*
get_caller_info(addr_t caller)
{
// find the caller info
for (int32 i = 0; i < sCallerInfoCount; i++) {
if (caller == sCallerInfoTable[i].caller)
return &sCallerInfoTable[i];
}
// not found, add a new entry, if there are free slots
if (sCallerInfoCount >= kCallerInfoTableSize)
return NULL;
caller_info* info = &sCallerInfoTable[sCallerInfoCount++];
info->caller = caller;
info->count = 0;
info->size = 0;
return info;
}
static int
caller_info_compare_size(const void* _a, const void* _b)
{
const caller_info* a = (const caller_info*)_a;
const caller_info* b = (const caller_info*)_b;
return (int)(b->size - a->size);
}
static int
caller_info_compare_count(const void* _a, const void* _b)
{
const caller_info* a = (const caller_info*)_a;
const caller_info* b = (const caller_info*)_b;
return (int)(b->count - a->count);
}
static bool
analyze_allocation_callers(heap_allocator *heap)
{
// go through all the pages in all the areas
heap_area *area = heap->all_areas;
while (area) {
heap_leak_check_info *info = NULL;
for (uint32 i = 0; i < area->page_count; i++) {
heap_page *page = &area->page_table[i];
if (!page->in_use)
continue;
addr_t base = area->base + i * heap->page_size;
if (page->bin_index < heap->bin_count) {
// page is used by a small allocation bin
uint32 elementCount = page->empty_index;
size_t elementSize = heap->bins[page->bin_index].element_size;
for (uint32 j = 0; j < elementCount; j++, base += elementSize) {
// walk the free list to see if this element is in use
bool elementInUse = true;
for (addr_t *temp = page->free_list; temp != NULL;
temp = (addr_t *)*temp) {
if ((addr_t)temp == base) {
elementInUse = false;
break;
}
}
if (!elementInUse)
continue;
info = (heap_leak_check_info *)(base + elementSize
- sizeof(heap_leak_check_info));
caller_info *callerInfo = get_caller_info(info->caller);
if (callerInfo == NULL) {
kprintf("out of space for caller infos\n");
return false;
}
callerInfo->count++;
callerInfo->size += info->size;
}
} else {
// page is used by a big allocation, find the page count
uint32 pageCount = 1;
while (i + pageCount < area->page_count
&& area->page_table[i + pageCount].in_use
&& area->page_table[i + pageCount].bin_index
== heap->bin_count
&& area->page_table[i + pageCount].allocation_id
== page->allocation_id) {
pageCount++;
}
info = (heap_leak_check_info *)(base + pageCount
* heap->page_size - sizeof(heap_leak_check_info));
caller_info *callerInfo = get_caller_info(info->caller);
if (callerInfo == NULL) {
kprintf("out of space for caller infos\n");
return false;
}
callerInfo->count++;
callerInfo->size += info->size;
// skip the allocated pages
i += pageCount - 1;
}
}
area = area->all_next;
}
return true;
}
static int
dump_allocations_per_caller(int argc, char **argv)
{
bool sortBySize = true;
heap_allocator *heap = NULL;
for (int32 i = 1; i < argc; i++) {
if (strcmp(argv[i], "-c") == 0) {
sortBySize = false;
} else if (strcmp(argv[i], "-h") == 0) {
uint64 heapAddress;
if (++i >= argc
|| !evaluate_debug_expression(argv[i], &heapAddress, true)) {
print_debugger_command_usage(argv[0]);
return 0;
}
heap = (heap_allocator*)(addr_t)heapAddress;
} else {
print_debugger_command_usage(argv[0]);
return 0;
}
}
sCallerInfoCount = 0;
if (heap != NULL) {
if (!analyze_allocation_callers(heap))
return 0;
} else {
for (uint32 heapIndex = 0; heapIndex < sHeapCount; heapIndex++) {
if (!analyze_allocation_callers(sHeaps[heapIndex]))
return 0;
}
}
// sort the array
qsort(sCallerInfoTable, sCallerInfoCount, sizeof(caller_info),
sortBySize ? &caller_info_compare_size : &caller_info_compare_count);
kprintf("%ld different callers, sorted by %s...\n\n", sCallerInfoCount,
sortBySize ? "size" : "count");
kprintf(" count size caller\n");
kprintf("----------------------------------\n");
for (int32 i = 0; i < sCallerInfoCount; i++) {
caller_info& info = sCallerInfoTable[i];
kprintf("%10ld %10ld %#08lx", info.count, info.size, info.caller);
const char *symbol;
const char *imageName;
bool exactMatch;
addr_t baseAddress;
if (elf_debug_lookup_symbol_address(info.caller, &baseAddress, &symbol,
&imageName, &exactMatch) == B_OK) {
kprintf(" %s + 0x%lx (%s)%s\n", symbol,
info.caller - baseAddress, imageName,
exactMatch ? "" : " (nearest)");
} else
kprintf("\n");
}
return 0;
}
#endif // KERNEL_HEAP_LEAK_CHECK
#if PARANOID_HEAP_VALIDATION
static void
heap_validate_heap(heap_allocator *heap)
{
ReadLocker areaReadLocker(heap->area_lock);
for (uint32 i = 0; i < heap->bin_count; i++)
mutex_lock(&heap->bins[i].lock);
MutexLocker pageLocker(heap->page_lock);
uint32 totalPageCount = 0;
uint32 totalFreePageCount = 0;
heap_area *area = heap->all_areas;
while (area != NULL) {
// validate the free pages list
uint32 freePageCount = 0;
heap_page *lastPage = NULL;
heap_page *page = area->free_pages;
while (page) {
if ((addr_t)page < (addr_t)&area->page_table[0]
|| (addr_t)page >= (addr_t)&area->page_table[area->page_count])
panic("free page is not part of the page table\n");
if (page->index >= area->page_count)
panic("free page has invalid index\n");
if ((addr_t)&area->page_table[page->index] != (addr_t)page)
panic("free page index does not lead to target page\n");
if (page->prev != lastPage)
panic("free page entry has invalid prev link\n");
if (page->in_use)
panic("free page marked as in use\n");
lastPage = page;
page = page->next;
freePageCount++;
}
totalPageCount += freePageCount;
totalFreePageCount += freePageCount;
if (area->free_page_count != freePageCount)
panic("free page count doesn't match free page list\n");
// validate the page table
uint32 usedPageCount = 0;
for (uint32 i = 0; i < area->page_count; i++) {
if (area->page_table[i].in_use)
usedPageCount++;
}
totalPageCount += usedPageCount;
if (freePageCount + usedPageCount != area->page_count) {
panic("free pages and used pages do not add up (%lu + %lu != %lu)\n",
freePageCount, usedPageCount, area->page_count);
}
area = area->all_next;
}
// validate the areas
area = heap->areas;
heap_area *lastArea = NULL;
uint32 lastFreeCount = 0;
while (area != NULL) {
if (area->free_page_count < lastFreeCount)
panic("size ordering of area list broken\n");
if (area->prev != lastArea)
panic("area list entry has invalid prev link\n");
lastArea = area;
lastFreeCount = area->free_page_count;
area = area->next;
}
lastArea = NULL;
area = heap->all_areas;
while (area != NULL) {
if (lastArea != NULL && lastArea->base < area->base)
panic("base ordering of all_areas list broken\n");
lastArea = area;
area = area->all_next;
}
// validate the bins
for (uint32 i = 0; i < heap->bin_count; i++) {
heap_bin *bin = &heap->bins[i];
heap_page *lastPage = NULL;
heap_page *page = bin->page_list;
lastFreeCount = 0;
while (page) {
area = heap->all_areas;
while (area) {
if (area == page->area)
break;
area = area->all_next;
}
if (area == NULL) {
panic("page area not present in area list\n");
page = page->next;
continue;
}
if ((addr_t)page < (addr_t)&area->page_table[0]
|| (addr_t)page >= (addr_t)&area->page_table[area->page_count])
panic("used page is not part of the page table\n");
if (page->index >= area->page_count)
panic("used page has invalid index\n");
if ((addr_t)&area->page_table[page->index] != (addr_t)page)
panic("used page index does not lead to target page\n");
if (page->prev != lastPage) {
panic("used page entry has invalid prev link (%p vs %p bin "
"%lu)\n", page->prev, lastPage, i);
}
if (!page->in_use)
panic("used page marked as not in use\n");
if (page->bin_index != i) {
panic("used page with bin index %u in page list of bin %lu\n",
page->bin_index, i);
}
if (page->free_count < lastFreeCount)
panic("ordering of bin page list broken\n");
// validate the free list
uint32 freeSlotsCount = 0;
addr_t *element = page->free_list;
addr_t pageBase = area->base + page->index * heap->page_size;
while (element) {
if ((addr_t)element < pageBase
|| (addr_t)element >= pageBase + heap->page_size)
panic("free list entry out of page range\n");
if (((addr_t)element - pageBase) % bin->element_size != 0)
panic("free list entry not on a element boundary\n");
element = (addr_t *)*element;
freeSlotsCount++;
}
uint32 slotCount = bin->max_free_count;
if (page->empty_index > slotCount) {
panic("empty index beyond slot count (%u with %lu slots)\n",
page->empty_index, slotCount);
}
freeSlotsCount += (slotCount - page->empty_index);
if (freeSlotsCount > slotCount)
panic("more free slots than fit into the page\n");
lastPage = page;
lastFreeCount = page->free_count;
page = page->next;
}
}
pageLocker.Unlock();
for (uint32 i = 0; i < heap->bin_count; i++)
mutex_unlock(&heap->bins[i].lock);
areaReadLocker.Unlock();
}
#endif // PARANOID_HEAP_VALIDATION
// #pragma mark - Heap functions
void
heap_add_area(heap_allocator *heap, area_id areaID, addr_t base, size_t size)
{
heap_area *area = (heap_area *)base;
area->area = areaID;
base += sizeof(heap_area);
size -= sizeof(heap_area);
uint32 pageCount = size / heap->page_size;
size_t pageTableSize = pageCount * sizeof(heap_page);
area->page_table = (heap_page *)base;
base += pageTableSize;
size -= pageTableSize;
// the rest is now actually usable memory (rounded to the next page)
area->base = ROUNDUP(base, B_PAGE_SIZE);
area->size = size & ~(B_PAGE_SIZE - 1);
// now we know the real page count
pageCount = area->size / heap->page_size;
area->page_count = pageCount;
// zero out the page table and fill in page indexes
memset((void *)area->page_table, 0, pageTableSize);
for (uint32 i = 0; i < pageCount; i++) {
area->page_table[i].area = area;
area->page_table[i].index = i;
}
// add all pages up into the free pages list
for (uint32 i = 1; i < pageCount; i++) {
area->page_table[i - 1].next = &area->page_table[i];
area->page_table[i].prev = &area->page_table[i - 1];
}
area->free_pages = &area->page_table[0];
area->free_page_count = pageCount;
area->page_table[0].prev = NULL;
area->next = NULL;
WriteLocker areaWriteLocker(heap->area_lock);
MutexLocker pageLocker(heap->page_lock);
if (heap->areas == NULL) {
// it's the only (empty) area in that heap
area->prev = NULL;
heap->areas = area;
} else {
// link in this area as the last one as it is completely empty
heap_area *lastArea = heap->areas;
while (lastArea->next != NULL)
lastArea = lastArea->next;
lastArea->next = area;
area->prev = lastArea;
}
// insert this area in the all_areas list so it stays ordered by base
if (heap->all_areas == NULL || heap->all_areas->base < area->base) {
area->all_next = heap->all_areas;
heap->all_areas = area;
} else {
heap_area *insert = heap->all_areas;
while (insert->all_next && insert->all_next->base > area->base)
insert = insert->all_next;
area->all_next = insert->all_next;
insert->all_next = area;
}
heap->total_pages += area->page_count;
heap->total_free_pages += area->free_page_count;
if (areaID >= 0) {
// this later on deletable area is yet empty - the empty count will be
// decremented as soon as this area is used for the first time
heap->empty_areas++;
}
pageLocker.Unlock();
areaWriteLocker.Unlock();
dprintf("heap_add_area: area %" B_PRId32 " added to %s heap %p - usable "
"range %p - %p\n", area->area, heap->name, heap, (void *)area->base,
(void *)(area->base + area->size));
}
static status_t
heap_remove_area(heap_allocator *heap, heap_area *area)
{
if (area->free_page_count != area->page_count) {
panic("tried removing heap area that has still pages in use");
return B_ERROR;
}
if (area->prev == NULL && area->next == NULL) {
panic("tried removing the last non-full heap area");
return B_ERROR;
}
if (heap->areas == area)
heap->areas = area->next;
if (area->prev != NULL)
area->prev->next = area->next;
if (area->next != NULL)
area->next->prev = area->prev;
if (heap->all_areas == area)
heap->all_areas = area->all_next;
else {
heap_area *previous = heap->all_areas;
while (previous) {
if (previous->all_next == area) {
previous->all_next = area->all_next;
break;
}
previous = previous->all_next;
}
if (previous == NULL)
panic("removing heap area that is not in all list");
}
heap->total_pages -= area->page_count;
heap->total_free_pages -= area->free_page_count;
dprintf("heap_remove_area: area %" B_PRId32 " with range %p - %p removed "
"from %s heap %p\n", area->area, (void *)area->base,
(void *)(area->base + area->size), heap->name, heap);
return B_OK;
}
heap_allocator *
heap_create_allocator(const char *name, addr_t base, size_t size,
const heap_class *heapClass, bool allocateOnHeap)
{
heap_allocator *heap;
if (allocateOnHeap) {
// allocate seperately on the heap
heap = (heap_allocator *)malloc(sizeof(heap_allocator)
+ sizeof(heap_bin) * MAX_BIN_COUNT);
} else {
// use up the first part of the area
heap = (heap_allocator *)base;
base += sizeof(heap_allocator);
size -= sizeof(heap_allocator);
}
heap->name = name;
heap->page_size = heapClass->page_size;
heap->total_pages = heap->total_free_pages = heap->empty_areas = 0;
heap->areas = heap->all_areas = NULL;
heap->bins = (heap_bin *)((addr_t)heap + sizeof(heap_allocator));
#if KERNEL_HEAP_LEAK_CHECK
heap->get_caller = &get_caller;
#endif
heap->bin_count = 0;
size_t binSize = 0, lastSize = 0;
uint32 count = heap->page_size / heapClass->min_bin_size;
for (; count >= heapClass->min_count_per_page; count--, lastSize = binSize) {
if (heap->bin_count >= MAX_BIN_COUNT)
panic("heap configuration invalid - max bin count reached\n");
binSize = (heap->page_size / count) & ~(heapClass->bin_alignment - 1);
if (binSize == lastSize)
continue;
if (heap->page_size - count * binSize > heapClass->max_waste_per_page)
continue;
heap_bin *bin = &heap->bins[heap->bin_count];
mutex_init(&bin->lock, "heap bin lock");
bin->element_size = binSize;
bin->max_free_count = heap->page_size / binSize;
bin->page_list = NULL;
heap->bin_count++;
};
if (!allocateOnHeap) {
base += heap->bin_count * sizeof(heap_bin);
size -= heap->bin_count * sizeof(heap_bin);
}
rw_lock_init(&heap->area_lock, "heap area rw lock");
mutex_init(&heap->page_lock, "heap page lock");
heap_add_area(heap, -1, base, size);
return heap;
}
static inline void
heap_free_pages_added(heap_allocator *heap, heap_area *area, uint32 pageCount)
{
area->free_page_count += pageCount;
heap->total_free_pages += pageCount;
if (area->free_page_count == pageCount) {
// we need to add ourselfs to the area list of the heap
area->prev = NULL;
area->next = heap->areas;
if (area->next)
area->next->prev = area;
heap->areas = area;
} else {
// we might need to move back in the area list
if (area->next && area->next->free_page_count < area->free_page_count) {
// move ourselfs so the list stays ordered
heap_area *insert = area->next;
while (insert->next
&& insert->next->free_page_count < area->free_page_count)
insert = insert->next;
if (area->prev)
area->prev->next = area->next;
if (area->next)
area->next->prev = area->prev;
if (heap->areas == area)
heap->areas = area->next;
area->prev = insert;
area->next = insert->next;
if (area->next)
area->next->prev = area;
insert->next = area;
}
}
if (area->free_page_count == area->page_count && area->area >= 0)
heap->empty_areas++;
}
static inline void
heap_free_pages_removed(heap_allocator *heap, heap_area *area, uint32 pageCount)
{
if (area->free_page_count == area->page_count && area->area >= 0) {
// this area was completely empty
heap->empty_areas--;
}
area->free_page_count -= pageCount;
heap->total_free_pages -= pageCount;
if (area->free_page_count == 0) {
// the area is now full so we remove it from the area list
if (area->prev)
area->prev->next = area->next;
if (area->next)
area->next->prev = area->prev;
if (heap->areas == area)
heap->areas = area->next;
area->next = area->prev = NULL;
} else {
// we might need to move forward in the area list
if (area->prev && area->prev->free_page_count > area->free_page_count) {
// move ourselfs so the list stays ordered
heap_area *insert = area->prev;
while (insert->prev
&& insert->prev->free_page_count > area->free_page_count)
insert = insert->prev;
if (area->prev)
area->prev->next = area->next;
if (area->next)
area->next->prev = area->prev;
area->prev = insert->prev;
area->next = insert;
if (area->prev)
area->prev->next = area;
if (heap->areas == insert)
heap->areas = area;
insert->prev = area;
}
}
}
static inline void
heap_link_page(heap_page *page, heap_page **list)
{
page->prev = NULL;
page->next = *list;
if (page->next)
page->next->prev = page;
*list = page;
}
static inline void
heap_unlink_page(heap_page *page, heap_page **list)
{
if (page->prev)
page->prev->next = page->next;
if (page->next)
page->next->prev = page->prev;
if (list && *list == page) {
*list = page->next;
if (page->next)
page->next->prev = NULL;
}
}
static heap_page *
heap_allocate_contiguous_pages(heap_allocator *heap, uint32 pageCount,
size_t alignment)
{
MutexLocker pageLocker(heap->page_lock);
heap_area *area = heap->areas;
while (area) {
if (area->free_page_count < pageCount) {
area = area->next;
continue;
}
uint32 step = 1;
uint32 firstValid = 0;
const uint32 lastValid = area->page_count - pageCount + 1;
if (alignment > heap->page_size) {
firstValid = (ROUNDUP(area->base, alignment) - area->base)
/ heap->page_size;
step = alignment / heap->page_size;
}
int32 first = -1;
for (uint32 i = firstValid; i < lastValid; i += step) {
if (area->page_table[i].in_use)
continue;
first = i;
for (uint32 j = 1; j < pageCount; j++) {
if (area->page_table[i + j].in_use) {
first = -1;
i += j / step * step;
break;
}
}
if (first >= 0)
break;
}
if (first < 0) {
area = area->next;
continue;
}
for (uint32 i = first; i < first + pageCount; i++) {
heap_page *page = &area->page_table[i];
page->in_use = 1;
page->bin_index = heap->bin_count;
heap_unlink_page(page, &area->free_pages);
page->next = page->prev = NULL;
page->free_list = NULL;
page->allocation_id = (uint16)first;
}
heap_free_pages_removed(heap, area, pageCount);
return &area->page_table[first];
}
return NULL;
}
#if KERNEL_HEAP_LEAK_CHECK
static void
heap_add_leak_check_info(heap_allocator *heap, addr_t address, size_t allocated,
size_t size)
{
heap_leak_check_info *info = (heap_leak_check_info *)(address + allocated
- sizeof(heap_leak_check_info));
info->size = size - sizeof(heap_leak_check_info);
info->thread = (gKernelStartup ? 0 : thread_get_current_thread_id());
info->team = (gKernelStartup ? 0 : team_get_current_team_id());
info->caller = heap->get_caller();
}
#endif
static void *
heap_raw_alloc(heap_allocator *heap, size_t size, size_t alignment)
{
TRACE(("heap %p: allocate %lu bytes from raw pages with alignment %lu\n",
heap, size, alignment));
uint32 pageCount = (size + heap->page_size - 1) / heap->page_size;
heap_page *firstPage = heap_allocate_contiguous_pages(heap, pageCount,
alignment);
if (firstPage == NULL) {
TRACE(("heap %p: found no contiguous pages to allocate %ld bytes\n",
heap, size));
return NULL;
}
addr_t address = firstPage->area->base + firstPage->index * heap->page_size;
#if KERNEL_HEAP_LEAK_CHECK
heap_add_leak_check_info(heap, address, pageCount * heap->page_size, size);
#endif
return (void *)address;
}
static void *
heap_allocate_from_bin(heap_allocator *heap, uint32 binIndex, size_t size)
{
heap_bin *bin = &heap->bins[binIndex];
TRACE(("heap %p: allocate %lu bytes from bin %lu with element_size %lu\n",
heap, size, binIndex, bin->element_size));
MutexLocker binLocker(bin->lock);
heap_page *page = bin->page_list;
if (page == NULL) {
MutexLocker pageLocker(heap->page_lock);
heap_area *area = heap->areas;
if (area == NULL) {
TRACE(("heap %p: no free pages to allocate %lu bytes\n", heap,
size));
return NULL;
}
// by design there are only areas in the list that still have
// free pages available
page = area->free_pages;
area->free_pages = page->next;
if (page->next)
page->next->prev = NULL;
heap_free_pages_removed(heap, area, 1);
if (page->in_use)
panic("got an in use page %p from the free pages list\n", page);
page->in_use = 1;
pageLocker.Unlock();
page->bin_index = binIndex;
page->free_count = bin->max_free_count;
page->empty_index = 0;
page->free_list = NULL;
page->next = page->prev = NULL;
bin->page_list = page;
}
// we have a page where we have a free slot
void *address = NULL;
if (page->free_list) {
// there's a previously freed entry we can use
address = page->free_list;
page->free_list = (addr_t *)*page->free_list;
} else {
// the page hasn't been fully allocated so use the next empty_index
address = (void *)(page->area->base + page->index * heap->page_size
+ page->empty_index * bin->element_size);
page->empty_index++;
}
page->free_count--;
if (page->free_count == 0) {
// the page is now full so we remove it from the page_list
bin->page_list = page->next;
if (page->next)
page->next->prev = NULL;
page->next = page->prev = NULL;
}
#if KERNEL_HEAP_LEAK_CHECK
binLocker.Unlock();
heap_add_leak_check_info(heap, (addr_t)address, bin->element_size, size);
#endif
return address;
}
static bool
is_valid_alignment(size_t number)
{
// this cryptic line accepts zero and all powers of two
return ((~number + 1) | ((number << 1) - 1)) == ~0UL;
}
inline bool
heap_should_grow(heap_allocator *heap)
{
// suggest growing if there is less than 20% of a grow size available
return heap->total_free_pages * heap->page_size < HEAP_GROW_SIZE / 5;
}
void *
heap_memalign(heap_allocator *heap, size_t alignment, size_t size)
{
TRACE(("memalign(alignment = %lu, size = %lu)\n", alignment, size));
#if DEBUG
if (!is_valid_alignment(alignment))
panic("memalign() with an alignment which is not a power of 2\n");
#endif
#if KERNEL_HEAP_LEAK_CHECK
size += sizeof(heap_leak_check_info);
#endif
void *address = NULL;
if (alignment < B_PAGE_SIZE) {
if (alignment != 0) {
// TODO: The alignment is done by ensuring that the element size
// of the target bin is aligned with the requested alignment. This
// has the problem that it wastes space because a better (smaller)
// bin could possibly be selected. We should pick the best bin and
// check if there is an aligned block in the free list or if a new
// (page aligned) page has to be allocated anyway.
size = ROUNDUP(size, alignment);
for (uint32 i = 0; i < heap->bin_count; i++) {
if (size <= heap->bins[i].element_size
&& is_valid_alignment(heap->bins[i].element_size)) {
address = heap_allocate_from_bin(heap, i, size);
break;
}
}
} else {
for (uint32 i = 0; i < heap->bin_count; i++) {
if (size <= heap->bins[i].element_size) {
address = heap_allocate_from_bin(heap, i, size);
break;
}
}
}
}
if (address == NULL)
address = heap_raw_alloc(heap, size, alignment);
#if KERNEL_HEAP_LEAK_CHECK
size -= sizeof(heap_leak_check_info);
#endif
TRACE(("memalign(): asked to allocate %lu bytes, returning pointer %p\n",
size, address));
T(Allocate((addr_t)address, size));
if (address == NULL)
return address;
#if PARANOID_KERNEL_MALLOC
memset(address, 0xcc, size);
#endif
#if PARANOID_KERNEL_FREE
// make sure 0xdeadbeef is cleared if we do not overwrite the memory
// and the user does not clear it
if (((uint32 *)address)[1] == 0xdeadbeef)
((uint32 *)address)[1] = 0xcccccccc;
#endif
return address;
}
status_t
heap_free(heap_allocator *heap, void *address)
{
if (address == NULL)
return B_OK;
ReadLocker areaReadLocker(heap->area_lock);
heap_area *area = heap->all_areas;
while (area) {
// since the all_areas list is ordered by base with the biggest
// base at the top, we need only find the first area with a base
// smaller than our address to become our only candidate for freeing
if (area->base <= (addr_t)address) {
if ((addr_t)address >= area->base + area->size) {
// none of the other areas can contain the address as the list
// is ordered
return B_ENTRY_NOT_FOUND;
}
// this area contains the allocation, we're done searching
break;
}
area = area->all_next;
}
if (area == NULL) {
// this address does not belong to us
return B_ENTRY_NOT_FOUND;
}
TRACE(("free(): asked to free pointer %p\n", address));
heap_page *page = &area->page_table[((addr_t)address - area->base)
/ heap->page_size];
TRACE(("free(): page %p: bin_index %d, free_count %d\n", page,
page->bin_index, page->free_count));
if (page->bin_index > heap->bin_count) {
panic("free(): page %p: invalid bin_index %d\n", page, page->bin_index);
return B_ERROR;
}
if (page->bin_index < heap->bin_count) {
// small allocation
heap_bin *bin = &heap->bins[page->bin_index];
#if PARANOID_KERNEL_FREE
if (((uint32 *)address)[1] == 0xdeadbeef) {
// This block looks like it was freed already, walk the free list
// on this page to make sure this address doesn't exist.
MutexLocker binLocker(bin->lock);
for (addr_t *temp = page->free_list; temp != NULL;
temp = (addr_t *)*temp) {
if (temp == address) {
panic("free(): address %p already exists in page free "
"list\n", address);
return B_ERROR;
}
}
}
// the first 4 bytes are overwritten with the next free list pointer
// later
uint32 *dead = (uint32 *)address;
for (uint32 i = 1; i < bin->element_size / sizeof(uint32); i++)
dead[i] = 0xdeadbeef;
#endif
MutexLocker binLocker(bin->lock);
if (((addr_t)address - area->base - page->index
* heap->page_size) % bin->element_size != 0) {
panic("free(): passed invalid pointer %p supposed to be in bin for "
"element size %" B_PRIu32 "\n", address, bin->element_size);
return B_ERROR;
}
// add the address to the page free list
*(addr_t *)address = (addr_t)page->free_list;
page->free_list = (addr_t *)address;
page->free_count++;
if (page->free_count == bin->max_free_count) {
// we are now empty, remove the page from the bin list
MutexLocker pageLocker(heap->page_lock);
heap_unlink_page(page, &bin->page_list);
page->in_use = 0;
heap_link_page(page, &area->free_pages);
heap_free_pages_added(heap, area, 1);
} else if (page->free_count == 1) {
// we need to add ourselfs to the page list of the bin
heap_link_page(page, &bin->page_list);
} else {
// we might need to move back in the free pages list
if (page->next && page->next->free_count < page->free_count) {
// move ourselfs so the list stays ordered
heap_page *insert = page->next;
while (insert->next
&& insert->next->free_count < page->free_count)
insert = insert->next;
heap_unlink_page(page, &bin->page_list);
page->prev = insert;
page->next = insert->next;
if (page->next)
page->next->prev = page;
insert->next = page;
}
}
} else {
// large allocation, just return the pages to the page free list
uint32 allocationID = page->allocation_id;
uint32 maxPages = area->page_count - page->index;
uint32 pageCount = 0;
MutexLocker pageLocker(heap->page_lock);
for (uint32 i = 0; i < maxPages; i++) {
// loop until we find the end of this allocation
if (!page[i].in_use || page[i].bin_index != heap->bin_count
|| page[i].allocation_id != allocationID)
break;
// this page still belongs to the same allocation
page[i].in_use = 0;
page[i].allocation_id = 0;
// return it to the free list
heap_link_page(&page[i], &area->free_pages);
pageCount++;
}
heap_free_pages_added(heap, area, pageCount);
}
T(Free((addr_t)address));
areaReadLocker.Unlock();
if (heap->empty_areas > 1) {
WriteLocker areaWriteLocker(heap->area_lock);
MutexLocker pageLocker(heap->page_lock);
area_id areasToDelete[heap->empty_areas - 1];
int32 areasToDeleteIndex = 0;
area = heap->areas;
while (area != NULL && heap->empty_areas > 1) {
heap_area *next = area->next;
if (area->area >= 0
&& area->free_page_count == area->page_count
&& heap_remove_area(heap, area) == B_OK) {
areasToDelete[areasToDeleteIndex++] = area->area;
heap->empty_areas--;
}
area = next;
}
pageLocker.Unlock();
areaWriteLocker.Unlock();
for (int32 i = 0; i < areasToDeleteIndex; i++)
delete_area(areasToDelete[i]);
}
return B_OK;
}
#if KERNEL_HEAP_LEAK_CHECK
extern "C" void
heap_set_get_caller(heap_allocator* heap, addr_t (*getCaller)())
{
heap->get_caller = getCaller;
}
#endif
#if USE_DEBUG_HEAP_FOR_MALLOC
static status_t
heap_realloc(heap_allocator *heap, void *address, void **newAddress,
size_t newSize)
{
ReadLocker areaReadLocker(heap->area_lock);
heap_area *area = heap->all_areas;
while (area) {
// since the all_areas list is ordered by base with the biggest
// base at the top, we need only find the first area with a base
// smaller than our address to become our only candidate for
// reallocating
if (area->base <= (addr_t)address) {
if ((addr_t)address >= area->base + area->size) {
// none of the other areas can contain the address as the list
// is ordered
return B_ENTRY_NOT_FOUND;
}
// this area contains the allocation, we're done searching
break;
}
area = area->all_next;
}
if (area == NULL) {
// this address does not belong to us
return B_ENTRY_NOT_FOUND;
}
TRACE(("realloc(address = %p, newSize = %lu)\n", address, newSize));
heap_page *page = &area->page_table[((addr_t)address - area->base)
/ heap->page_size];
if (page->bin_index > heap->bin_count) {
panic("realloc(): page %p: invalid bin_index %d\n", page,
page->bin_index);
return B_ERROR;
}
// find out the size of the old allocation first
size_t minSize = 0;
size_t maxSize = 0;
if (page->bin_index < heap->bin_count) {
// this was a small allocation
heap_bin *bin = &heap->bins[page->bin_index];
maxSize = bin->element_size;
if (page->bin_index > 0)
minSize = heap->bins[page->bin_index - 1].element_size + 1;
} else {
// this was a large allocation
uint32 allocationID = page->allocation_id;
uint32 maxPages = area->page_count - page->index;
maxSize = heap->page_size;
MutexLocker pageLocker(heap->page_lock);
for (uint32 i = 1; i < maxPages; i++) {
if (!page[i].in_use || page[i].bin_index != heap->bin_count
|| page[i].allocation_id != allocationID)
break;
minSize += heap->page_size;
maxSize += heap->page_size;
}
}
areaReadLocker.Unlock();
#if KERNEL_HEAP_LEAK_CHECK
newSize += sizeof(heap_leak_check_info);
#endif
// does the new allocation simply fit in the old allocation?
if (newSize > minSize && newSize <= maxSize) {
#if KERNEL_HEAP_LEAK_CHECK
// update the size info (the info is at the end so stays where it is)
heap_leak_check_info *info = (heap_leak_check_info *)((addr_t)address
+ maxSize - sizeof(heap_leak_check_info));
info->size = newSize - sizeof(heap_leak_check_info);
newSize -= sizeof(heap_leak_check_info);
#endif
T(Reallocate((addr_t)address, (addr_t)address, newSize));
*newAddress = address;
return B_OK;
}
#if KERNEL_HEAP_LEAK_CHECK
// new leak check info will be created with the malloc below
newSize -= sizeof(heap_leak_check_info);
#endif
// if not, allocate a new chunk of memory
*newAddress = memalign(0, newSize);
T(Reallocate((addr_t)address, (addr_t)*newAddress, newSize));
if (*newAddress == NULL) {
// we tried but it didn't work out, but still the operation is done
return B_OK;
}
// copy the old data and free the old allocation
memcpy(*newAddress, address, min_c(maxSize, newSize));
heap_free(heap, address);
return B_OK;
}
inline uint32
heap_index_for(size_t size, int32 cpu)
{
#if KERNEL_HEAP_LEAK_CHECK
// take the extra info size into account
size += sizeof(heap_leak_check_info_s);
#endif
uint32 index = 0;
for (; index < HEAP_CLASS_COUNT - 1; index++) {
if (size <= sHeapClasses[index].max_allocation_size)
break;
}
return (index + cpu * HEAP_CLASS_COUNT) % sHeapCount;
}
static void *
memalign_nogrow(size_t alignment, size_t size)
{
// use dedicated memory in the grow thread by default
if (thread_get_current_thread_id() == sHeapGrowThread) {
void *result = heap_memalign(sGrowHeap, alignment, size);
if (!sAddGrowHeap && heap_should_grow(sGrowHeap)) {
// hopefully the heap grower will manage to create a new heap
// before running out of private memory...
dprintf("heap: requesting new grow heap\n");
sAddGrowHeap = true;
release_sem_etc(sHeapGrowSem, 1, B_DO_NOT_RESCHEDULE);
}
if (result != NULL)
return result;
}
// try public memory, there might be something available
void *result = NULL;
int32 cpuCount = MIN(smp_get_num_cpus(),
(int32)sHeapCount / HEAP_CLASS_COUNT);
int32 cpuNumber = smp_get_current_cpu();
for (int32 i = 0; i < cpuCount; i++) {
uint32 heapIndex = heap_index_for(size, cpuNumber++ % cpuCount);
heap_allocator *heap = sHeaps[heapIndex];
result = heap_memalign(heap, alignment, size);
if (result != NULL)
return result;
}
// no memory available
if (thread_get_current_thread_id() == sHeapGrowThread)
panic("heap: all heaps have run out of memory while growing\n");
else
dprintf("heap: all heaps have run out of memory\n");
return NULL;
}
static status_t
heap_create_new_heap_area(heap_allocator *heap, const char *name, size_t size)
{
void *address = NULL;
area_id heapArea = create_area(name, &address,
B_ANY_KERNEL_BLOCK_ADDRESS, size, B_FULL_LOCK,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (heapArea < B_OK) {
TRACE(("heap: couldn't allocate heap area \"%s\"\n", name));
return heapArea;
}
heap_add_area(heap, heapArea, (addr_t)address, size);
#if PARANOID_HEAP_VALIDATION
heap_validate_heap(heap);
#endif
return B_OK;
}
static int32
heap_grow_thread(void *)
{
while (true) {
// wait for a request to grow the heap list
if (acquire_sem(sHeapGrowSem) < B_OK)
continue;
if (sAddGrowHeap) {
// the grow heap is going to run full soon, try to allocate a new
// one to make some room.
TRACE(("heap_grower: grow heaps will run out of memory soon\n"));
if (heap_create_new_heap_area(sGrowHeap, "additional grow heap",
HEAP_DEDICATED_GROW_SIZE) != B_OK)
dprintf("heap_grower: failed to create new grow heap area\n");
}
for (uint32 i = 0; i < sHeapCount; i++) {
heap_allocator *heap = sHeaps[i];
if (sLastGrowRequest[i] > sLastHandledGrowRequest[i]
|| heap_should_grow(heap)) {
// grow this heap if it is nearly full or if a grow was
// explicitly requested for this heap (happens when a large
// allocation cannot be fulfilled due to lack of contiguous
// pages)
if (heap_create_new_heap_area(heap, "additional heap",
HEAP_GROW_SIZE) != B_OK)
dprintf("heap_grower: failed to create new heap area\n");
sLastHandledGrowRequest[i] = sLastGrowRequest[i];
}
}
// notify anyone waiting for this request
release_sem_etc(sHeapGrownNotify, -1, B_RELEASE_ALL);
}
return 0;
}
#endif // USE_DEBUG_HEAP_FOR_MALLOC
static void
deferred_deleter(void *arg, int iteration)
{
// move entries and deletables to on-stack lists
InterruptsSpinLocker locker(sDeferredFreeListLock);
if (sDeferredFreeList.IsEmpty() && sDeferredDeletableList.IsEmpty())
return;
DeferredFreeList entries;
entries.MoveFrom(&sDeferredFreeList);
DeferredDeletableList deletables;
deletables.MoveFrom(&sDeferredDeletableList);
locker.Unlock();
// free the entries
while (DeferredFreeListEntry* entry = entries.RemoveHead())
free(entry);
// delete the deletables
while (DeferredDeletable* deletable = deletables.RemoveHead())
delete deletable;
}
// #pragma mark -
#if USE_DEBUG_HEAP_FOR_MALLOC
status_t
heap_init(addr_t base, size_t size)
{
for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
size_t partSize = size * sHeapClasses[i].initial_percentage / 100;
sHeaps[i] = heap_create_allocator(sHeapClasses[i].name, base, partSize,
&sHeapClasses[i], false);
sLastGrowRequest[i] = sLastHandledGrowRequest[i] = 0;
base += partSize;
sHeapCount++;
}
// set up some debug commands
add_debugger_command_etc("heap", &dump_heap_list,
"Dump infos about the kernel heap(s)",
"[(\"grow\" | \"stats\" | <heap>)]\n"
"Dump infos about the kernel heap(s). If \"grow\" is specified, only\n"
"infos about the dedicated grow heap are printed. If \"stats\" is\n"
"given as the argument, currently only the heap count is printed.\n"
"If <heap> is given, it is interpreted as the address of the heap to\n"
"print infos about.\n", 0);
#if !KERNEL_HEAP_LEAK_CHECK
add_debugger_command_etc("allocations", &dump_allocations,
"Dump current heap allocations",
"[\"stats\"] [<heap>]\n"
"If no parameters are given, all current alloactions are dumped.\n"
"If the optional argument \"stats\" is specified, only the allocation\n"
"counts and no individual allocations are printed\n"
"If a specific heap address is given, only allocations of this\n"
"allocator are dumped\n", 0);
#else // !KERNEL_HEAP_LEAK_CHECK
add_debugger_command_etc("allocations", &dump_allocations,
"Dump current heap allocations",
"[(\"team\" | \"thread\") <id>] [\"caller\" <address>] [\"address\" <address>] [\"stats\"]\n"
"If no parameters are given, all current alloactions are dumped.\n"
"If \"team\", \"thread\", \"caller\", and/or \"address\" is specified as the first\n"
"argument, only allocations matching the team ID, thread ID, caller\n"
"address or allocated address given in the second argument are printed.\n"
"If the optional argument \"stats\" is specified, only the allocation\n"
"counts and no individual allocations are printed.\n", 0);
add_debugger_command_etc("allocations_per_caller",
&dump_allocations_per_caller,
"Dump current heap allocations summed up per caller",
"[ \"-c\" ] [ -h <heap> ]\n"
"The current allocations will by summed up by caller (their count and\n"
"size) printed in decreasing order by size or, if \"-c\" is\n"
"specified, by allocation count. If given <heap> specifies the\n"
"address of the heap for which to print the allocations.\n", 0);
#endif // KERNEL_HEAP_LEAK_CHECK
return B_OK;
}
status_t
heap_init_post_area()
{
void *address = NULL;
area_id growHeapArea = create_area("dedicated grow heap", &address,
B_ANY_KERNEL_BLOCK_ADDRESS, HEAP_DEDICATED_GROW_SIZE, B_FULL_LOCK,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (growHeapArea < 0) {
panic("heap_init_post_area(): couldn't allocate dedicate grow heap "
"area");
return growHeapArea;
}
sGrowHeap = heap_create_allocator("grow", (addr_t)address,
HEAP_DEDICATED_GROW_SIZE, &sHeapClasses[0], false);
if (sGrowHeap == NULL) {
panic("heap_init_post_area(): failed to create dedicated grow heap\n");
return B_ERROR;
}
// create the VIP heap
static const heap_class heapClass = {
"VIP I/O", /* name */
100, /* initial percentage */
B_PAGE_SIZE / 8, /* max allocation size */
B_PAGE_SIZE, /* page size */
8, /* min bin size */
4, /* bin alignment */
8, /* min count per page */
16 /* max waste per page */
};
area_id vipHeapArea = create_area("VIP heap", &address,
B_ANY_KERNEL_ADDRESS, VIP_HEAP_SIZE, B_FULL_LOCK,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (vipHeapArea < 0) {
panic("heap_init_post_area(): couldn't allocate VIP heap area");
return B_ERROR;
}
sVIPHeap = heap_create_allocator("VIP heap", (addr_t)address,
VIP_HEAP_SIZE, &heapClass, false);
if (sVIPHeap == NULL) {
panic("heap_init_post_area(): failed to create VIP heap\n");
return B_ERROR;
}
dprintf("heap_init_post_area(): created VIP heap: %p\n", sVIPHeap);
return B_OK;
}
status_t
heap_init_post_sem()
{
sHeapGrowSem = create_sem(0, "heap_grow_sem");
if (sHeapGrowSem < 0) {
panic("heap_init_post_sem(): failed to create heap grow sem\n");
return B_ERROR;
}
sHeapGrownNotify = create_sem(0, "heap_grown_notify");
if (sHeapGrownNotify < 0) {
panic("heap_init_post_sem(): failed to create heap grown notify sem\n");
return B_ERROR;
}
return B_OK;
}
#endif // USE_DEBUG_HEAP_FOR_MALLOC
status_t
heap_init_post_thread()
{
#if USE_DEBUG_HEAP_FOR_MALLOC
sHeapGrowThread = spawn_kernel_thread(heap_grow_thread, "heap grower",
B_URGENT_PRIORITY, NULL);
if (sHeapGrowThread < 0) {
panic("heap_init_post_thread(): cannot create heap grow thread\n");
return sHeapGrowThread;
}
// create per-cpu heaps if there's enough memory
int32 heapCount = MIN(smp_get_num_cpus(),
(int32)vm_page_num_pages() / 60 / 1024);
for (int32 i = 1; i < heapCount; i++) {
addr_t base = 0;
size_t size = HEAP_GROW_SIZE * HEAP_CLASS_COUNT;
area_id perCPUHeapArea = create_area("per cpu initial heap",
(void **)&base, B_ANY_KERNEL_ADDRESS, size, B_FULL_LOCK,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (perCPUHeapArea < 0)
break;
for (uint32 j = 0; j < HEAP_CLASS_COUNT; j++) {
int32 heapIndex = i * HEAP_CLASS_COUNT + j;
size_t partSize = size * sHeapClasses[j].initial_percentage / 100;
sHeaps[heapIndex] = heap_create_allocator(sHeapClasses[j].name,
base, partSize, &sHeapClasses[j], false);
sLastGrowRequest[heapIndex] = 0;
sLastHandledGrowRequest[heapIndex] = 0;
base += partSize;
sHeapCount++;
}
}
resume_thread(sHeapGrowThread);
#else // USE_DEBUG_HEAP_FOR_MALLOC
// set up some debug commands
add_debugger_command_etc("heap", &dump_heap_list,
"Dump infos about a specific heap",
"[\"stats\"] <heap>\n"
"Dump infos about the specified kernel heap. If \"stats\" is given\n"
"as the argument, currently only the heap count is printed.\n", 0);
#if !KERNEL_HEAP_LEAK_CHECK
add_debugger_command_etc("heap_allocations", &dump_allocations,
"Dump current heap allocations",
"[\"stats\"] <heap>\n"
"If the optional argument \"stats\" is specified, only the allocation\n"
"counts and no individual allocations are printed.\n", 0);
#endif // KERNEL_HEAP_LEAK_CHECK
#endif // !USE_DEBUG_HEAP_FOR_MALLOC
// run the deferred deleter roughly once a second
if (register_kernel_daemon(deferred_deleter, NULL, 10) != B_OK)
panic("heap_init_post_thread(): failed to init deferred deleter");
return B_OK;
}
// #pragma mark - Public API
#if USE_DEBUG_HEAP_FOR_MALLOC
void *
memalign(size_t alignment, size_t size)
{
if (!gKernelStartup && !are_interrupts_enabled()) {
panic("memalign(): called with interrupts disabled\n");
return NULL;
}
if (!gKernelStartup && size > HEAP_AREA_USE_THRESHOLD) {
// don't even attempt such a huge allocation - use areas instead
size_t areaSize = ROUNDUP(size + sizeof(area_allocation_info)
+ alignment, B_PAGE_SIZE);
if (areaSize < size) {
// the size overflowed
return NULL;
}
void *address = NULL;
area_id allocationArea = create_area("memalign area", &address,
B_ANY_KERNEL_BLOCK_ADDRESS, areaSize, B_FULL_LOCK,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (allocationArea < B_OK) {
dprintf("heap: failed to create area for huge allocation\n");
return NULL;
}
area_allocation_info *info = (area_allocation_info *)address;
info->magic = kAreaAllocationMagic;
info->area = allocationArea;
info->base = address;
info->size = areaSize;
info->allocation_size = size;
info->allocation_alignment = alignment;
address = (void *)((addr_t)address + sizeof(area_allocation_info));
if (alignment != 0) {
address = (void *)ROUNDUP((addr_t)address, alignment);
ASSERT((addr_t)address % alignment == 0);
ASSERT((addr_t)address + size - 1 < (addr_t)info + areaSize - 1);
}
TRACE(("heap: allocated area %ld for huge allocation of %lu bytes\n",
allocationArea, size));
info->allocation_base = address;
#if PARANOID_KERNEL_MALLOC
memset(address, 0xcc, size);
#endif
return address;
}
void *result = NULL;
bool shouldGrow = false;
int32 cpuCount = MIN(smp_get_num_cpus(),
(int32)sHeapCount / HEAP_CLASS_COUNT);
int32 cpuNumber = smp_get_current_cpu();
for (int32 i = 0; i < cpuCount; i++) {
uint32 heapIndex = heap_index_for(size, cpuNumber++ % cpuCount);
heap_allocator *heap = sHeaps[heapIndex];
result = heap_memalign(heap, alignment, size);
if (result != NULL) {
shouldGrow = heap_should_grow(heap);
break;
}
#if PARANOID_HEAP_VALIDATION
heap_validate_heap(heap);
#endif
}
if (result == NULL) {
// request an urgent grow and wait - we don't do it ourselfs here to
// serialize growing through the grow thread, as otherwise multiple
// threads hitting this situation (likely when memory ran out) would
// all add areas
uint32 heapIndex = heap_index_for(size, smp_get_current_cpu());
sLastGrowRequest[heapIndex]++;
switch_sem(sHeapGrowSem, sHeapGrownNotify);
// and then try again
result = heap_memalign(sHeaps[heapIndex], alignment, size);
} else if (shouldGrow) {
// should grow sometime soon, notify the grower
release_sem_etc(sHeapGrowSem, 1, B_DO_NOT_RESCHEDULE);
}
if (result == NULL)
panic("heap: kernel heap has run out of memory\n");
return result;
}
void *
memalign_etc(size_t alignment, size_t size, uint32 flags)
{
if ((flags & HEAP_PRIORITY_VIP) != 0)
return heap_memalign(sVIPHeap, alignment, size);
if ((flags & (HEAP_DONT_WAIT_FOR_MEMORY | HEAP_DONT_LOCK_KERNEL_SPACE))
!= 0) {
return memalign_nogrow(alignment, size);
}
return memalign(alignment, size);
}
void
free_etc(void *address, uint32 flags)
{
if ((flags & HEAP_PRIORITY_VIP) != 0)
heap_free(sVIPHeap, address);
else
free(address);
}
void *
malloc(size_t size)
{
return memalign(0, size);
}
void
free(void *address)
{
if (!gKernelStartup && !are_interrupts_enabled()) {
panic("free(): called with interrupts disabled\n");
return;
}
int32 offset = smp_get_current_cpu() * HEAP_CLASS_COUNT;
for (uint32 i = 0; i < sHeapCount; i++) {
heap_allocator *heap = sHeaps[(i + offset) % sHeapCount];
if (heap_free(heap, address) == B_OK) {
#if PARANOID_HEAP_VALIDATION
heap_validate_heap(heap);
#endif
return;
}
}
// maybe it was allocated from the dedicated grow heap
if (heap_free(sGrowHeap, address) == B_OK)
return;
// or maybe it was allocated from the VIP heap
if (heap_free(sVIPHeap, address) == B_OK)
return;
// or maybe it was a huge allocation using an area
area_info areaInfo;
area_id area = area_for(address);
if (area >= B_OK && get_area_info(area, &areaInfo) == B_OK) {
area_allocation_info *info = (area_allocation_info *)areaInfo.address;
// just make extra sure it was allocated by us
if (info->magic == kAreaAllocationMagic && info->area == area
&& info->size == areaInfo.size && info->base == areaInfo.address
&& info->allocation_size < areaInfo.size) {
delete_area(area);
TRACE(("free(): freed huge allocation by deleting area %ld\n",
area));
return;
}
}
panic("free(): free failed for address %p\n", address);
}
void *
realloc(void *address, size_t newSize)
{
if (!gKernelStartup && !are_interrupts_enabled()) {
panic("realloc(): called with interrupts disabled\n");
return NULL;
}
if (address == NULL)
return memalign(0, newSize);
if (newSize == 0) {
free(address);
return NULL;
}
void *newAddress = NULL;
int32 offset = smp_get_current_cpu() * HEAP_CLASS_COUNT;
for (uint32 i = 0; i < sHeapCount; i++) {
heap_allocator *heap = sHeaps[(i + offset) % sHeapCount];
if (heap_realloc(heap, address, &newAddress, newSize) == B_OK) {
#if PARANOID_HEAP_VALIDATION
heap_validate_heap(heap);
#endif
return newAddress;
}
}
// maybe it was allocated from the dedicated grow heap
if (heap_realloc(sGrowHeap, address, &newAddress, newSize) == B_OK)
return newAddress;
// or maybe it was a huge allocation using an area
area_info areaInfo;
area_id area = area_for(address);
if (area >= B_OK && get_area_info(area, &areaInfo) == B_OK) {
area_allocation_info *info = (area_allocation_info *)areaInfo.address;
// just make extra sure it was allocated by us
if (info->magic == kAreaAllocationMagic && info->area == area
&& info->size == areaInfo.size && info->base == areaInfo.address
&& info->allocation_size < areaInfo.size) {
size_t available = info->size - ((addr_t)info->allocation_base
- (addr_t)info->base);
if (available >= newSize) {
// there is enough room available for the newSize
TRACE(("realloc(): new size %ld fits in old area %ld with %ld "
"available\n", newSize, area, available));
info->allocation_size = newSize;
return address;
}
// have to allocate/copy/free - TODO maybe resize the area instead?
newAddress = memalign(0, newSize);
if (newAddress == NULL) {
dprintf("realloc(): failed to allocate new block of %ld bytes\n",
newSize);
return NULL;
}
memcpy(newAddress, address, min_c(newSize, info->allocation_size));
delete_area(area);
TRACE(("realloc(): allocated new block %p for size %ld and deleted "
"old area %ld\n", newAddress, newSize, area));
return newAddress;
}
}
panic("realloc(): failed to realloc address %p to size %lu\n", address,
newSize);
return NULL;
}
#endif // USE_DEBUG_HEAP_FOR_MALLOC
void *
calloc(size_t numElements, size_t size)
{
void *address = memalign(0, numElements * size);
if (address != NULL)
memset(address, 0, numElements * size);
return address;
}
void
deferred_free(void *block)
{
if (block == NULL)
return;
DeferredFreeListEntry *entry = new(block) DeferredFreeListEntry;
InterruptsSpinLocker _(sDeferredFreeListLock);
sDeferredFreeList.Add(entry);
}
void *
malloc_referenced(size_t size)
{
int32 *referencedData = (int32 *)malloc(size + 4);
if (referencedData == NULL)
return NULL;
*referencedData = 1;
return referencedData + 1;
}
void *
malloc_referenced_acquire(void *data)
{
if (data != NULL) {
int32 *referencedData = (int32 *)data - 1;
atomic_add(referencedData, 1);
}
return data;
}
void
malloc_referenced_release(void *data)
{
if (data == NULL)
return;
int32 *referencedData = (int32 *)data - 1;
if (atomic_add(referencedData, -1) < 1)
free(referencedData);
}
DeferredDeletable::~DeferredDeletable()
{
}
void
deferred_delete(DeferredDeletable *deletable)
{
if (deletable == NULL)
return;
InterruptsSpinLocker _(sDeferredFreeListLock);
sDeferredDeletableList.Add(deletable);
}
↑ V575 The 'memalign' function processes '0' elements. Inspect the first argument.