#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "tm.h"
#include "tree.h"
#include "rtl.h"
#include "tm_p.h"
#include "toplev.h"
#include "varray.h"
#include "flags.h"
#include "ggc.h"
#include "timevar.h"
#include "params.h"
#include "bitmap.h"
#ifdef ENABLE_VALGRIND_CHECKING
# ifdef HAVE_VALGRIND_MEMCHECK_H
# include <valgrind/memcheck.h>
# elif defined HAVE_MEMCHECK_H
# include <memcheck.h>
# else
# include <valgrind.h>
# endif
#else
#define VALGRIND_DISCARD(x)
#define VALGRIND_MALLOCLIKE_BLOCK(w,x,y,z)
#define VALGRIND_FREELIKE_BLOCK(x,y)
#endif
#ifdef HAVE_MMAP_ANON
# undef HAVE_MMAP_DEV_ZERO
# include <sys/mman.h>
# ifndef MAP_FAILED
# define MAP_FAILED -1
# endif
# if !defined (MAP_ANONYMOUS) && defined (MAP_ANON)
# define MAP_ANONYMOUS MAP_ANON
# endif
# define USING_MMAP
#endif
#ifdef HAVE_MMAP_DEV_ZERO
# include <sys/mman.h>
# ifndef MAP_FAILED
# define MAP_FAILED -1
# endif
# define USING_MMAP
#endif
#ifndef USING_MMAP
#error "Zone collector requires mmap"
#endif
#if (GCC_VERSION < 3001)
#define prefetch(X) ((void) X)
#else
#define prefetch(X) __builtin_prefetch (X)
#endif
#define GGC_DEBUG_LEVEL (0)
#ifndef HOST_BITS_PER_PTR
#define HOST_BITS_PER_PTR HOST_BITS_PER_LONG
#endif
#ifdef COOKIE_CHECKING
#define CHUNK_MAGIC 0x95321123
#define DEADCHUNK_MAGIC 0x12817317
#endif
struct alloc_chunk {
#ifdef COOKIE_CHECKING
unsigned int magic;
#endif
unsigned int type:1;
unsigned int mark:1;
unsigned char large;
unsigned short size;
union {
struct alloc_chunk *next_free;
char data[1];
HOST_WIDEST_INT align_i;
#ifdef HAVE_LONG_DOUBLE
long double align_d;
#else
double align_d;
#endif
} u;
};
#define CHUNK_OVERHEAD (offsetof (struct alloc_chunk, u))
#define NUM_FREE_BINS 64
#define MAX_FREE_BIN_SIZE (64 * sizeof (void *))
#define FREE_BIN_DELTA (MAX_FREE_BIN_SIZE / NUM_FREE_BINS)
#define SIZE_BIN_UP(SIZE) (((SIZE) + FREE_BIN_DELTA - 1) / FREE_BIN_DELTA)
#define SIZE_BIN_DOWN(SIZE) ((SIZE) / FREE_BIN_DELTA)
#define LARGE_OBJECT_SIZE 0x7fff
struct max_alignment {
char c;
union {
HOST_WIDEST_INT i;
#ifdef HAVE_LONG_DOUBLE
long double d;
#else
double d;
#endif
} u;
};
#define MAX_ALIGNMENT (offsetof (struct max_alignment, u))
#define ROUND_UP_VALUE(x, f) ((f) - 1 - ((f) - 1 + (x)) % (f))
#define ROUND_UP(x, f) (CEIL (x, f) * (f))
typedef struct page_entry
{
struct page_entry *next;
size_t bytes;
size_t survived;
char *page;
unsigned short context_depth;
bool large_p;
struct alloc_zone *zone;
} page_entry;
static struct globals
{
struct alloc_zone *zones;
size_t pagesize;
size_t lg_pagesize;
#if defined (HAVE_MMAP_DEV_ZERO)
int dev_zero_fd;
#endif
FILE *debug_file;
} G;
struct alloc_zone
{
const char *name;
page_entry *pages;
struct alloc_chunk *free_chunks[NUM_FREE_BINS + 1];
size_t allocated;
size_t allocated_last_gc;
size_t bytes_mapped;
unsigned long context_depth_allocations;
unsigned long context_depth_collections;
unsigned short context_depth;
page_entry *free_pages;
struct alloc_zone *next_zone;
bool was_collected;
bool dead;
#ifdef GATHER_STATISTICS
struct
{
unsigned long long total_allocated;
unsigned long long total_overhead;
unsigned long long total_allocated_under32;
unsigned long long total_overhead_under32;
unsigned long long total_allocated_under64;
unsigned long long total_overhead_under64;
unsigned long long total_allocated_under128;
unsigned long long total_overhead_under128;
} stats;
#endif
} main_zone;
struct alloc_zone *rtl_zone;
struct alloc_zone *garbage_zone;
struct alloc_zone *tree_zone;
static int always_collect;
#define GGC_QUIRE_SIZE 16
static int ggc_allocated_p (const void *);
#ifdef USING_MMAP
static char *alloc_anon (char *, size_t, struct alloc_zone *);
#endif
static struct page_entry * alloc_small_page ( struct alloc_zone *);
static struct page_entry * alloc_large_page (size_t, struct alloc_zone *);
static void free_chunk (struct alloc_chunk *, size_t, struct alloc_zone *);
static void free_page (struct page_entry *);
static void release_pages (struct alloc_zone *);
static void sweep_pages (struct alloc_zone *);
static void * ggc_alloc_zone_1 (size_t, struct alloc_zone *, short MEM_STAT_DECL);
static bool ggc_collect_1 (struct alloc_zone *, bool);
static void check_cookies (void);
static inline int
ggc_allocated_p (const void *p)
{
struct alloc_chunk *chunk;
chunk = (struct alloc_chunk *) ((char *)p - CHUNK_OVERHEAD);
#ifdef COOKIE_CHECKING
gcc_assert (chunk->magic == CHUNK_MAGIC);
#endif
if (chunk->type == 1)
return true;
return false;
}
#ifdef USING_MMAP
static inline char *
alloc_anon (char *pref ATTRIBUTE_UNUSED, size_t size, struct alloc_zone *zone)
{
#ifdef HAVE_MMAP_ANON
char *page = (char *) mmap (pref, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
#endif
#ifdef HAVE_MMAP_DEV_ZERO
char *page = (char *) mmap (pref, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE, G.dev_zero_fd, 0);
#endif
VALGRIND_MALLOCLIKE_BLOCK(page, size, 0, 0);
if (page == (char *) MAP_FAILED)
{
perror ("virtual memory exhausted");
exit (FATAL_EXIT_CODE);
}
zone->bytes_mapped += size;
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (page, size));
return page;
}
#endif
static inline struct page_entry *
alloc_small_page (struct alloc_zone *zone)
{
struct page_entry *entry;
char *page;
page = NULL;
entry = zone->free_pages;
if (entry != NULL)
{
zone->free_pages = entry->next;
page = entry->page;
}
#ifdef USING_MMAP
else
{
struct page_entry *e, *f = zone->free_pages;
int i;
page = alloc_anon (NULL, G.pagesize * GGC_QUIRE_SIZE, zone);
for (i = GGC_QUIRE_SIZE - 1; i >= 1; i--)
{
e = (struct page_entry *) xmalloc (sizeof (struct page_entry));
e->bytes = G.pagesize;
e->page = page + (i << G.lg_pagesize);
e->next = f;
f = e;
}
zone->free_pages = f;
}
#endif
if (entry == NULL)
entry = (struct page_entry *) xmalloc (sizeof (struct page_entry));
entry->next = 0;
entry->bytes = G.pagesize;
entry->page = page;
entry->context_depth = zone->context_depth;
entry->large_p = false;
entry->zone = zone;
zone->context_depth_allocations |= (unsigned long)1 << zone->context_depth;
if (GGC_DEBUG_LEVEL >= 2)
fprintf (G.debug_file,
"Allocating %s page at %p, data %p-%p\n", entry->zone->name,
(PTR) entry, page, page + G.pagesize - 1);
return entry;
}
#define ROUND_UP(x, f) (CEIL (x, f) * (f))
static inline struct page_entry *
alloc_large_page (size_t size, struct alloc_zone *zone)
{
struct page_entry *entry;
char *page;
size = ROUND_UP (size, 1024);
page = (char *) xmalloc (size + CHUNK_OVERHEAD + sizeof (struct page_entry));
entry = (struct page_entry *) (page + size + CHUNK_OVERHEAD);
entry->next = 0;
entry->bytes = size;
entry->page = page;
entry->context_depth = zone->context_depth;
entry->large_p = true;
entry->zone = zone;
zone->context_depth_allocations |= (unsigned long)1 << zone->context_depth;
if (GGC_DEBUG_LEVEL >= 2)
fprintf (G.debug_file,
"Allocating %s large page at %p, data %p-%p\n", entry->zone->name,
(PTR) entry, page, page + size - 1);
return entry;
}
static inline void
free_page (page_entry *entry)
{
if (GGC_DEBUG_LEVEL >= 2)
fprintf (G.debug_file,
"Deallocating %s page at %p, data %p-%p\n", entry->zone->name, (PTR) entry,
entry->page, entry->page + entry->bytes - 1);
if (entry->large_p)
{
free (entry->page);
VALGRIND_FREELIKE_BLOCK (entry->page, entry->bytes);
}
else
{
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (entry->page, entry->bytes));
entry->next = entry->zone->free_pages;
entry->zone->free_pages = entry;
}
}
static void
release_pages (struct alloc_zone *zone)
{
#ifdef USING_MMAP
page_entry *p, *next;
char *start;
size_t len;
p = zone->free_pages;
while (p)
{
start = p->page;
next = p->next;
len = p->bytes;
free (p);
p = next;
while (p && p->page == start + len)
{
next = p->next;
len += p->bytes;
free (p);
p = next;
}
munmap (start, len);
zone->bytes_mapped -= len;
}
zone->free_pages = NULL;
#endif
}
static inline void
free_chunk (struct alloc_chunk *chunk, size_t size, struct alloc_zone *zone)
{
size_t bin = 0;
bin = SIZE_BIN_DOWN (size);
gcc_assert (bin);
if (bin > NUM_FREE_BINS)
bin = 0;
#ifdef COOKIE_CHECKING
gcc_assert (chunk->magic == CHUNK_MAGIC || chunk->magic == DEADCHUNK_MAGIC);
chunk->magic = DEADCHUNK_MAGIC;
#endif
chunk->u.next_free = zone->free_chunks[bin];
zone->free_chunks[bin] = chunk;
if (GGC_DEBUG_LEVEL >= 3)
fprintf (G.debug_file, "Deallocating object, chunk=%p\n", (void *)chunk);
VALGRIND_DISCARD (VALGRIND_MAKE_READABLE (chunk, sizeof (struct alloc_chunk)));
}
static void *
ggc_alloc_zone_1 (size_t orig_size, struct alloc_zone *zone,
short type ATTRIBUTE_UNUSED
MEM_STAT_DECL)
{
size_t bin = 0;
size_t lsize = 0;
struct page_entry *entry;
struct alloc_chunk *chunk, *lchunk, **pp;
void *result;
size_t size = orig_size;
if (size < FREE_BIN_DELTA)
size = FREE_BIN_DELTA;
size = (size + MAX_ALIGNMENT - 1) & -MAX_ALIGNMENT;
if (size >= G.pagesize - 2*CHUNK_OVERHEAD - FREE_BIN_DELTA)
{
size = ROUND_UP (size, 1024);
entry = alloc_large_page (size, zone);
entry->survived = 0;
entry->next = entry->zone->pages;
entry->zone->pages = entry;
chunk = (struct alloc_chunk *) entry->page;
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (chunk, sizeof (struct alloc_chunk)));
chunk->large = 1;
chunk->size = CEIL (size, 1024);
goto found;
}
bin = SIZE_BIN_UP (size);
if (bin <= NUM_FREE_BINS)
{
chunk = zone->free_chunks[bin];
if (chunk)
{
zone->free_chunks[bin] = chunk->u.next_free;
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (chunk, sizeof (struct alloc_chunk)));
goto found;
}
}
pp = &(zone->free_chunks[0]);
chunk = *pp;
while (chunk && chunk->size < size)
{
pp = &chunk->u.next_free;
chunk = *pp;
}
if (!chunk)
{
entry = alloc_small_page (zone);
entry->next = entry->zone->pages;
entry->zone->pages = entry;
chunk = (struct alloc_chunk *) entry->page;
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (chunk, sizeof (struct alloc_chunk)));
chunk->size = G.pagesize - CHUNK_OVERHEAD;
chunk->large = 0;
}
else
{
*pp = chunk->u.next_free;
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (chunk, sizeof (struct alloc_chunk)));
chunk->large = 0;
}
lsize = chunk->size - size;
if (lsize >= CHUNK_OVERHEAD + FREE_BIN_DELTA)
{
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (chunk, sizeof (struct alloc_chunk)));
chunk->size = size;
lsize -= CHUNK_OVERHEAD;
lchunk = (struct alloc_chunk *)(chunk->u.data + size);
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (lchunk, sizeof (struct alloc_chunk)));
#ifdef COOKIE_CHECKING
lchunk->magic = CHUNK_MAGIC;
#endif
lchunk->type = 0;
lchunk->mark = 0;
lchunk->size = lsize;
lchunk->large = 0;
free_chunk (lchunk, lsize, zone);
lsize = 0;
}
found:
#ifdef COOKIE_CHECKING
chunk->magic = CHUNK_MAGIC;
#endif
chunk->type = 1;
chunk->mark = 0;
result = chunk->u.data;
#ifdef ENABLE_GC_CHECKING
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (result, size));
memset (result, 0xaf, size);
#endif
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (result, size));
zone->allocated += size;
#ifdef GATHER_STATISTICS
ggc_record_overhead (orig_size, size + CHUNK_OVERHEAD - orig_size PASS_MEM_STAT);
{
size_t object_size = size + CHUNK_OVERHEAD;
size_t overhead = object_size - orig_size;
zone->stats.total_overhead += overhead;
zone->stats.total_allocated += object_size;
if (orig_size <= 32)
{
zone->stats.total_overhead_under32 += overhead;
zone->stats.total_allocated_under32 += object_size;
}
if (orig_size <= 64)
{
zone->stats.total_overhead_under64 += overhead;
zone->stats.total_allocated_under64 += object_size;
}
if (orig_size <= 128)
{
zone->stats.total_overhead_under128 += overhead;
zone->stats.total_allocated_under128 += object_size;
}
}
#endif
if (GGC_DEBUG_LEVEL >= 3)
fprintf (G.debug_file, "Allocating object, chunk=%p size=%lu at %p\n",
(void *)chunk, (unsigned long) size, result);
return result;
}
void *
ggc_alloc_typed_stat (enum gt_types_enum gte, size_t size
MEM_STAT_DECL)
{
switch (gte)
{
case gt_ggc_e_14lang_tree_node:
return ggc_alloc_zone_1 (size, tree_zone, gte PASS_MEM_STAT);
case gt_ggc_e_7rtx_def:
return ggc_alloc_zone_1 (size, rtl_zone, gte PASS_MEM_STAT);
case gt_ggc_e_9rtvec_def:
return ggc_alloc_zone_1 (size, rtl_zone, gte PASS_MEM_STAT);
default:
return ggc_alloc_zone_1 (size, &main_zone, gte PASS_MEM_STAT);
}
}
void *
ggc_alloc_stat (size_t size MEM_STAT_DECL)
{
return ggc_alloc_zone_1 (size, &main_zone, -1 PASS_MEM_STAT);
}
void *
ggc_alloc_zone_stat (size_t size, struct alloc_zone *zone MEM_STAT_DECL)
{
return ggc_alloc_zone_1 (size, zone, -1 PASS_MEM_STAT);
}
#ifdef ENABLE_GC_CHECKING
#define poison_chunk(CHUNK, SIZE) \
memset ((CHUNK)->u.data, 0xa5, (SIZE))
#else
#define poison_chunk(CHUNK, SIZE)
#endif
void
ggc_free (void *p)
{
struct alloc_chunk *chunk;
chunk = (struct alloc_chunk *) ((char *)p - CHUNK_OVERHEAD);
poison_chunk (chunk, ggc_get_size (p));
}
int
ggc_set_mark (const void *p)
{
struct alloc_chunk *chunk;
chunk = (struct alloc_chunk *) ((char *)p - CHUNK_OVERHEAD);
#ifdef COOKIE_CHECKING
gcc_assert (chunk->magic == CHUNK_MAGIC);
#endif
if (chunk->mark)
return 1;
chunk->mark = 1;
if (GGC_DEBUG_LEVEL >= 4)
fprintf (G.debug_file, "Marking %p\n", p);
return 0;
}
int
ggc_marked_p (const void *p)
{
struct alloc_chunk *chunk;
chunk = (struct alloc_chunk *) ((char *)p - CHUNK_OVERHEAD);
#ifdef COOKIE_CHECKING
gcc_assert (chunk->magic == CHUNK_MAGIC);
#endif
return chunk->mark;
}
size_t
ggc_get_size (const void *p)
{
struct alloc_chunk *chunk;
chunk = (struct alloc_chunk *) ((char *)p - CHUNK_OVERHEAD);
#ifdef COOKIE_CHECKING
gcc_assert (chunk->magic == CHUNK_MAGIC);
#endif
if (chunk->large)
return chunk->size * 1024;
return chunk->size;
}
void
init_ggc (void)
{
main_zone.name = "Main zone";
G.zones = &main_zone;
rtl_zone = new_ggc_zone ("RTL zone");
tree_zone = new_ggc_zone ("Tree zone");
garbage_zone = new_ggc_zone ("Garbage zone");
G.pagesize = getpagesize();
G.lg_pagesize = exact_log2 (G.pagesize);
#ifdef HAVE_MMAP_DEV_ZERO
G.dev_zero_fd = open ("/dev/zero", O_RDONLY);
gcc_assert (G.dev_zero_fd != -1);
#endif
#if 0
G.debug_file = fopen ("ggc-mmap.debug", "w");
setlinebuf (G.debug_file);
#else
G.debug_file = stdout;
#endif
#ifdef USING_MMAP
{
char *p = alloc_anon (NULL, G.pagesize, &main_zone);
struct page_entry *e;
if ((size_t)p & (G.pagesize - 1))
{
p = alloc_anon (NULL, G.pagesize, &main_zone);
gcc_assert (!((size_t)p & (G.pagesize - 1)));
}
e = (struct page_entry *) xmalloc (sizeof (struct page_entry));
e->bytes = G.pagesize;
e->page = p;
e->next = main_zone.free_pages;
main_zone.free_pages = e;
}
#endif
}
struct alloc_zone *
new_ggc_zone (const char * name)
{
struct alloc_zone *new_zone = xcalloc (1, sizeof (struct alloc_zone));
new_zone->name = name;
new_zone->next_zone = G.zones->next_zone;
G.zones->next_zone = new_zone;
return new_zone;
}
void
destroy_ggc_zone (struct alloc_zone * dead_zone)
{
struct alloc_zone *z;
for (z = G.zones; z && z->next_zone != dead_zone; z = z->next_zone)
continue;
gcc_assert (z);
z->dead= true;
}
void
ggc_push_context (void)
{
struct alloc_zone *zone;
for (zone = G.zones; zone; zone = zone->next_zone)
++(zone->context_depth);
gcc_assert (main_zone.context_depth < HOST_BITS_PER_LONG);
}
static void
ggc_pop_context_1 (struct alloc_zone *zone)
{
unsigned long omask;
unsigned depth;
page_entry *p;
depth = --(zone->context_depth);
omask = (unsigned long)1 << (depth + 1);
if (!((zone->context_depth_allocations | zone->context_depth_collections) & omask))
return;
zone->context_depth_allocations |= (zone->context_depth_allocations & omask) >> 1;
zone->context_depth_allocations &= omask - 1;
zone->context_depth_collections &= omask - 1;
for (p = zone->pages; p != NULL; p = p->next)
if (p->context_depth > depth)
p->context_depth = depth;
}
void
ggc_pop_context (void)
{
struct alloc_zone *zone;
for (zone = G.zones; zone; zone = zone->next_zone)
ggc_pop_context_1 (zone);
}
static void
sweep_pages (struct alloc_zone *zone)
{
page_entry **pp, *p, *next;
struct alloc_chunk *chunk, *last_free, *end;
size_t last_free_size, allocated = 0;
bool nomarksinpage;
memset (zone->free_chunks, 0, sizeof (zone->free_chunks));
pp = &zone->pages;
for (p = zone->pages; p ; p = next)
{
next = p->next;
if (p->large_p)
{
if (((struct alloc_chunk *)p->page)->mark == 1)
{
((struct alloc_chunk *)p->page)->mark = 0;
allocated += p->bytes - CHUNK_OVERHEAD;
pp = &p->next;
}
else
{
*pp = next;
#ifdef ENABLE_GC_CHECKING
memset (p->page, 0xb5, p->bytes);
#endif
free_page (p);
}
continue;
}
p->survived++;
chunk = (struct alloc_chunk *)p->page;
end = (struct alloc_chunk *)(p->page + G.pagesize);
last_free = NULL;
last_free_size = 0;
nomarksinpage = true;
do
{
prefetch ((struct alloc_chunk *)(chunk->u.data + chunk->size));
if (chunk->mark || p->context_depth < zone->context_depth)
{
nomarksinpage = false;
if (last_free)
{
last_free->type = 0;
last_free->size = last_free_size;
last_free->mark = 0;
poison_chunk (last_free, last_free_size);
free_chunk (last_free, last_free_size, zone);
last_free = NULL;
}
if (chunk->mark)
{
allocated += chunk->size;
}
chunk->mark = 0;
}
else
{
if (last_free)
{
last_free_size += CHUNK_OVERHEAD + chunk->size;
}
else
{
last_free = chunk;
last_free_size = chunk->size;
}
}
chunk = (struct alloc_chunk *)(chunk->u.data + chunk->size);
}
while (chunk < end);
if (nomarksinpage)
{
*pp = next;
#ifdef ENABLE_GC_CHECKING
memset (p->page, 0xb5, p->bytes);
#endif
free_page (p);
continue;
}
else if (last_free)
{
last_free->type = 0;
last_free->size = last_free_size;
last_free->mark = 0;
poison_chunk (last_free, last_free_size);
free_chunk (last_free, last_free_size, zone);
}
pp = &p->next;
}
zone->allocated = allocated;
}
static bool
ggc_collect_1 (struct alloc_zone *zone, bool need_marking)
{
if (!quiet_flag)
fprintf (stderr, " {%s GC %luk -> ",
zone->name, (unsigned long) zone->allocated / 1024);
zone->allocated = 0;
release_pages (zone);
zone->context_depth_collections
= ((unsigned long)1 << (zone->context_depth + 1)) - 1;
if (need_marking)
ggc_mark_roots ();
sweep_pages (zone);
zone->was_collected = true;
zone->allocated_last_gc = zone->allocated;
if (!quiet_flag)
fprintf (stderr, "%luk}", (unsigned long) zone->allocated / 1024);
return true;
}
static float
calculate_average_page_survival (struct alloc_zone *zone)
{
float count = 0.0;
float survival = 0.0;
page_entry *p;
for (p = zone->pages; p; p = p->next)
{
count += 1.0;
survival += p->survived;
}
return survival/count;
}
static inline void
check_cookies (void)
{
#ifdef COOKIE_CHECKING
page_entry *p;
struct alloc_zone *zone;
for (zone = G.zones; zone; zone = zone->next_zone)
{
for (p = zone->pages; p; p = p->next)
{
if (!p->large_p)
{
struct alloc_chunk *chunk = (struct alloc_chunk *)p->page;
struct alloc_chunk *end = (struct alloc_chunk *)(p->page + G.pagesize);
do
{
gcc_assert (chunk->magic == CHUNK_MAGIC
|| chunk->magic == DEADCHUNK_MAGIC);
chunk = (struct alloc_chunk *)(chunk->u.data + chunk->size);
}
while (chunk < end);
}
}
}
#endif
}
void
ggc_collect (void)
{
struct alloc_zone *zone;
bool marked = false;
float f;
timevar_push (TV_GC);
check_cookies ();
if (!always_collect)
{
float allocated_last_gc = 0, allocated = 0, min_expand;
for (zone = G.zones; zone; zone = zone->next_zone)
{
allocated_last_gc += zone->allocated_last_gc;
allocated += zone->allocated;
}
allocated_last_gc =
MAX (allocated_last_gc,
(size_t) PARAM_VALUE (GGC_MIN_HEAPSIZE) * 1024);
min_expand = allocated_last_gc * PARAM_VALUE (GGC_MIN_EXPAND) / 100;
if (allocated < allocated_last_gc + min_expand)
{
timevar_pop (TV_GC);
return;
}
}
main_zone.was_collected = false;
marked |= ggc_collect_1 (&main_zone, true);
if (main_zone.was_collected)
{
struct alloc_zone *zone;
for (zone = main_zone.next_zone; zone; zone = zone->next_zone)
{
check_cookies ();
zone->was_collected = false;
marked |= ggc_collect_1 (zone, !marked);
}
}
if (GGC_DEBUG_LEVEL >= 2)
{
for (zone = G.zones; zone; zone = zone->next_zone)
{
if (zone->was_collected)
{
f = calculate_average_page_survival (zone);
printf ("Average page survival in zone `%s' is %f\n",
zone->name, f);
}
}
}
if (marked)
{
page_entry *p;
for (zone = G.zones; zone; zone = zone->next_zone)
{
if (zone->was_collected)
continue;
for (p = zone->pages; p; p = p->next)
{
if (!p->large_p)
{
struct alloc_chunk *chunk = (struct alloc_chunk *)p->page;
struct alloc_chunk *end = (struct alloc_chunk *)(p->page + G.pagesize);
do
{
prefetch ((struct alloc_chunk *)(chunk->u.data + chunk->size));
if (chunk->mark || p->context_depth < zone->context_depth)
{
chunk->mark = 0;
}
chunk = (struct alloc_chunk *)(chunk->u.data + chunk->size);
}
while (chunk < end);
}
else
{
((struct alloc_chunk *)p->page)->mark = 0;
}
}
}
}
for (zone = G.zones; zone && zone->next_zone; zone = zone->next_zone)
{
if (zone->next_zone->dead)
{
struct alloc_zone *dead_zone = zone->next_zone;
printf ("Zone `%s' is dead and will be freed.\n", dead_zone->name);
gcc_assert (!dead_zone->allocated);
zone->next_zone = zone->next_zone->next_zone;
release_pages (dead_zone);
free (dead_zone);
}
}
timevar_pop (TV_GC);
}
#define SCALE(x) ((unsigned long) ((x) < 1024*10 \
? (x) \
: ((x) < 1024*1024*10 \
? (x) / 1024 \
: (x) / (1024*1024))))
#define LABEL(x) ((x) < 1024*10 ? ' ' : ((x) < 1024*1024*10 ? 'k' : 'M'))
void
ggc_print_statistics (void)
{
struct alloc_zone *zone;
struct ggc_statistics stats;
size_t total_overhead = 0, total_allocated = 0, total_bytes_mapped = 0;
memset (&stats, 0, sizeof (stats));
always_collect = 1;
ggc_print_common_statistics (stderr, &stats);
always_collect = 0;
for (zone = G.zones; zone; zone = zone->next_zone)
release_pages (zone);
fprintf (stderr,
"Memory still allocated at the end of the compilation process\n");
fprintf (stderr, "%20s %10s %10s %10s\n",
"Zone", "Allocated", "Used", "Overhead");
for (zone = G.zones; zone; zone = zone->next_zone)
{
page_entry *p;
size_t allocated;
size_t in_use;
size_t overhead;
if (!zone->pages)
continue;
overhead = allocated = in_use = 0;
for (p = zone->pages; p; p = p->next)
{
struct alloc_chunk *chunk;
allocated += p->bytes;
overhead += sizeof (page_entry);
if (p->large_p)
{
in_use += p->bytes - CHUNK_OVERHEAD;
chunk = (struct alloc_chunk *) p->page;
overhead += CHUNK_OVERHEAD;
gcc_assert (chunk->type && !chunk->mark);
continue;
}
for (chunk = (struct alloc_chunk *) p->page;
(char *) chunk < (char *) p->page + p->bytes;
chunk = (struct alloc_chunk *)(chunk->u.data + chunk->size))
{
overhead += CHUNK_OVERHEAD;
if (chunk->type)
in_use += chunk->size;
gcc_assert (!chunk->mark);
}
}
fprintf (stderr, "%20s %10lu%c %10lu%c %10lu%c\n",
zone->name,
SCALE (allocated), LABEL (allocated),
SCALE (in_use), LABEL (in_use),
SCALE (overhead), LABEL (overhead));
gcc_assert (in_use == zone->allocated);
total_overhead += overhead;
total_allocated += zone->allocated;
total_bytes_mapped += zone->bytes_mapped;
}
fprintf (stderr, "%20s %10lu%c %10lu%c %10lu%c\n", "Total",
SCALE (total_bytes_mapped), LABEL (total_bytes_mapped),
SCALE (total_allocated), LABEL(total_allocated),
SCALE (total_overhead), LABEL (total_overhead));
#ifdef GATHER_STATISTICS
{
unsigned long long all_overhead = 0, all_allocated = 0;
unsigned long long all_overhead_under32 = 0, all_allocated_under32 = 0;
unsigned long long all_overhead_under64 = 0, all_allocated_under64 = 0;
unsigned long long all_overhead_under128 = 0, all_allocated_under128 = 0;
fprintf (stderr, "\nTotal allocations and overheads during the compilation process\n");
for (zone = G.zones; zone; zone = zone->next_zone)
{
all_overhead += zone->stats.total_overhead;
all_allocated += zone->stats.total_allocated;
all_allocated_under32 += zone->stats.total_allocated_under32;
all_overhead_under32 += zone->stats.total_overhead_under32;
all_allocated_under64 += zone->stats.total_allocated_under64;
all_overhead_under64 += zone->stats.total_overhead_under64;
all_allocated_under128 += zone->stats.total_allocated_under128;
all_overhead_under128 += zone->stats.total_overhead_under128;
fprintf (stderr, "%20s: %10lld\n",
zone->name, zone->stats.total_allocated);
}
fprintf (stderr, "\n");
fprintf (stderr, "Total Overhead: %10lld\n",
all_overhead);
fprintf (stderr, "Total Allocated: %10lld\n",
all_allocated);
fprintf (stderr, "Total Overhead under 32B: %10lld\n",
all_overhead_under32);
fprintf (stderr, "Total Allocated under 32B: %10lld\n",
all_allocated_under32);
fprintf (stderr, "Total Overhead under 64B: %10lld\n",
all_overhead_under64);
fprintf (stderr, "Total Allocated under 64B: %10lld\n",
all_allocated_under64);
fprintf (stderr, "Total Overhead under 128B: %10lld\n",
all_overhead_under128);
fprintf (stderr, "Total Allocated under 128B: %10lld\n",
all_allocated_under128);
}
#endif
}
struct ggc_pch_data
{
struct ggc_pch_ondisk
{
unsigned total;
} d;
size_t base;
size_t written;
};
struct ggc_pch_data *
init_ggc_pch (void)
{
return xcalloc (sizeof (struct ggc_pch_data), 1);
}
void
ggc_pch_count_object (struct ggc_pch_data *d, void *x ATTRIBUTE_UNUSED,
size_t size, bool is_string)
{
if (!is_string)
{
d->d.total += size + CHUNK_OVERHEAD;
}
else
d->d.total += size;
}
size_t
ggc_pch_total_size (struct ggc_pch_data *d)
{
return d->d.total;
}
void
ggc_pch_this_base (struct ggc_pch_data *d, void *base)
{
d->base = (size_t) base;
}
char *
ggc_pch_alloc_object (struct ggc_pch_data *d, void *x,
size_t size, bool is_string)
{
char *result;
result = (char *)d->base;
if (!is_string)
{
struct alloc_chunk *chunk = (struct alloc_chunk *) ((char *)x - CHUNK_OVERHEAD);
if (chunk->large)
d->base += ggc_get_size (x) + CHUNK_OVERHEAD;
else
d->base += chunk->size + CHUNK_OVERHEAD;
return result + CHUNK_OVERHEAD;
}
else
{
d->base += size;
return result;
}
}
void
ggc_pch_prepare_write (struct ggc_pch_data *d ATTRIBUTE_UNUSED,
FILE *f ATTRIBUTE_UNUSED)
{
}
void
ggc_pch_write_object (struct ggc_pch_data *d ATTRIBUTE_UNUSED,
FILE *f, void *x, void *newx ATTRIBUTE_UNUSED,
size_t size, bool is_string)
{
if (!is_string)
{
struct alloc_chunk *chunk = (struct alloc_chunk *) ((char *)x - CHUNK_OVERHEAD);
size = ggc_get_size (x);
if (fwrite (chunk, size + CHUNK_OVERHEAD, 1, f) != 1)
fatal_error ("can't write PCH file: %m");
d->written += size + CHUNK_OVERHEAD;
}
else
{
if (fwrite (x, size, 1, f) != 1)
fatal_error ("can't write PCH file: %m");
d->written += size;
}
}
void
ggc_pch_finish (struct ggc_pch_data *d, FILE *f)
{
if (fwrite (&d->d, sizeof (d->d), 1, f) != 1)
fatal_error ("can't write PCH file: %m");
free (d);
}
void
ggc_pch_read (FILE *f, void *addr)
{
struct ggc_pch_ondisk d;
struct page_entry *entry;
struct alloc_zone *pch_zone;
if (fread (&d, sizeof (d), 1, f) != 1)
fatal_error ("can't read PCH file: %m");
entry = xcalloc (1, sizeof (struct page_entry));
entry->bytes = d.total;
entry->page = addr;
entry->context_depth = 0;
pch_zone = new_ggc_zone ("PCH zone");
entry->zone = pch_zone;
entry->next = entry->zone->pages;
entry->zone->pages = entry;
}