common-lisp/sbcl.sbcl
1
0
Fork 0
mirror of git://git.code.sf.net/p/sbcl/sbcl synced 2026-09-10 07:26:40 -04:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Add files for mark-region GC

Browse source 
Separating these out makes reasoning about whether the diffs to pre-existing
files applied to the baseline without mark-region (in a subsequent patch)
could in any way adversely affect the production-ready gencgc.
This change itself builds nothing new.
This commit is contained in:
Douglas Katzman 2023-08-09 21:24:31 -04:00
parent 57293154bd
commit 3c99f4ab5d
10 changed files with 4942 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
src/runtime/gc-thread-pool.c Normal file
View file

@ -0,0 +1,65 @@
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdlib.h>
#include <semaphore.h>
#include "interr.h"
static unsigned int gc_threads;
static pthread_t *threads;
static os_sem_t start_semaphore;
static os_sem_t join_semaphore;
static int semaphores_initializedp;
static void (*action)(void);
static void *worker(void *nothing) {
(void)nothing;
while (1) {
os_sem_wait(&start_semaphore);
action();
os_sem_post(&join_semaphore);
}
return NULL;
}
void thread_pool_init() {
if (!semaphores_initializedp) {
os_sem_init(&start_semaphore, 0);
os_sem_init(&join_semaphore, 0);
semaphores_initializedp = 1;
}
char *str = getenv("GC_THREADS"), *tail;
if (str == NULL) {
gc_threads = 3;
} else {
unsigned long parse = strtoul(str, &tail, 10);
if (tail == str || parse >= 256) lose("%s isn't a number of GC threads", str);
gc_threads = parse;
}
threads = successful_malloc(sizeof(pthread_t) * gc_threads);
for (unsigned int i = 0; i < gc_threads; i++)
if (pthread_create(threads + i, NULL, worker, NULL))
lose("Failed to create GC thread #%d", i);
else {
#ifdef LISP_FEATURE_LINUX
pthread_setname_np(threads[i], "Parallel GC");
#endif
}
}
static void wake_gc_threads() {
for (unsigned int i = 0; i < gc_threads; i++) os_sem_post(&start_semaphore);
}
static void join_gc_threads() {
for (unsigned int i = 0; i < gc_threads; i++) os_sem_wait(&join_semaphore);
}
void run_on_thread_pool(void (*act)(void)) {
action = act;
wake_gc_threads();
act();
join_gc_threads();
}

7
src/runtime/gc-thread-pool.h Normal file
View file

@ -0,0 +1,7 @@
#ifndef GC_THREAD_POOL_H
#define GC_THREAD_POOL_H
extern void thread_pool_init();
extern void run_on_thread_pool(void (*act)(void));
#endif

292
src/runtime/incremental-compact.c Normal file
View file

@ -0,0 +1,292 @@
#include <stdio.h>
#include <sys/time.h>
#include "os.h"
#include "gc.h"
#include "code.h"
#include "lispobj.h"
#include "gc-assert.h"
#include "queue.h"
#include "tiny-lock.h"
#include "queue-suballocator.h"
#include "incremental-compact.h"
#include "genesis/static-symbols.h"
#include "genesis/fdefn.h"
#include "genesis/closure.h"
#include "genesis/gc-tables.h"
#include "genesis/symbol.h"
/* The fix_slots loop does less work per pointer, so we
* prefetch further than we do in the tracing loop. */
#define FIX_PREFETCH_DISTANCE 128
/* Duplicated from mark-region.c */
#define METER(name, action) \
{ uword_t before = get_time(); \
action; \
atomic_fetch_add(name, get_time() - before); }
static uword_t get_time() {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
return t.tv_sec * 1000000 + t.tv_nsec/1000;
}
/* Maximum ratio between pages used and pages "needed" to compact. */
float page_overhead_threshold = 1.3;
/* Minimum fraction of bytes used on a page to compact it. */
float page_utilisation_threshold = 0.5;
/* Maximum number of bytes to copy in one collection. */
uword_t bytes_to_copy = 30000000;
/* Minimum generation to consider compacting when collecting. */
generation_index_t minimum_compact_gen = 1;
/* To force compacting GC or not. Set by prepare_dynamic_space_for_final_gc. */
bool force_compaction = 0;
static generation_index_t target_generation;
/* A queue of interesting slots. */
static struct Qblock *remset;
static lock_t remset_lock = LOCK_INITIALIZER;
static struct suballocator remset_suballocator = SUBALLOCATOR_INITIALIZER("compaction remset");
bool compacting;
unsigned char *target_pages;
void compactor_init() {
target_pages = calloc(dynamic_space_size / GENCGC_PAGE_BYTES, 1);
if (!target_pages)
lose("Failed to allocate target pages table");
}
/* Deciding how to compact */
static bool should_compact(char __attribute__((unused)) *why) {
/* If there are many more small-object pages than there could
* be, start compacting. */
uword_t pages = 0, bytes = 0;
for (page_index_t p = 0; p < page_table_pages; p++) {
if (!page_free_p(p) && !page_single_obj_p(p)) {
pages++;
bytes += page_bytes_used(p);
}
}
float ratio = (float)(pages * GENCGC_PAGE_BYTES) / bytes;
// fprintf(stderr, "%s, ratio = %.2f\n", why, ratio);
return force_compaction || ratio > page_overhead_threshold;
}
static void pick_targets() {
uword_t bytes_moving = 0, pages_moving = 0;
page_index_t p = page_table_pages - 1;
/* Ideally we'd like to avoid selecting pinned pages here, but the phase
* ordering is tricky. We need to know (an over-estimation of) the pages
* to move for mark() to log, we only find out which pages are pinned
* while marking from the roots. */
while (bytes_moving < bytes_to_copy) {
if (!page_single_obj_p(p) &&
!page_free_p(p) &&
page_table[p].gen < PSEUDO_STATIC_GENERATION &&
(float)(page_bytes_used(p)) / GENCGC_PAGE_BYTES <= page_utilisation_threshold) {
bytes_moving += page_bytes_used(p);
pages_moving++;
target_pages[p] = 1;
}
if (p == 0) break;
p--;
}
}
void consider_compaction(generation_index_t gen) {
if (gen >= minimum_compact_gen && should_compact("Enabling remset")) {
compacting = 1;
target_generation = gen;
pick_targets();
} else {
compacting = 0;
}
}
/* Remset */
/* We cram a source into the low bits of a pointer, to save space in the
* remset. */
static inline lispobj tag_source(lispobj *where, enum source s) { return (lispobj)where | (lispobj)s; }
static inline enum source source_from_tagged(lispobj t) { return t & 7; }
static inline lispobj *slot_from_tagged(lispobj t) { return (lispobj*)(t &~ 7); }
/* Each tracing thread records sources into thread-local blocks, like they
* do with grey objects. */
static _Thread_local struct Qblock *remset_block = NULL;
void commit_thread_local_remset() {
if (remset_block && remset_block->count) {
acquire_lock(&remset_lock);
remset_block->next = remset;
remset = remset_block;
remset_block = NULL;
release_lock(&remset_lock);
}
}
/* We need to know which object a slot resides in for two reasons:
* - If the object has been moved, we need to adjust the location of
* the slot to update accordingly. We assume the slot does not move
* w.r.t the object, which seems fine to do.
* - We need to inform some hash tables that they need to be rehashed,
* if we just invalidated some address-based hash.
* We could figure out the start of an object from its slot, but I'd
* prefer not to.
*/
void log_relevant_slot(lispobj *slot, lispobj *source, enum source source_type) {
if (!remset_block || remset_block->count == QBLOCK_CAPACITY) {
commit_thread_local_remset();
remset_block = suballoc_allocate(&remset_suballocator);
}
remset_block->elements[remset_block->count] = tag_source(slot, source_type);
remset_block->elements[remset_block->count+1] = (lispobj)source;
remset_block->count += 2;
}
/* Compacting */
static void apply_pins() {
/* Now that we know which pages are pinned, we should update the targeted
* pages for compaction. */
for (page_index_t p = 0; p < page_table_pages; p++)
target_pages[p] &= !gc_page_pins[p];
}
static void move_objects() {
/* Note that this function is very un-thread-safe; list linearisation
* can cause a thread to copy any objects. But if it weren't for list
* linearisation, no synchronisation between threads mightn't be needed, as
* each thread would be confined to the pages it claimed.
* But early experiements in parallel copying suggested we're bottlenecked
* by refilling TLABs too. */
uword_t pages_moved = 0;
for (page_index_t p = 0; p < page_table_pages; p++)
// What's wrong with testing page_single_obj_p in pick_targets?
if (target_pages[p] && !page_single_obj_p(p)) {
pages_moved++;
lispobj *end = (lispobj*)page_address(p + 1);
/* Move every object in this page in the right generation. */
for (lispobj *where = next_object((lispobj*)page_address(p), 0, end);
where;
/* We install forwarding pointers, so we'll conservatively assume
* the object is at least two words large. */
where = next_object(where, 2, end)) {
if (gc_gen_of((lispobj)where, -1) == target_generation) {
lispobj bogus = compute_lispobj(where);
scavenge(&bogus, 1);
}
}
/* Free all lines we just copied from. */
uword_t decrement = 0;
char *allocation = (char*)allocation_bitmap;
for_lines_in_page (l, p)
if (DECODE_GEN(line_bytemap[l]) == target_generation) {
line_bytemap[l] = 0;
allocation[l] = 0;
decrement++;
}
set_page_bytes_used(p, page_bytes_used(p) - LINE_SIZE * decrement);
generations[target_generation].bytes_allocated -= LINE_SIZE * decrement;
bytes_allocated -= LINE_SIZE * decrement;
if (page_words_used(p) == 0) {
page_table[p].type = FREE_PAGE_FLAG;
page_table[p].scan_start_offset_ = 0;
}
}
if (force_compaction) fprintf(stderr, "Forced compaction moved %ld pages\n", pages_moved);
}
/* Fix up the address of a slot, when the object containing the slot
* may have been forwarded. */
static inline lispobj *forward_slot(lispobj *slot, lispobj *source) {
lispobj *forwarded_source = native_pointer(follow_fp((lispobj)source));
return (lispobj*)((lispobj)forwarded_source + ((lispobj)slot - (lispobj)source));
}
static void fix_slot(lispobj *slot, lispobj *source, enum source source_type) {
/* TLS may not be moved, and isn't really an object.
* mr_preserve_range eventually calls fix_slot with
* source == NULL to indicate that the source cannot move.
* And remember that NIL has a funny alignment, which
* we shouldn't "fix". Nor does NIL move. */
if (source && source != native_pointer(NIL) - 1) {
slot = forward_slot(slot, source);
source = native_pointer(follow_fp((lispobj)source));
}
switch (source_type) {
case SOURCE_NORMAL:
*slot = follow_maybe_fp(*slot);
if (source &&
widetag_of(source) == SIMPLE_VECTOR_WIDETAG &&
vector_flagp(*source, VectorHashing)) {
/* Tell any hash tables to rehash. This causes unnecessary rehashing,
* but we compact infrequently and incrementally, so it shouldn't
* hurt much. */
/* TODO: should "rehash a hash table" be part of the source type? */
struct vector* kv_vector = (struct vector*)source;
KV_PAIRS_REHASH(kv_vector->data) |= make_fixnum(1);
}
break;
case SOURCE_ZERO_TAG:
/* follow_maybe_fp doesn't care what the tag is, just that it
* satisfies is_lisp_pointer. */
*slot = (lispobj)native_pointer(follow_maybe_fp(*slot | INSTANCE_POINTER_LOWTAG));
break;
case SOURCE_CLOSURE: {
/* SOURCE_CLOSURE can only be the source type of the taggedptr of
* a closure. */
struct closure *closure = (struct closure*)source;
closure->fun = fun_self_from_taggedptr(follow_fp(fun_taggedptr_from_self(closure->fun)));
break;
}
case SOURCE_SYMBOL_NAME: {
/* SOURCE_SYMBOL_NAME can only be the source type of the s->name
* slot. */
struct symbol *s = (struct symbol*)source;
set_symbol_name(s, follow_fp(decode_symbol_name(s->name)));
break;
}
case SOURCE_FDEFN_RAW: {
/* SOURCE_FDEFN_RAW can only be the source type of the fdefn->raw_addr
* slot. */
struct fdefn *f = (struct fdefn*)source;
lispobj obj = decode_fdefn_rawfun(f);
f->raw_addr += (sword_t)(follow_fp(obj) - obj);
}
}
}
static void fix_slots() {
int c = 0;
for (; remset; remset = remset->next) {
for (int n = 0; n < remset->count; n += 2) {
lispobj *slot = slot_from_tagged(remset->elements[n]),
*source = (lispobj*)remset->elements[n + 1];
enum source source_type = source_from_tagged(remset->elements[n]);
fix_slot(slot, source, source_type);
if (n + FIX_PREFETCH_DISTANCE < remset->count)
__builtin_prefetch(slot_from_tagged(remset->elements[n]));
c++;
}
}
}
void run_compaction(_Atomic(uword_t) *copy_meter, _Atomic(uword_t) *fix_meter) {
if (compacting) {
/* Check again, in case fragmentation somehow improves.
* Not likely, but it's a cheap test which avoids effort. */
if (should_compact("Performing compaction")) {
apply_pins();
METER(copy_meter, move_objects());
gc_close_collector_regions(0);
METER(fix_meter, fix_slots());
should_compact("I just moved, but still");
}
memset(target_pages, 0, page_table_pages);
remset = NULL;
suballoc_release(&remset_suballocator);
}
compacting = 0;
}

34
src/runtime/incremental-compact.h Normal file
View file

@ -0,0 +1,34 @@
#ifdef LISP_FEATURE_MARK_REGION_GC
#ifndef INCREMENTAL_COMPACT_H
#define INCREMENTAL_COMPACT_H
#include "os.h"
#include "lispobj.h"
/* Compactor tuning */
extern bool force_compaction;
extern float page_overhead_threshold, page_utilisation_threshold;
extern generation_index_t minimum_compact_gen;
extern uword_t bytes_to_copy;
/* Entry-points for compactor */
extern bool compacting;
extern void compactor_init();
extern void consider_compaction(generation_index_t gen);
extern void run_compaction(_Atomic(uword_t) *copy_meter, _Atomic(uword_t) *fix_meter);
/* Tracing/logging interface */
extern unsigned char *target_pages;
extern void log_relevant_slot(lispobj *where, lispobj *source_object, enum source source_type);
extern unsigned char *target_pages;
/* Avoid a full call unless the slot is relevant. */
static inline void log_slot(lispobj target, lispobj *where,
lispobj *source_object, enum source source_type) {
if (!compacting) return;
page_index_t p = find_page_index(native_pointer(target));
if (p != -1 && target_pages[p])
log_relevant_slot(where, source_object, source_type);
}
extern void commit_thread_local_remset();
#endif
#endif

1368
src/runtime/mark-region.c Normal file
View file

File diff suppressed because it is too large Load diff

84
src/runtime/mark-region.h Normal file
View file

@ -0,0 +1,84 @@
#ifndef MARK_REGION_H
#define MARK_REGION_H
#ifdef LISP_FEATURE_MARK_REGION_GC
#include "lispobj.h"
#include "core.h"
/* Set line size so that every line byte corresponds to one mark
* bitmap byte. */
#define LINE_SIZE (8 << N_LOWTAG_BITS)
extern uword_t *allocation_bitmap;
extern _Atomic(uword_t) *mark_bitmap;
extern unsigned char *line_bytemap;
typedef intptr_t line_index_t;
/* Line arithmetic */
static inline char *line_address(line_index_t line) {
return (char*)(DYNAMIC_SPACE_START + (line * LINE_SIZE));
}
static inline line_index_t address_line(void *address) {
return ((uintptr_t)address - DYNAMIC_SPACE_START) / LINE_SIZE;
}
#define for_lines_in_page(l, p) \
for (line_index_t l = address_line(page_address(p)), limit = address_line(page_address(p + 1)); \
l < limit; l++)
/* Line metadata */
/* Two highest bits are unused, then a bit for "fresh" lines which need
* the allocation bitmap to be materialised, then a bit for line
* marking, then four lowest bits hold (generation + 1). A 0 means a
* free line. */
#define MARK_GEN(l) ((l) | 16)
#define FRESHEN_GEN(l) ((l) | 32)
#define UNMARK_GEN(l) ((l) & 15)
#define UNFRESHEN_GEN(l) ((l) & 31)
#define ENCODE_GEN(g) ((g) + 1)
#define DECODE_GEN(l) (UNMARK_GEN(l) - 1)
#define IS_MARKED(l) ((l) & 16)
#define IS_FRESH(l) ((l) & 32)
#define COPY_MARK(from, to) (((from) & 0x30) | (to))
/* All the GC functions. */
/* I/O */
extern void mrgc_init();
extern void load_corefile_bitmaps(int fd, core_entry_elt_t n_ptes);
extern sword_t bitmap_sizes(core_entry_elt_t n_ptes);
/* Allocation */
extern bool try_allocate_small_from_pages(sword_t nbytes, struct alloc_region *region,
int page_type, generation_index_t gen,
page_index_t *start, page_index_t end);
extern bool try_allocate_small_after_region(sword_t nbytes,
struct alloc_region *region);
extern page_index_t try_allocate_large(uword_t nbytes,
int page_type, generation_index_t gen,
page_index_t *start, page_index_t end,
uword_t *largest_hole);
extern void mr_update_closed_region(struct alloc_region *region, generation_index_t gen);
/* Bitmaps */
extern bool allocation_bit_marked(void *pointer);
extern void set_allocation_bit_mark(void *pointer);
extern bool line_marked(void *pointer);
extern lispobj *search_dynamic_space(void *pointer);
/* Liveness */
extern void mr_preserve_ambiguous(uword_t address);
extern void mr_preserve_range(lispobj *from, sword_t nwords);
extern void mr_preserve_leaf(lispobj obj);
extern void mr_preserve_object(lispobj obj);
extern void mr_trace_object(lispobj *obj);
extern void mr_trace_bump_range(lispobj* start, lispobj *end);
//extern bool pointer_survived_gc_yet(lispobj object);
/* Running the GC */
extern void mr_pre_gc(generation_index_t generation);
extern void mr_collect_garbage(bool raise);
extern void zero_all_free_ranges();
extern void prepare_lines_for_final_gc();
#endif
#endif

873
src/runtime/pmrgc-impl.h Normal file
View file

@ -0,0 +1,873 @@
/*
* This software is part of the SBCL system. See the README file for
* more information.
*
* This software is derived from the CMU CL system, which was
* written at Carnegie Mellon University and released into the
* public domain. The software is in the public domain and is
* provided with absolutely no warranty. See the COPYING and CREDITS
* files for more information.
*/
#ifndef GENCGC_IS_PRECISE
#error "GENCGC_IS_PRECISE must be #defined as 0 or 1"
#endif
/* One bit of page_words_t is the need_zerofill flag.
* That leaves 15 bits to store page_words_used. This can represent
* a page size of up to 64KiB on 32-bit and 128KiB on 64-bit.
* Note that since the allocation quantum is actually 2 words
* the words_used is always an even number, and so technically
* we could store as "dualwords used" to achieve double the range */
#define GENCGC_PAGE_WORDS (GENCGC_PAGE_BYTES/N_WORD_BYTES)
#if GENCGC_PAGE_WORDS > 32767
# error "GENCGC_PAGE_WORDS unexpectedly large."
#endif
#include "gencgc-alloc-region.h"
#include "genesis/cardmarks.h"
#include "genesis/cons.h"
#include "genesis/weak-pointer.h"
#include "globals.h" // for DYNAMIC_SPACE_START
#include "hopscotch.h"
#include <limits.h> // for UINT_MAX
#if N_WORD_BITS == 64
// It's more economical to store scan_start_offset using 4 bytes than 8.
// Doing so makes struct page fit in 8 bytes if words_used takes 2 bytes.
// scan_start_offset = 4
// words_used = 2
// flags = 1
// gen = 1
// If words_used takes 4 bytes, then the above is 10 bytes which is padded to
// 12, which is still an improvement over the 16 that it would have been.
# define CONDENSED_PAGE_TABLE 1
#else
# define CONDENSED_PAGE_TABLE 0
#endif
typedef unsigned short page_words_t;
/* Note that this structure is also used from Lisp-side in
* src/code/room.lisp, and the Lisp-side structure layout is currently
* not groveled from C code but hardcoded. Any changes to the
* structure layout need to be also made there.
*
* FIXME: We should probably just define this structure in Lisp, and
* output the C version in genesis. -- JES, 2006-12-30.
*/
struct page {
/* This is the offset from the first byte of some object in memory
* prior to and no closer than the start of the page to the start
* of the page. Lower values here are better, 0 is ideal. This
* is useful for determining where to start when scanning forward
* through a heap page (either for conservative root validation or
* for scavenging). MUST be 0 for unallocated pages.
*/
#if CONDENSED_PAGE_TABLE
// The low bit of the offset indicates the scale factor:
// 0 = double-lispwords, 1 = gc cards. Large objects are card-aligned,
// and this representation allows for a 32TB contiguous block using 32K
// card size. Larger allocations will have pages that can't directly
// store the full offset. That has to be dealt with by the accessor.
unsigned int scan_start_offset_;
#else
os_vm_size_t scan_start_offset_;
#endif
/* the number of lispwords of this page that are used. This may be less
* than the usage at an instant in time for pages within the current
* allocation regions. The 0th bit of the physical uint16 indicates
* that the page needs to be zero-filled for the next use.
* Let the C compiler figure it out, so we can't get it wrong in C.
* But we need to reverse the order of the packed fields depending on
* endianness so that the Lisp side is easier to understand */
#ifdef LISP_FEATURE_BIG_ENDIAN
page_words_t words_used_ : 15;
page_words_t need_zerofill : 1;
#else
page_words_t need_zerofill : 1;
page_words_t words_used_ : 15;
#endif
// !!! If bit positions are changed, be sure to reflect the changes into
// page_extensible_p() as well as ALLOCATION-INFORMATION in sb-introspect
// and WALK-DYNAMIC-SPACE.
/*
* The 4 low bits of 'type' are defined by PAGE_TYPE_x constants.
* 0000 free
* ?001 strictly boxed data (pointers, immediates, object headers)
* ?010 strictly unboxed data
* ?011 mixed boxed/unboxed non-code objects
* ?111 code
* The next two bits are SINGLE_OBJECT and OPEN_REGION.
* The top two can be used for segregating objects by widetag
* which will important once we have "destructors" to run for a
* for a category of object, such as SYMBOL, hypothetically for
* recycling TLS indices or something like that. */
unsigned char type;
/* the generation that this page belongs to. This should be valid
* for all pages that may have objects allocated, even current
* allocation region pages - this allows the space of an object to
* be easily determined. */
generation_index_t gen;
};
extern struct page *page_table;
extern page_index_t page_table_pages;
/* Find the page index within the page_table for the given
* address. Return -1 on failure. */
static inline page_index_t find_page_index(void *addr)
{
if (addr >= (void*)DYNAMIC_SPACE_START) {
page_index_t index = ((uintptr_t)addr -
(uintptr_t)DYNAMIC_SPACE_START) / GENCGC_PAGE_BYTES;
if (index < page_table_pages)
return (index);
}
return (-1);
}
extern char *page_address(page_index_t);
/* New objects are allocated to PAGE_TYPE_MIXED or PAGE_TYPE_CONS */
/* If you change these constants, then possibly also change the following
* functions in 'room.lisp':
* MAP-CODE-OBJECTS
* PRINT-ALL-CODE
* PRINT-LARGE-CODE
* PRINT-LARGE-UNBOXED
*/
// 4 bits for the type, 1 bit for SINGLE_OBJECT, 1 bit for OPEN_REGION
#define PAGE_TYPE_MASK 15 // mask out 'single-object' and 'open-region' flags
#define PAGE_TYPE_UNBOXED 1 // #b001
#define PAGE_TYPE_BOXED 2 // #b010
#define PAGE_TYPE_MIXED 3 // #b011
/* Small-Mixed pages hold objects that don't span cards. This is relatively easy to
* arrange for by adding filler at the end of a card to align prior to the next object,
* subject to a maximum allowable waste. Root scavenging can respect card boundaries
* and use scavenge functions specific to each object - the best of both worlds */
#define PAGE_TYPE_SMALL_MIXED 4 // #b100
#define PAGE_TYPE_CONS 5 // #b101
#define PAGE_TYPE_CODE 7 // #b111
#define THREAD_PAGE_FLAG 8
#define SINGLE_OBJECT_FLAG 16
#define OPEN_REGION_PAGE_FLAG 32
#define FREE_PAGE_FLAG 0
/* As usually configured, generations 0-5 are normal collected generations,
6 is pseudo-static (the objects in which are never moved nor reclaimed),
and 7 is scratch space used when collecting a generation without promotion,
wherein it is moved to generation 7 and back again.
*/
/*
* SCRATCH_GENERATION as we've defined it is kinda stupid because "<"
* doesn't do what you want. Other choices of value do, and since this an
* enum, it should be possible to change. Except it isn't because .. reasons.
* Here are some alternatives:
* A: gen 0 through 6 remain as-is and SCRATCH becomes -1
*
* B: 1 = nursery, 2 = older, ... up through "old" 6 which becomes the new 7;
* and SCRATCH becomes 0. This is like alternative (A) but avoids negatives.
*
* C: (probably the best)
* generations are stored with an implied decimal and one bit of fraction
* representing a half step so that:
* #b0000 = 0, #b0001 = 1/2 | #b0010 = 1, #b0011 = 1 1/2
* #b0100 = 2, #b0101 = 2 1/2 | #b0110 = 3, #b0111 = 3 1/2 ...
* up to 6 1/2. When GCing without promotion, we'd raise each object by half
* a generation, and then demote en masse, which is good because it makes the
* scratch pages older than from_space but younger than the youngest root gen.
*
* Of course, you could try to solve all this by keeping the existing numbering,
* but expressing comparison "a < b" as either:
* "logical_gen(a) < logical_gen(b)" // re-map numerically before compare
* or "gen_lessp(a,b)" // just rename the comparator
*
* I generally prefer numeric comparison to just work, though we have a further
* difficulty that page_table[page].gen is not always the generation of an object,
* as when it is non-large and pinned. So the helpers might be needed anyway.
*/
enum {
SCRATCH_GENERATION = PSEUDO_STATIC_GENERATION+1,
NUM_GENERATIONS
};
#define ARTIFICIALLY_HIGH_GEN 8
/* a structure to hold the state of a generation
*
* CAUTION: If you modify this, make sure to touch up the alien
* definition in src/code/gc.lisp accordingly. ...or better yes,
* deal with the FIXME there...
*/
struct generation {
/* the bytes allocated to this generation */
_Atomic(os_vm_size_t) bytes_allocated;
/* the number of bytes at which to trigger a GC */
os_vm_size_t gc_trigger;
/* to calculate a new level for gc_trigger */
os_vm_size_t bytes_consed_between_gc;
/* the number of GCs since the last raise */
int num_gc;
/* the number of GCs to run on the generations before raising objects to the
* next generation */
int number_of_gcs_before_promotion;
/* the cumulative sum of the bytes allocated to this generation. It is
* cleared after a GC on this generations, and update before new
* objects are added from a GC of a younger generation. Dividing by
* the bytes_allocated will give the average age of the memory in
* this generation since its last GC. */
os_vm_size_t cum_sum_bytes_allocated;
/* a minimum average memory age before a GC will occur helps
* prevent a GC when a large number of new live objects have been
* added, in which case a GC could be a waste of time */
double minimum_age_before_gc;
};
extern struct generation generations[NUM_GENERATIONS];
struct __attribute__((packed)) corefile_pte {
uword_t sso; // scan start offset
page_words_t words_used; // low bit is the 'large object' flag
};
/* When computing a card index we never subtract the heap base, which simplifies
* code generation. The heap base is guaranteed to be GC-page-aligned.
* The low bits can wraparound from all 1s to all 0s such that lowest numbered
* page index in linear order may have a higher card index.
* Two small examples of the distinction between page index and card index.
* For both examples: heap base = 0xEB00, page-size = 256b, npages = 8.
*
* Scenario A:
* CARDS_PER_PAGE = 1, card-size = 256b, ncards = 8, right-shift = 8, mask = #b111
* page page card
* index base index
* 0 EB00 3
* 1 EC00 4
* 2 ED00 5
* 3 EE00 6
* 4 EF00 7
* 5 F000 0
* 6 F100 1
* 7 F200 2
*
* Scenario B:
* CARDS_PER_PAGE = 2, card-size = 128b, ncards = 16, right-shift = 7, mask = #b1111
* page page card
* index base indices
* 0 EB00, EB80 6, 7
* 1 EC00, EC80 8, 9
* 2 ED00, ED80 10, 11
* 3 EE00, EE80 12, 13
* 4 EF00, EF80 14, 15
* 5 F000, F080 0, 1
* 6 F100, F180 2, 3
* 7 F200, F280 4, 5
*
*/
extern unsigned char *gc_card_mark;
extern long gc_card_table_mask;
#define addr_to_card_index(addr) ((((uword_t)addr)>>GENCGC_CARD_SHIFT) & gc_card_table_mask)
#define page_to_card_index(n) addr_to_card_index(page_address(n))
extern const char *widetag_names[];
// Distinguish penultimate GC (iteration 1) from ultimate GC (iteration 2) in save-lisp
extern int save_lisp_gc_iteration;
// This bit can be anything that doesn't conflict with a bit seen by lisp.
// Byte index 0 is the widetag, byte indices 1 and 2 are for the array-rank
// and vector-flags, depending on how src/compiler/generic/early-objdef assigns them.
#define WEAK_VECTOR_VISITED_BIT (1<<30)
// Assert that the 'v' is a visited weak object, and then clear the visited bit.
#define UNSET_WEAK_VECTOR_VISITED(v) \
gc_assert(v->header & WEAK_VECTOR_VISITED_BIT); v->header ^= WEAK_VECTOR_VISITED_BIT
lispobj *gc_search_space3(void *pointer, lispobj *start, void *limit);
static inline lispobj *gc_search_space(lispobj *start, void *pointer) {
return gc_search_space3(pointer,
start,
(void*)(1+((lispobj)pointer | LOWTAG_MASK)));
}
#ifdef LISP_FEATURE_X86
#include "code.h"
void gencgc_apply_code_fixups(struct code *old_code, struct code *new_code);
#else
#define gencgc_apply_code_fixups(ignore1,ignore2)
#endif
extern void gc_close_collector_regions(int);
/* The various sorts of pointer swizzling in SBCL. */
enum source {
SOURCE_NORMAL,
SOURCE_ZERO_TAG, /* code, lflist */
SOURCE_CLOSURE,
SOURCE_SYMBOL_NAME,
SOURCE_FDEFN_RAW
};
extern void set_allocation_bit_mark(void *address);
#define SET_ALLOCATED_BIT(x) set_allocation_bit_mark(x)
void *collector_alloc_fallback(struct alloc_region*,sword_t,int);
static inline void* __attribute__((unused))
gc_general_alloc(struct alloc_region* region, sword_t nbytes, int page_type)
{
/* We don't need small mixed pages. */
if (small_mixed_region == region &&
PAGE_TYPE_SMALL_MIXED == page_type) {
region = mixed_region; page_type = PAGE_TYPE_MIXED;
}
void *new_obj = region->free_pointer;
void *new_free_pointer = (char*)new_obj + nbytes;
lispobj *address;
// Large objects will never fit in a region, so we automatically dtrt
if (new_free_pointer <= region->end_addr) {
region->free_pointer = new_free_pointer;
address = new_obj;
} else {
address = collector_alloc_fallback(region, nbytes, page_type);
}
SET_ALLOCATED_BIT(address);
return address;
}
lispobj copy_potential_large_object(lispobj object, sword_t nwords,
struct alloc_region*, int page_type);
#define CHECK_COPY_PRECONDITIONS(object, nwords) \
gc_dcheck(is_lisp_pointer(object)); \
gc_dcheck(from_space_p(object)); \
gc_dcheck((nwords & 0x01) == 0)
#define CHECK_COPY_POSTCONDITIONS(copy, lowtag) \
gc_dcheck(lowtag_of(copy) == lowtag); \
gc_dcheck(!from_space_p(copy));
#define GC_LOGGING 0
/* For debugging purposes, you can make this macro as complicated as you like,
* such as checking various other aspects of the object in 'old' */
#if GC_LOGGING
#define NOTE_TRANSPORTING(old, new, nwords) really_note_transporting(old,new,nwords)
void really_note_transporting(lispobj old,void*new,sword_t nwords);
#elif defined COLLECT_GC_STATS && COLLECT_GC_STATS
#define NOTE_TRANSPORTING(old, new, nwords) gc_copied_nwords += nwords
#else
#define NOTE_TRANSPORTING(old, new, nwords) /* do nothing */
#endif
// In-situ live objects are those which get logically "moved" from oldspace to newspace
// by frobbing the generation byte in the page table, not copying.
extern uword_t gc_copied_nwords, gc_in_situ_live_nwords;
static inline lispobj
gc_copy_object(lispobj object, size_t nwords, void* region, int page_type)
{
CHECK_COPY_PRECONDITIONS(object, nwords);
/* Allocate space. */
lispobj *new = gc_general_alloc(region, nwords*N_WORD_BYTES, page_type);
NOTE_TRANSPORTING(object, new, nwords);
/* Copy the object. */
memcpy(new,native_pointer(object),nwords*N_WORD_BYTES);
return make_lispobj(new, lowtag_of(object));
}
// Like above but copy potentially fewer words than are allocated.
// ('old_nwords' can be, but does not have to be, smaller than 'nwords')
static inline lispobj
gc_copy_object_resizing(lispobj object, long nwords, void* region, int page_type,
int old_nwords)
{
CHECK_COPY_PRECONDITIONS(object, nwords);
lispobj *new = gc_general_alloc(region, nwords*N_WORD_BYTES, page_type);
NOTE_TRANSPORTING(object, new, old_nwords);
memcpy(new, native_pointer(object), old_nwords*N_WORD_BYTES);
return make_lispobj(new, lowtag_of(object));
}
extern sword_t (*const scavtab[256])(lispobj *where, lispobj object);
extern struct cons *weak_vectors; /* in gc-common.c */
extern struct hash_table *weak_hash_tables; /* in gc-common.c */
extern void acquire_gc_page_table_lock(void), release_gc_page_table_lock(void);
void gc_mark_range(lispobj*start, long count);
void gc_mark_obj(lispobj);
void gc_dispose_private_pages();
void add_to_weak_vector_list(lispobj* vector, lispobj header);
extern void heap_scavenge(lispobj *start, lispobj *limit);
extern sword_t scavenge(lispobj *start, sword_t n_words);
extern void scav_binding_stack(lispobj*, lispobj*, void(*)(lispobj));
extern void scan_binding_stack(void);
extern void scan_finalizers();
extern void cull_weak_hash_tables(int (*[4])(lispobj,lispobj));
extern void smash_weak_pointers(void);
extern bool scan_weak_hashtable(struct hash_table *hash_table,
int (*)(lispobj,lispobj),
void (*)(lispobj*));
extern int (*weak_ht_alivep_funs[4])(lispobj,lispobj);
extern void gc_scav_pair(lispobj where[2]);
extern void gc_common_init();
extern bool test_weak_triggers(bool (*)(lispobj), void (*)(lispobj));
lispobj copy_unboxed_object(lispobj object, sword_t nwords);
lispobj *search_read_only_space(void *pointer);
lispobj *search_static_space(void *pointer);
lispobj *search_dynamic_space(void *pointer);
extern int properly_tagged_p_internal(lispobj pointer, lispobj *start_addr);
static inline int properly_tagged_descriptor_p(void *pointer, lispobj *start_addr) {
return is_lisp_pointer((lispobj)pointer) &&
properly_tagged_p_internal((lispobj)pointer, start_addr);
}
static inline void add_to_weak_pointer_chain(struct weak_pointer *wp) {
// Better already be fixed in position or we're in trouble
gc_dcheck(!compacting_p() || !from_space_p(make_lispobj(wp,OTHER_POINTER_LOWTAG)));
/* Link 'wp' into weak_pointer_chain using its 'next' field.
* We ensure that 'next' is always NULL when the weak pointer isn't
* in the chain, and not NULL otherwise. The end of the chain
* is denoted by WEAK_POINTER_CHAIN_END which is distinct from NULL.
* The test of whether the weak pointer has been placed in the chain
* is performed in 'scav_weak_pointer' */
set_weak_pointer_next(wp, weak_pointer_chain);
weak_pointer_chain = wp;
}
extern unsigned char* gc_card_mark;
// "assign" as the operation name is a little clearer than "set"
// which tends to be synonymous with setting a bit to 1.
#define assign_page_card_marks(page, val) \
memset(gc_card_mark+page_to_card_index(page), val, CARDS_PER_PAGE)
#ifdef LISP_FEATURE_SOFT_CARD_MARKS
#define NON_FAULTING_STORE(operation, addr) { operation; }
// The low bit of 0 implies "marked". So CARD_MARKED and STICKY_MARK
// are both considered marked. All bits of UNMARKED are 1s, so that
// one word full of mark bytes reads as -1. See the autogenerated
// functions in "genesis/cardmarks.h" for the use-case.
#define CARD_MARKED 0
#define STICKY_MARK 2
#define CARD_UNMARKED 0xff
#define MARK_BYTE_MASK 0xff
#else
// With physical page protection, bit index 0 is the MARKED bit (inverted WP bit)
// and bit index 1 is the WP_CLEARED (write_protect_cleared) bit.
// The fault handler always sets both bits.
// The only other bit pair value would be illegal.
#define CARD_UNMARKED 0 /* write-protected = 1 */
#define CARD_MARKED 1 /* write-protected = 0 */
#define WP_CLEARED_AND_MARKED 3 /* write-protected = 0, wp-cleared = 1 */
// CODE-HEADER-SET can store the low byte from NULL-TN into the mark array.
// This sets the two low bits on, but also spuriously sets other bits,
// which we can ignore when reading the byte.
#define MARK_BYTE_MASK 3
#define PAGE_WRITEPROTECTED_P(n) (~gc_card_mark[page_to_card_index(n)] & CARD_MARKED)
#define SET_PAGE_PROTECTED(n,val) gc_card_mark[page_to_card_index(n)] = \
(val ? CARD_UNMARKED : CARD_MARKED)
#define cardseq_any_marked(card_index) (gc_card_mark[card_index] & CARD_MARKED)
#define cardseq_all_marked_nonsticky(card_index) cardseq_any_marked(card_index)
#define page_cards_all_marked_nonsticky(page_index) \
cardseq_all_marked_nonsticky(page_to_card_index(page_index))
/* NON_FAULTING_STORE is only for fixnums and GC metadata where we need
* the ability to write-through the store barrier.
* The uses are limited to updating the weak vector chain, weak hash-table
* chain, and vector rehash flags. Those don't affect a page's marked state */
#define PAGE_BASE(addr) ((char*)ALIGN_DOWN((uword_t)(addr),GENCGC_PAGE_BYTES))
#define NON_FAULTING_STORE(operation, addr) { \
page_index_t page_index = find_page_index(addr); \
if (page_index < 0 || !PAGE_WRITEPROTECTED_P(page_index)) { operation; } \
else { os_protect(PAGE_BASE(addr), GENCGC_PAGE_BYTES, OS_VM_PROT_JIT_ALL); \
operation; \
os_protect(PAGE_BASE(addr), GENCGC_PAGE_BYTES, OS_VM_PROT_JIT_READ); } }
/* This is used by the fault handler, and potentially during GC
* if we need to remember that a pointer store occurred.
* The fault handler should supply WP_CLEARED_AND_MARKED as the mark,
* but the collector should use CARD_MARKED */
static inline void unprotect_page(void* addr, unsigned char mark)
{
// No atomic op needed for a 1-byte store.
gc_card_mark[addr_to_card_index(addr)] = mark;
os_protect(PAGE_BASE(addr), GENCGC_PAGE_BYTES, OS_VM_PROT_JIT_ALL);
}
#endif
// Helpers to avoid invoking the memory fault signal handler.
// For clarity, distinguish between words which *actually* need to frob
// physical (MMU-based) protection versus those which don't,
// but are forced to call mprotect() because it's the only choice.
// Unlike with NON_FAULTING_STORE, in this case we actually do want to record that
// the ensuing store toggles the WP bit without invoking the fault handler.
static inline void notice_pointer_store(__attribute__((unused)) void* base_addr,
__attribute__((unused)) void* slot_addr) {
#ifdef LISP_FEATURE_SOFT_CARD_MARKS
int card = addr_to_card_index(base_addr);
// STICKY is stronger than MARKED. Only change if UNMARKED.
if (gc_card_mark[card] == CARD_UNMARKED) gc_card_mark[card] = CARD_MARKED;
#else
page_index_t index = find_page_index(slot_addr);
gc_assert(index >= 0);
if (PAGE_WRITEPROTECTED_P(index)) unprotect_page(slot_addr, CARD_MARKED);
#endif
}
static inline void vector_notice_pointer_store(void* addr) {
notice_pointer_store(addr, addr);
}
static inline void ensure_non_ptr_word_writable(__attribute__((unused)) void* addr)
{
// there's nothing to "ensure" if using software card marks
#ifndef LISP_FEATURE_SOFT_CARD_MARKS
// #-soft-card-marks ignores the first argument of notice_pointer_store()
notice_pointer_store(0, addr);
#endif
}
// #+soft-card-mark: this expresion is true of both CARD_MARKED and STICKY_MARK
// #-soft-card-mark: this expresion is true of both CARD_MARKED and WP_CLEARED_AND_MARKED
#define card_dirtyp(index) (gc_card_mark[index] & MARK_BYTE_MASK) != CARD_UNMARKED
#include "forwarding-ptr.inc"
/* Check whether 'pointee' was forwarded. If it has been, update the contents
* of 'cell' to point to it. Otherwise, set 'cell' to 'broken'.
* Note that this macro has no braces around the body because one of the uses
* of it needs to stick on another 'else' or two */
#define TEST_WEAK_CELL(cell, pointee, broken) \
lispobj *native = native_pointer(pointee); \
if (from_space_p(pointee)) \
cell = forwarding_pointer_p(native) ? forwarding_pointer_value(native) : broken; \
else if (immobile_space_p(pointee)) { \
if (immobile_obj_gen_bits(base_pointer(pointee)) == from_space) cell = broken; \
}
#ifdef MAX_CONSES_PER_PAGE
static const int CONS_PAGE_USABLE_BYTES = MAX_CONSES_PER_PAGE*CONS_SIZE*N_WORD_BYTES;
#endif
/* Return true if 'addr' has a lowtag and widetag that correspond,
* given that the words at 'addr' are within range for an allocated page.
* 'addr' could be a pointer to random data, and this check is merely
* a heuristic. False positives are possible. */
static inline bool plausible_tag_p(lispobj addr)
{
if (listp(addr))
return is_cons_half(CONS(addr)->car)
&& is_cons_half(CONS(addr)->cdr)
// -1 can be left by the */signed=>integer vop
// and is also useful as filler on cons pages.
&& CONS(addr)->car != (uword_t)-1;
unsigned char widetag = widetag_of(native_pointer(addr));
return other_immediate_lowtag_p(widetag)
&& lowtag_of(addr) == LOWTAG_FOR_WIDETAG(widetag);
}
extern char * gc_logfile;
extern void log_generation_stats(char *logfile, char *header);
extern void print_generation_stats(void);
extern double generation_average_age(generation_index_t);
#define PAGE_INDEX_FMT PRIdPTR
static inline os_vm_size_t npage_bytes(page_index_t npages)
{
gc_assert(npages>=0);
return ((os_vm_size_t)npages)*GENCGC_PAGE_BYTES;
}
extern os_vm_size_t auto_gc_trigger;
void zeroize_pages_if_needed(page_index_t start, page_index_t end, int page_type);
typedef unsigned int page_bytes_t;
#define page_words_used(index) page_table[index].words_used_
#define page_bytes_used(index) ((page_bytes_t)page_table[index].words_used_<<WORD_SHIFT)
#if defined LISP_FEATURE_RISCV && defined LISP_FEATURE_LINUX // KLUDGE
#define page_need_to_zero(index) (mmap_does_not_zero || page_table[index].need_zerofill)
#else
#define page_need_to_zero(index) page_table[index].need_zerofill
#endif
#define set_page_bytes_used(index,val) page_table[index].words_used_ = ((val)>>WORD_SHIFT)
#define set_page_need_to_zero(index,val) page_table[index].need_zerofill = val
#if !CONDENSED_PAGE_TABLE
// 32-bit doesn't need magic to reduce the size of scan_start_offset.
#define set_page_scan_start_offset(index,val) \
page_table[index].scan_start_offset_ = val
#define page_scan_start_offset(index) page_table[index].scan_start_offset_
#else
/// A "condensed" offset reduces page table size, which improves scan locality.
/// As stored, the offset is scaled down either by card size or double-lispwords.
/// If the offset is the maximum, then we must check if the page pointed to by
/// that offset is actually the start of a region, and retry if not.
/// For debugging the iterative algorithm it helps to use a max value
/// that is less than UINT_MAX to get a pass/fail more quickly.
//#define SCAN_START_OFS_MAX 0x3fff
#define SCAN_START_OFS_MAX UINT_MAX
static void __attribute__((unused))
set_page_scan_start_offset(page_index_t index, os_vm_size_t offset)
{
// If the offset is nonzero and page-aligned
unsigned int lsb = offset !=0 && IS_ALIGNED(offset, GENCGC_PAGE_BYTES);
os_vm_size_t scaled = (offset >> (lsb ? GENCGC_CARD_SHIFT-1 : WORD_SHIFT)) | lsb;
if (scaled > SCAN_START_OFS_MAX) {
// Assert that if offset exceed the max representable value,
// then it is a page-aligned offset, not a cons-aligned offset.
gc_assert(lsb == 1);
// Clip it to the maximum representable value.
// The accessor will have to iterate to find the true start of region.
scaled = SCAN_START_OFS_MAX;
}
page_table[index].scan_start_offset_ = scaled;
}
static os_vm_size_t scan_start_offset_iterated(page_index_t index)
{
// The low bit of the MAX is the 'scale' bit. The max pages we can look
// backwards is therefore the max shifted right by 1 bit.
page_index_t tot_offset_in_pages = 0;
unsigned int offset;
do {
page_index_t lookback_page = index - tot_offset_in_pages;
offset = page_table[lookback_page].scan_start_offset_;
tot_offset_in_pages += offset >> 1;
} while (offset == SCAN_START_OFS_MAX);
return (os_vm_size_t)tot_offset_in_pages << GENCGC_CARD_SHIFT;
}
static os_vm_size_t __attribute__((unused)) page_scan_start_offset(page_index_t index)
{
return page_table[index].scan_start_offset_ != SCAN_START_OFS_MAX
? (os_vm_size_t)(page_table[index].scan_start_offset_ & ~1)
<< ((page_table[index].scan_start_offset_ & 1) ?
(GENCGC_CARD_SHIFT-1) : WORD_SHIFT)
: scan_start_offset_iterated(index);
}
#endif
#define is_code(type) ((type & PAGE_TYPE_MASK) == PAGE_TYPE_CODE)
// If *all* pages use soft card marks, then protection_mode() is not a thing.
// Otherwise, only pages of code use soft card marks; return an enum indicating
// whether the page protection for the specified page is applied in harware.
#ifndef LISP_FEATURE_SOFT_CARD_MARKS
enum prot_mode { PHYSICAL, LOGICAL };
static inline enum prot_mode protection_mode(page_index_t page) {
// code pages can be marked as logically read-only without OS protection,
// and everything else uses hardware-based protection where applicable.
return is_code(page_table[page].type) ? LOGICAL : PHYSICAL;
}
#endif
static inline bool page_free_p(page_index_t page) {
return (page_table[page].type == FREE_PAGE_FLAG);
}
/* Check that X is a higher address than Y and return offset from Y to
* X in bytes. */
static inline os_vm_size_t
addr_diff(void *x, void *y)
{
gc_assert(x >= y);
return (uintptr_t)x - (uintptr_t)y;
}
extern os_vm_size_t bytes_allocated;
extern void reset_page_flags(page_index_t page);
/* True if the page starts a contiguous block. */
static inline bool
page_starts_contiguous_block_p(page_index_t page_index)
{
// Don't use the preprocessor macro: 0 means 0.
return page_table[page_index].scan_start_offset_ == 0;
}
/* True if the page is the last page in a contiguous block. */
static inline bool
page_ends_contiguous_block_p(page_index_t page_index,
generation_index_t __attribute__((unused)) gen)
{
/* Re. this next test: git rev c769dd53 said that there was a bug when we don't
* test page_bytes_used, but I fail to see how 'page_starts_contiguous_block_p'
* on the next page is not a STRONGER condition, i.e. it should imply that
* 'page_index' ends a block without regard for the number of bytes used.
* Apparently at some point I understood this and now I don't again.
* That's what comments are for, damnit.
* Anyway, I *think* the issue was, at some point, as follows:
* | page | page |
* pinned-obj
* <------------------- scan-start
* where the first of the two pages had a small object pinned. This used to
* adjust the bytes used to account _only_ for the pins. That was wrong -
* the page has to be counted as if it is completely full.
* So _maybe_ both these conditions do not need to be present now ?
*/
// There is *always* a next page in the page table.
bool answer = page_words_used(page_index) < GENCGC_PAGE_WORDS
|| page_starts_contiguous_block_p(page_index+1);
#ifdef DEBUG
bool safe_answer =
(/* page doesn't fill block */
(page_words_used(page_index) < GENCGC_PAGE_WORDS)
/* page is last allocated page */
|| ((page_index + 1) >= next_free_page)
/* next page contains no data */
|| !page_words_used(page_index + 1)
/* next page is in different generation */
|| (page_table[page_index + 1].gen != gen)
/* next page starts its own contiguous block */
|| (page_starts_contiguous_block_p(page_index + 1)));
gc_assert(answer == safe_answer);
#endif
return answer;
}
static inline page_index_t contiguous_block_final_page(page_index_t first) {
page_index_t last = first;
while (!page_ends_contiguous_block_p(last, page_table[first].gen)) ++last;
return last;
}
int gencgc_handle_wp_violation(void*, void*);
// The flags control the behavior of sync_close_regions()
#define LOCK_PAGE_TABLE 1
#define LOCK_CODE_ALLOCATOR 2
#define CONSUME_REMAINDER 4
void gc_close_region(struct alloc_region *alloc_region, int page_type);
static inline void ensure_region_closed(struct alloc_region *alloc_region,
int page_type)
{
if (alloc_region->start_addr)
gc_close_region(alloc_region, page_type);
}
static inline void gc_set_region_empty(struct alloc_region *region)
{
/* Free-pointer has to be not equal to 0 because it's undefined behavior
* to add any value whatsoever to the null pointer.
* Annoying, isn't it. http://c-faq.com/null/machexamp.html */
region->free_pointer = region->end_addr = (void*)0x1000;
/* Start 0 is the indicator of closed-ness. */
region->start_addr = 0;
/* last_page is not reset. It can be used as a hint where to resume
* allocating after closing and re-opening the region */
}
static inline void gc_init_region(struct alloc_region *region)
{
// A distinction without a difference (it used to do one more assignment)
gc_set_region_empty(region);
}
#define compacting_p() (from_space>=0)
#define page_single_obj_p(page) ((page_table[page].type & SINGLE_OBJECT_FLAG)!=0)
extern unsigned char* gc_page_pins;
#define pinned_p(dummy1,dummy2) 0
extern generation_index_t from_space, new_space;
generation_index_t gc_gen_of(lispobj obj, int defaultval);
static bool __attribute__((unused))
from_space_p(lispobj obj)
{
/* There'd be a cyclic dependency between pmrgc-impl.h and
* incremental-compact.h would we try to #include the latter. */
extern unsigned char *target_pages;
page_index_t page_index = find_page_index((void*)obj);
if (page_index == -1) return 0;
/* We can only move objects in or younger than the current
* generation, as we can't build a complete remset for older
* objects in a younger GC. */
return target_pages[page_index] && gc_gen_of(obj, 0) <= new_space;
}
static bool __attribute__((unused)) new_space_p(lispobj obj)
{
gc_dcheck(compacting_p());
page_index_t page_index = find_page_index((void*)obj);
// NOTE: It is legal to access page_table at index -1,
// and the 'gen' of page -1 is an otherwise unused value.
return page_table[page_index].gen == new_space;
}
#ifdef LISP_FEATURE_IMMOBILE_SPACE
struct fixedobj_page { // 8 bytes per page
unsigned int free_index; // index is in bytes. 4 bytes
union immobile_page_attr {
int packed;
struct {
unsigned char flags;
unsigned char obj_align; // object spacing expressed in lisp words
unsigned char unused1;
/* Which generations have data on this page */
unsigned char gens_; // a bitmap
} parts;
} attr;
};
extern struct fixedobj_page *fixedobj_pages;
#define fixedobj_page_obj_align(i) (fixedobj_pages[i].attr.parts.obj_align<<WORD_SHIFT)
#endif
extern page_index_t next_free_page;
extern uword_t
walk_generation(uword_t (*proc)(lispobj*,lispobj*,uword_t),
generation_index_t generation, uword_t extra);
/* The minimum heap occupancy to force more aggressive collections above. */
#define PANIC_THRESHOLD 0.9
// The inverse of page_overhead_threshold.
#define FRAGMENTATION_COMPENSATION (1.0 / 1.3)
extern bool taggedptr_alivep_impl(lispobj obj);
__attribute__((unused)) static lispobj *next_object(lispobj *previous, uword_t size, lispobj *end) {
/* Dynamic space is non-contiguous, static space is contiguous.
* TODO: This really should whiz along by words like find_object
* does. */
extern bool allocation_bit_marked(void *pointer);
if (find_page_index(previous) != -1) {
for (lispobj *where = previous + size; where < end; where += 2)
if (allocation_bit_marked(where))
return where;
return NULL;
} else {
if (previous + size >= end) return NULL;
return previous + size;
}
}
__attribute__((unused)) static lispobj *page_limit(page_index_t page) {
return (lispobj*)page_address(page + 1);
}

2102
src/runtime/pmrgc.c Normal file
View file

File diff suppressed because it is too large Load diff

84
src/runtime/queue-suballocator.h Normal file
View file

@ -0,0 +1,84 @@
/* To allocate queue blocks without mmaping every time, we allocate larger and
* larger "chunks" of memory. As the size of each subsequent chunk grows
* exponentially, the metadata needed to represent a mark stack of any size
* and the number of mmaps grow logarithmically. */
#define INITIAL_SIZE (QBLOCK_BYTES * 32)
#define CHUNKS 20
/* Only give back chunks when we haven't used them after this many
* calls to suballoc_release. */
#define AGE_LIMIT 10
struct suballocator_chunk {
uword_t start;
uword_t free;
uword_t size;
uword_t age;
};
/* suballoc_allocate is lazily written and skips over the first chunk, so
* add one here to make CHUNKS make sense. No big loss. */
struct suballocator {
char *name;
struct suballocator_chunk chunks[CHUNKS + 1];
unsigned int current_chunk;
uword_t hwm;
lock_t suballocator_lock;
};
/* Zero-initialising structs in an array in a struct - hooray */
#define SUBALLOCATOR_INITIALIZER(name) { name, { { 0 } }, 0, 0, LOCK_INITIALIZER }
static bool chunk_has_space(struct suballocator *s, int index) {
return s->chunks[index].free < s->chunks[index].start + s->chunks[index].size;
}
/* Free all memory used by the mark stack, giving back to the OS. */
static void suballoc_release(struct suballocator *s) {
uword_t used = 0;
for (int i = 0; i < CHUNKS; i++) {
if (s->chunks[i].start) {
used += s->chunks[i].size;
if (s->chunks[i].age >= AGE_LIMIT) {
os_deallocate((void*)s->chunks[i].start, s->chunks[i].size);
s->chunks[i].start = s->chunks[i].free = s->chunks[i].size = 0;
} else {
s->chunks[i].age++;
s->chunks[i].free = s->chunks[i].start;
}
}
}
if (used > s->hwm) {
#if 0
fprintf(stderr, "Used %ld bytes for %s\n", used, s->name);
#endif
s->hwm = used;
}
s->current_chunk = 0;
}
static struct Qblock *suballoc_allocate(struct suballocator *s) {
acquire_lock(&s->suballocator_lock);
struct suballocator_chunk *chunk = s->chunks + s->current_chunk;
/* Check if we need to make a new chunk first. */
if (!chunk_has_space(s, s->current_chunk)) {
/* This skips over chunk[0]. Oh well. */
if (s->current_chunk == CHUNKS) lose("Ran out of suballocator chunks.");
s->current_chunk++;
chunk = s->chunks + s->current_chunk;
/* Check if we can reuse a chunk which was allocated before. */
if (!chunk_has_space(s, s->current_chunk)) {
uword_t size = INITIAL_SIZE << s->current_chunk;
uword_t address = (uword_t)os_allocate(size);
if (!address) lose("Failed to allocate suballocator chunk with %ld bytes.", size);
// fprintf(stderr, "alloc #%d: %ld at %ld\n", current_chunk, size, address);
chunk->start = chunk->free = address;
chunk->size = size;
}
chunk->age = 0;
}
/* Now get another block from the current chunk. */
struct Qblock *where = (struct Qblock*)chunk->free;
gc_assert(where != NULL);
where->count = 0;
chunk->free += QBLOCK_BYTES;
release_lock(&s->suballocator_lock);
return where;
}

33
src/runtime/tiny-lock.h Normal file
View file

@ -0,0 +1,33 @@
#ifdef USE_PTHREAD_LOCK
/* Probably not something you actually want to do in a signal handler,
* but it lets you use things like mutrace. */
#include <pthread.h>
typedef pthread_mutex_t lock_t;
static void acquire_lock(lock_t *l) { gc_assert(!pthread_mutex_lock(l)); }
static void release_lock(lock_t *l) { gc_assert(!pthread_mutex_unlock(l)); }
#define LOCK_INITIALIZER PTHREAD_MUTEX_INITIALIZER
#else
/* A lock that has the sole benefit of not being the pthreads lock.
* And the sole downside of not being the pthreads lock. */
#include <sched.h>
#include <stdatomic.h>
struct lock { _Atomic(int) grabbed; };
typedef struct lock lock_t;
static void acquire_lock(lock_t *l) {
int expected = 0, cycles = 0;
/* atomic_compare_exchange_strong kindly clobbers expected for us,
* when CAS fails. */
while (!atomic_compare_exchange_strong(&l->grabbed, &expected, 1)) {
expected = 0;
if (cycles++ > 1000) sched_yield();
}
}
static void release_lock(lock_t *l) {
atomic_store(&l->grabbed, 0);
}
#define LOCK_INITIALIZER { 0 }
#endif