Generalize initializing and closing all TLABs for a thread

Support independent TLABs for equal GC page types so that objects may be
segregated by criteria other than page type. One case would be free_oldspace
wanting to scan symbol pages to recycle TLS indices more efficiently than
scanning everything. Another would be co-locating PCL metadata for locality,
and/or discounting it in SB-APROF profiling.
No harm if extra TLABs aren't used for anything. (They aren't yet)

src/compiler/generic/genesis.lisp

@@ -3355,6 +3355,13 @@ lispobj symbol_package(struct symbol*);~%" (genesis-header-prefix))
                        sb-vm:code-slots-per-simple-fun))
              (when (and (eq name 'sb-vm::thread)
                         (find 'sb-vm::pseudo-atomic-bits slots :key #'sb-vm:slot-name))
+               (format t "#define INIT_THREAD_REGIONS(x) \\~%")
+               (let ((tlabs (map 'list
+                                 (lambda (x) (c-name (string-downcase (second x))))
+                                 (remove-if-not (lambda (x)
+                                                  (tailwise-equal (string (second x)) "-TLAB"))
+                                                slots))))
+                 (format t "~{ gc_init_region(&x->~A)~^,\\~%~}~2%" tlabs))
                (format t "#define HAVE_THREAD_PSEUDO_ATOMIC_BITS_SLOT 1~2%")
                #+(or sparc ppc ppc64) (format t "typedef char pa_bits_t[~d];~2%" sb-vm:n-word-bytes)
                #-(or sparc ppc ppc64) (format t "typedef lispobj pa_bits_t;~2%"))

src/compiler/generic/objdef.lisp

@@ -578,6 +578,10 @@ during backtrace.
   (mach-port-name :c-type "mach_port_name_t")
   #+ppc64 (card-table)
 
+  ;; A few extra thread-local allocation buffers for special purposes
+  (symbol-tlab :c-type "struct alloc_region" :length 3)
+  (sys-mixed-tlab :c-type "struct alloc_region" :length 3)
+  (sys-cons-tlab :c-type "struct alloc_region" :length 3)
   ;; allocation instrumenting
   (tot-bytes-alloc-boxed)
   (tot-bytes-alloc-unboxed)

src/runtime/gc-internal.h

@@ -162,6 +162,6 @@ instance_scan(void (*proc)(lispobj*, sword_t, uword_t),
 extern int simple_fun_index(struct code*, struct simple_fun*);
 
 extern lispobj decode_fdefn_rawfun(struct fdefn *fdefn);
-extern void gc_close_thread_regions(struct thread*);
+extern void gc_close_thread_regions(struct thread*, int);
 extern void gc_close_collector_regions(int);
 #endif /* _GC_INTERNAL_H_ */

src/runtime/gencgc-internal.h

@@ -59,6 +59,9 @@ int gencgc_handle_wp_violation(void*, void*);
 #endif
 typedef unsigned short page_words_t;
 
+#define LOCK_PAGE_TABLE 1
+#define LOCK_CODE_ALLOCATOR 2
+
 /* New objects are allocated to PAGE_TYPE_MIXED or PAGE_TYPE_CONS */
 /* If you change these constants, then possibly also change the following
  * functions in 'room.lisp':

src/runtime/gencgc.c

@@ -4542,10 +4542,7 @@ collect_garbage(generation_index_t last_gen)
      * So we need to close them for those two cases.
      */
     struct thread *th;
-    for_each_thread(th) {
-        ensure_region_closed(THREAD_ALLOC_REGION(th,mixed), PAGE_TYPE_MIXED);
-        ensure_region_closed(THREAD_ALLOC_REGION(th,cons), PAGE_TYPE_CONS);
-    }
+    for_each_thread(th) gc_close_thread_regions(th, 0);
     ensure_region_closed(code_region, PAGE_TYPE_CODE);
     if (gencgc_verbose > 2) fprintf(stderr, "[%d] BEGIN gc(%d)\n", n_gcs, last_gen);
 
@@ -5263,27 +5260,31 @@ NO_SANITIZE_MEMORY lispobj listify_rest_arg(lispobj* context, sword_t context_by
 }
 #endif
 
-void sync_close_regions(int block_signals,
-                        struct alloc_region *region_1, int page_type_1,
-                        struct alloc_region *region_2, int page_type_2)
+typedef struct { struct alloc_region* r; int type; } close_region_arg;
+
+static void sync_close_regions(int block_signals, int locking,
+                               close_region_arg* a, int count)
 {
     sigset_t savedmask;
-    int result;
-    int need_code_lock = (page_type_1 == PAGE_TYPE_CODE || page_type_2 == PAGE_TYPE_CODE);
+    __attribute__((unused)) int result;
     if (block_signals) block_blockable_signals(&savedmask);
-    if (need_code_lock) {
+    if (locking & LOCK_CODE_ALLOCATOR) {
         result = mutex_acquire(&code_allocator_lock);
-        gc_assert(result);
+        gc_dcheck(result);
     }
-    result = mutex_acquire(&free_pages_lock);
-    gc_assert(result);
-    if (region_1) ensure_region_closed(region_1, page_type_1);
-    if (region_2) ensure_region_closed(region_2, page_type_2);
-    result = mutex_release(&free_pages_lock);
-    gc_assert(result);
-    if (need_code_lock) {
+    if (locking & LOCK_PAGE_TABLE) {
+        result = mutex_acquire(&free_pages_lock);
+        gc_dcheck(result);
+    }
+    int i;
+    for(i=0; i<count; ++i) ensure_region_closed(a[i].r, a[i].type);
+    if (locking & LOCK_PAGE_TABLE) {
+        result = mutex_release(&free_pages_lock);
+        gc_dcheck(result);
+    }
+    if (locking & LOCK_CODE_ALLOCATOR) {
         result = mutex_release(&code_allocator_lock);
-        gc_assert(result);
+        gc_dcheck(result);
     }
     if (block_signals) thread_sigmask(SIG_SETMASK, &savedmask, 0);
 }
@@ -5294,16 +5295,29 @@ void sync_close_regions(int block_signals,
  * these are plain foreign calls without aid of a vop. */
 void close_current_thread_tlab() {
     __attribute__((unused)) struct thread *self = get_sb_vm_thread();
-    sync_close_regions(1, THREAD_ALLOC_REGION(self,mixed), PAGE_TYPE_MIXED,
-                          THREAD_ALLOC_REGION(self,cons), PAGE_TYPE_CONS);
+    close_region_arg a[4] = {
+      { THREAD_ALLOC_REGION(self,mixed), PAGE_TYPE_MIXED },
+      { THREAD_ALLOC_REGION(self,cons), PAGE_TYPE_CONS },
+      { THREAD_ALLOC_REGION(self,sys_mixed), PAGE_TYPE_MIXED },
+      { THREAD_ALLOC_REGION(self,sys_cons), PAGE_TYPE_CONS }
+    };
+    sync_close_regions(1, LOCK_PAGE_TABLE, a, 4);
 }
 void close_code_region() {
-    sync_close_regions(1, code_region, PAGE_TYPE_CODE, 0, 0);
+    close_region_arg a = { code_region, PAGE_TYPE_CODE };
+    sync_close_regions(1, LOCK_PAGE_TABLE|LOCK_CODE_ALLOCATOR, &a, 1);
 }
-/* This is called by unregister_thread() with STOP_FOR_GC blocked */
-void gc_close_thread_regions(struct thread* th) {
-    sync_close_regions(0, &th->mixed_tlab, PAGE_TYPE_MIXED,
-                          &th->cons_tlab, PAGE_TYPE_CONS);
+/* This is called by unregister_thread() with STOP_FOR_GC blocked,
+   which needs to lock the page table. It is also called at the start of GC
+   to close all thread regions; no locks are needed in that case */
+void gc_close_thread_regions(struct thread* th, int locking) {
+    close_region_arg a[4] = {
+      { &th->mixed_tlab, PAGE_TYPE_MIXED },
+      { &th->cons_tlab, PAGE_TYPE_CONS },
+      { &th->sys_mixed_tlab, PAGE_TYPE_MIXED },
+      { &th->sys_cons_tlab, PAGE_TYPE_CONS }
+    };
+    sync_close_regions(0, locking, a, 4);
 }
 
 #ifdef LISP_FEATURE_SPARC
@@ -5567,7 +5581,7 @@ gc_and_save(char *filename, boolean prepend_runtime, boolean purify,
     // From here on until exit, there is no chance of continuing
     // in Lisp if something goes wrong during GC.
     // Flush regions to ensure heap scan in copy_rospace doesn't miss anything
-    gc_close_thread_regions(thread);
+    gc_close_thread_regions(thread, 0);
     gc_close_collector_regions(0);
     move_rospace_to_dynamic(0);
     pre_verify_gen_0 = 1;

src/runtime/monitor.c

@@ -247,7 +247,7 @@ static void suspend_other_threads() {
     // It might make sense for each thread's stop-for-gc handler to close its region
     // versus doing this loop
     struct thread *th;
-    for_each_thread(th) { gc_close_thread_regions(th); }
+    for_each_thread(th) { gc_close_thread_regions(th, 0); }
     gc_close_collector_regions(0);
 }
 static void unsuspend_other_threads() {

src/runtime/thread.c

@@ -452,7 +452,7 @@ unregister_thread(struct thread *th,
     int lock_ret;
 
     block_blockable_signals(0);
-    gc_close_thread_regions(th);
+    gc_close_thread_regions(th, LOCK_PAGE_TABLE);
 #ifdef LISP_FEATURE_SB_SAFEPOINT
     pop_gcing_safety(&scribble->safety);
 #else
@@ -1015,10 +1015,7 @@ alloc_thread_struct(void* spaces) {
     clear_pseudo_atomic_interrupted(th);
 #endif
 
-#ifdef LISP_FEATURE_GENCGC
-    gc_init_region(&th->mixed_tlab);
-    gc_init_region(&th->cons_tlab);
-#endif
+    INIT_THREAD_REGIONS(th);
 #ifdef LISP_FEATURE_SB_THREAD
     /* This parallels the same logic in globals.c for the
      * single-threaded foreign_function_call_active, KLUDGE and