arm64, return ldp/stp optimization: fix constants reuse

- Allow EVEX printer generation for vpmovsx* (#x20-#x25) and vpmovzx*
  (#x30-#x35) in map 0F38 in avx2-inst-printer-list.
- Use xmmreg/mem for 4x/8x widening conversions and half-ymmreg/mem for 2x
  widening conversions so source operands disassemble with their correct
  architectural register width (XMM or YMM).
- Define evex-ymmreg-b to incorporate EVEX.X / B' (bit 4) for register-direct
  operands in EVEX-ymm-ymm-imm format, correctly decoding registers 16-31 for
  vpslldq, vpsrldq, and shift/rotate instructions.
- Widen register bounds checks in print-ymmreg-rm, print-ymmreg-vvvv,
  print-ymmreg, and print-xmmreg from (unsigned-byte 4) to (mod 32).
- Add regression tests in tests/simd-pack-512.pure.lisp.

.github/workflows/linux-arm.yml

@@ -29,6 +29,8 @@ jobs:
         CC: arm-linux-gnueabihf-gcc
       run: ./make.sh --xc-host='sbcl --dynamic-space-size 700MB --lose-on-corruption --disable-ldb --disable-debugger' --arch=arm
 
+    - name: build documentation
+      run: cd ./doc/manual && sh generate-texinfo.sh
     - name: test
       env:
         CC: arm-linux-gnueabihf-gcc

.github/workflows/linux-arm64.yml

@@ -47,6 +47,8 @@ jobs:
         name: sbcl-linux-binary-arm64${{ matrix.options }}
         path: sbcl-linux-binary-arm64${{ matrix.options }}
 
+    - name: build documentation
+      run: cd ./doc/manual && sh generate-texinfo.sh
     - name: test
       run: cd tests; ./run-tests.sh --slow
     - name: ansi-test

.github/workflows/linux.yml

@@ -13,7 +13,7 @@ jobs:
         arch: [x86, x86-64]
         subfeatures: ['']
         include:
-          - { arch: x86-64, subfeatures: sse4, options: --with-sb-thread --with-nonstop-foreign-call }
+          - { arch: x86-64, subfeatures: sse4, options: --with-sb-thread --with-nonstop-foreign-call --with-tls-based-mv-return }
           - { arch: x86-64, subfeatures: fasteval, options: --with-sb-fasteval --without-sb-eval --with-nonstop-foreign-call }
           - { arch: x86-64, options: --with-mark-region-gc --with-nonstop-foreign-call }
 
@@ -50,6 +50,8 @@ jobs:
     - name: test-fasteval
       if: matrix.subfeatures == 'fasteval'
       run: cd tests; ./run-tests.sh --evaluator-mode interpret
+    - name: build documentation
+      run: cd ./doc/manual && sh generate-texinfo.sh
     - name: test
       run: cd tests; ./run-tests.sh --slow
     - name: ansi-test

.github/workflows/mac.yml

@@ -10,7 +10,7 @@ jobs:
     strategy: 
       matrix:
         arch: [arm64, x86-64]
-        options: [--with-sb-thread --with-nonstop-foreign-call, --with-mark-region-gc --with-nonstop-foreign-call]
+        options: [--with-sb-thread --with-nonstop-foreign-call --with-tls-based-mv-return, --with-mark-region-gc --with-nonstop-foreign-call --with-tls-based-mv-return]
       fail-fast: false
 
     steps:
@@ -21,7 +21,7 @@ jobs:
       env:
         SBCL_MAKE_JOBS: -j4
         SBCL_MAKE_TARGET_2_OPTIONS: --disable-ldb --disable-debugger
-      run: ./make.sh ${{ matrix.options }} --with-sb-core-compression --xc-host='sbcl --lose-on-corruption --disable-ldb --disable-debugger'
+      run: ./make.sh ${{ matrix.options }} --with-sb-core-compression --xc-host='sbcl --lose-on-corruption --disable-ldb --dynamic-space-size 2GB --disable-debugger'
     - name: make binary
       run: |
         name=sbcl-`cat version.lisp-expr | ./run-sbcl.sh --noinform --noprint --eval '(write-line (read))'`-darwin-${{ matrix.arch }}
@@ -37,6 +37,8 @@ jobs:
       with:
         name: sbcl-mac-binary-${{ matrix.arch }}${{ matrix.options }}
         path: sbcl-mac-binary-${{ matrix.arch }}${{ matrix.options }}
+    - name: build documentation
+      run: cd ./doc/manual && sh generate-texinfo.sh
     - name: test
       run: cd tests; ./run-tests.sh --slow
     - name: ansi-test

.github/workflows/windows-arm64.yml

@@ -58,6 +58,8 @@ jobs:
       with:
         name: sbcl-windows-installer-${{ matrix.sys }}
         path: sbcl-windows-installer
+    - name: build documentation
+      run: cd ./doc/manual && sh generate-texinfo.sh
     - name: tests
       working-directory: tests
       run: ./run-tests.sh

.github/workflows/windows.yml

@@ -49,12 +49,11 @@ jobs:
       with:
         name: sbcl-windows-installer-${{ matrix.sys }}
         path: sbcl-windows-installer
+    - name: build documentation
+      run: cd ./doc/manual && sh generate-texinfo.sh
     - name: tests
       working-directory: tests
       run: ./run-tests.sh
     - name: ansi-tests
       working-directory: tests
       run: ./ansi-tests.sh
-        
-      
-    

.gitignore

@@ -11,42 +11,42 @@
 *.diff
 *.patch
 .gdb_history
-output
-obj
-local-target-features.lisp-expr
-customize-target-features.*
-customize-backend-subfeatures.*
-src/assembly/target
-src/compiler/assembly
-src/compiler/target
-src/runtime/Config
-src/runtime/TAGS
-src/runtime/genesis
-src/runtime/openbsd-sigcontext.h
-src/runtime/sbcl
-src/runtime/ldb
-src/runtime/sbcl.exe
-src/runtime/sbcl.mk
-src/runtime/shrinkwrap-sbcl*
-src/runtime/target-arch-os.h
-src/runtime/target-arch.h
-src/runtime/target-lispregs.h
-src/runtime/target-os.h
-src/runtime/embedcore-sbcl
-tests/test-status.lisp-expr
-tests/test.log
-tests/*.so
-tests/run-tests-*
-tests/last-random-state.lisp-expr
-tests/ansi-test/
-tools-for-build/avx2*
-tools-for-build/determine-endianness
-tools-for-build/determine-endianness.exe
-tools-for-build/grovel-headers
-tools-for-build/grovel-headers.exe
-tools-for-build/mmap-rwx
-tools-for-build/where-is-mcontext
-tools-for-build/perfecthash*
-contrib/asdf/asdf-upstream
-doc/manual/*.html
-version.lisp-expr
+/output
+/obj
+/local-target-features.lisp-expr
+/customize-target-features.*
+/customize-backend-subfeatures.*
+/src/assembly/target
+/src/compiler/assembly
+/src/compiler/target
+/src/runtime/Config
+/src/runtime/TAGS
+/src/runtime/genesis
+/src/runtime/openbsd-sigcontext.h
+/src/runtime/sbcl
+/src/runtime/ldb
+/src/runtime/sbcl.exe
+/src/runtime/sbcl.mk
+/src/runtime/shrinkwrap-sbcl*
+/src/runtime/target-arch-os.h
+/src/runtime/target-arch.h
+/src/runtime/target-lispregs.h
+/src/runtime/target-os.h
+/src/runtime/embedcore-sbcl
+/tests/test-status.lisp-expr
+/tests/test.log
+/tests/*.so
+/tests/run-tests-*
+/tests/last-random-state.lisp-expr
+/tests/ansi-test/
+/tools-for-build/avx2*
+/tools-for-build/determine-endianness
+/tools-for-build/determine-endianness.exe
+/tools-for-build/grovel-headers
+/tools-for-build/grovel-headers.exe
+/tools-for-build/mmap-rwx
+/tools-for-build/where-is-mcontext
+/tools-for-build/perfecthash*
+/contrib/asdf/asdf-upstream
+/doc/manual/*.html
+/version.lisp-expr

.mailmap

@@ -20,6 +20,8 @@ David Lichteblau <david@lichteblau.com> <david@knowledgetools.de>
 
 Francois-Rene Rideau <tunes@google.com> <fare@tunes.org>
 
+Gabor Melis <mega@retes.hu> <mega@hotpop.com>
+
 Jim Wise <jimwise@users.sourceforge.net>
 Jim Wise <jimwise@users.sourceforge.net> <jimwise>
 <jimwise@users.sourceforge.net> <jwise@draga.com>

BUGS

@@ -14,4 +14,8 @@ Historical note: before Launchpad was adopted this file contained a
 list of currently open bugs. If you run into an SBCL bug number in the
 range 1-431 inclusive, it refers to that list.
 
-Refer to User Manual for more details.
+Refer to the User Manual for more details at
+
+  https://www.sbcl.org/manual/#Reporting-Bugs
+
+or SB-MANUAL:@REPORTING-BUGS.

CREDITS

@@ -168,7 +168,7 @@ Guy Steele wrote the original character functions
 	code/char.lisp
 They were subsequently rewritten by David Dill, speeded up by Scott
 Fahlman, and rewritten without fonts and with a new type system by Rob
-MachLachlan.
+MacLachlan.
 
 Lee Schumacher made the Spice Lisp version of backquote. The comment
 in the CMU CL sources suggests he based it on someone else's code for
@@ -695,8 +695,8 @@ Dave McDonald:
 Gabor Melis:
   He mainly worked on robustness related to signal handling, threads,
   timers with small excursions to constraint propagation, weak hash
-  tables (based on CMUCL code) and optimizing x86/x86-64 calling
-  convention.
+  tables (based on CMUCL code), adaptive hash tables, and optimizing
+  x86/x86-64 calling convention.
 
 Perry E. Metzger:
   He ported SBCL to NetBSD with newer signals, building on the

DOCUMENTATION

@@ -0,0 +1,26 @@
+Files:
+
+- doc/sbcl.1: man page
+
+- doc/manual/sbcl.{info*,pdf,html}: The user manual. See the "INSTALL"
+  file on how to build them.
+
+A prebuilt manual for the latest release is available for download in
+HTML and PDF formats at <https://www.sbcl.org>.
+
+Sections of the manual are defined and exported from the SB-MANUAL
+package (available after (REQUIRE :SB-MANUAL)). The top-level section
+is SB-MANUAL:@SBCL-MANUAL. You can browse these directly (e.g. with
+Slime's M-.) if the SBCL sources are available. Note that all other
+formats (including the intermediate Texinfo) are generated from these
+sections and the docstrings of individual Lisp definitions (of e.g.
+functions, variables).
+
+> An alternative, unofficial (and unsupported by the SBCL project)
+> rendering of the manual is available at <https://fixnum.com> in
+> HTML, PDF, Markdown and plain text for the latest development
+> version. This version is heavily linked both internally and to the
+> HyperSpec, and documents e.g. the default values of arguments and
+> the initial values of variables. It is generated with MGL-PAX v0.5+
+> (<https://fixnum.com/pax-manual.html>), which also supports browsing
+> the documentation live.

HACKING

@@ -3,7 +3,7 @@ SBCL Hacking Guide
   (This is not a most actively maintained file, but recommended
   reading anyways.)
 
-Table of Contests
+Table of Contents
 
 * Modifying
 * Patch Submissions
@@ -49,7 +49,8 @@ We aren't always as well-educated as we'd like to be...
 
 Ready-to-apply patches should be submitted via Launchpad: please add
 the tag "review" to the associated bug (create new bug with name if
-there isn't one about the issue yet.)
+there isn't one about the issue yet). Alternatively, they may be sent
+to the sbcl-bugs mailing list.
 
 Patches requiring more widespread discussion and feedback should be
 sent to the sbcl-devel mailing list.

INSTALL

@@ -127,7 +127,7 @@ INSTALLING SBCL
   If you don't already have an SBCL binary installed as "sbcl" on your
   system, you'll need to tell make.sh what Lisp to use as the
   cross-compilation host (cf. make.sh for detailed instructions on
-  how to cross compile). For example, to use CMUCL (assuming has
+  how to cross compile). For example, to use CMUCL (assuming it has
   been installed under its default name "lisp") as the
   cross-compilation host:
 
@@ -145,10 +145,10 @@ INSTALLING SBCL
 
     $ cd ./doc/manual && make
 
-  This builds the Info, HTML and PDF documentation from the Texinfo
-  sources. The manual includes documentation string from the build
-  SBCL, but if SBCL itself has not been yet built, but one if found
-  installed documentation strings from the installed version are used.
+  This builds the Info, HTML and PDF documentation from the SB-MANUAL
+  contrib. The manual includes documentation strings from the built
+  SBCL. If SBCL itself has not been built yet, but an installed one is
+  found, documentation strings from the installed version are used.
 
   Now you should have the same src/runtime/sbcl and output/sbcl.core
   files that come with the binary distribution, and you can install
@@ -230,7 +230,7 @@ INSTALLING SBCL
       files under "src/runtime", down- or upgrading GCC may help.
 
     * Ask for help on the mailing lists referenced from
-      <http://www.sbcl.org/>.
+      <https://www.sbcl.org/>.
 
 2.4. Tracking SBCL sources
 
@@ -288,9 +288,7 @@ INSTALLING SBCL
     by e.g. testing during the monthly freeze periods, and most
     importantly by reporting any problems.
 
-    For further support, see Getting Support and Reporting Bugs
-    in the manual, or
+    For further support, see "Getting Support and Reporting Bugs"
+    (SB-MANUAL:@SUPPORT-AND-BUGS) in the manual locally or at
 
-      http://www.sbcl.org/manual/Getting-Support-and-Reporting-Bugs.html
-
-    if you do not have the manual for some reason.
+      https://www.sbcl.org/manual/#Getting-Support-and-Reporting-Bugs

NEWS

@@ -1,4 +1,257 @@
 ;;;; -*- coding: utf-8; fill-column: 78 -*-
+
+changes in sbcl-2.6.8 relative to sbcl-2.6.7:
+  * minor incompatible change: FILE-POSITION on composite streams returns a
+    more accurate result for CONCATENATED-STREAM, and NULL on TWO-WAY-STREAM
+    and ECHO-STREAM.
+  * platform support:
+    ** improvements to code generation for the ARM64 backend.
+    ** improved support for AVX512 instruction encoding and decoding.  (thanks
+       to Arthur Miller)
+  * various enhancements to the SB-SIMD contributed module.  (thanks to Sylvia
+    Harrington and Arthur Miller)
+  * bug fix: FILE-POSITION on a stream after reading input into a list or
+    generic sequence returns the correct result.
+  * bug fix: allow the manual to build on more platforms.  (lp#2162021)
+  * bug fix: infinite loop in the compiler while attempting to remove data
+    moves.  (lp#2162597)
+  * bug fix: debugger failed to handle (COMPLEX DOUBLE-FLOAT) entities on the
+    stack.  (reported by Clint Seinen)
+  * bug fix: error reporting from STRING-TO-OCTETS reported using the wrong
+    string indices.  (lp#2164924)
+  * optimization: the compiler performs assignment conversion in nearly all
+    possible cases now, along the lines of Fluet and Weeks' "Contification
+    using Dominators" of 2001., which transforms state-machine-like code
+    written in terms of local functions into the equivalent tagbody/go control
+    structures.
+  * optimization: type inference of parameters is now possible for local
+    functions which recursively call themselves.  (lp#486416, reported by
+    Tobias Rittweiler)
+  * optimization: special-case DPB with newbyte of 0 or -1.
+  * optimization: more improvements to UTF8 encoding and decoding.
+  * optimization: multiple structure TYPEP tests in sequence can elide
+    redundant memory loads of layouts.
+  * optimization: transform ZEROP of (LDB ...) to an equivalent zero test with
+    a mask.  (lp#2162824, thanks to Vasily Postnicov)
+
+changes in sbcl-2.6.7 relative to sbcl-2.6.6:
+  * new contrib module: SB-MANUAL contains the SBCL manual in docstrings of
+    section definitions, which tie together the docstrings of normal Lisp
+    definitions. The manual can thus be explored interactively in the usual
+    way (e.g. with Slime's M-.), and it is browsable with the MGL-PAX library
+    (out of tree). Also, https://fixnum.com/ (similarly unrelated to the SBCL
+    project) provides alternative renderings of the SBCL manual as heavily
+    linked PDF and HTML documents as well as Markdown and plain text.
+  * new feature: DOCUMENTATION supports DOC-TYPE DECLARATION.
+  * platform support:
+    ** the SB-SIMD contrib now supports ARM64.  (Thanks to Sylvia Harrington)
+    ** AVX512 instructions are now supported on X86-64.  (Thanks to Robert
+       Smith and Arthur Miller)
+    ** additional support for SIMD instructions on ARM64 and X86-64.  (Thanks
+       to Arthur Miller)
+    ** fix miscompilation of SAP-REF-N on ARM64.  (Thanks to Hayley Patton)
+    ** implement INTEGER-LENGTH on primitive types without a loop on MIPS and
+       LoongArch.
+  * bug fix: READing with *READ-SUPPRESS* T no longer emits warnings like
+    "<internal-feature> no longer present on *FEATURES*".
+  * bug fix: compiler type-error when compiling calls to CONCATENATE with
+    conditional known non-sequence arguments.  (lp#2160747)
+  * bug fix: (EQL <complex>) types were not being treated as numeric by the
+    type system.  (lp#2160429)
+  * bug fix: improve the handling of quiet (non-signalling) NaN inputs to LOG.
+    (lp#2160268, reported by Woodrow Kiang)
+  * bug fix: miscompilation of MULTIPLE-VALUE-CALL.  (lp#2160207, reported by
+    Vasily Postnicov)
+  * optimization: passing constant complex numbers to local functions can be
+    done without consing.
+  * optimization: where available, use enhanced SIMD routines for UTF-8
+    conversions.
+  * optimization: compiler transforms of COUNT are applicable with a wider
+    variety of keyword arguments.
+  * optimization: remove at least one redundant instruction from SB-ALIEN:DEREF.
+  * optimization: the sparse set implementation in the compiler has been tuned
+    to improve performance on real-world workloads.
+  * documentation: many typos and typesetting issues were fixed.
+  * documentation: SB-MANUAL:@FOREIGN-FUNCTION-INTERFACE now correctly states
+    that arrays are row-major (not column-major).  (lp#2158033, thanks to Scott
+    L. Burson)
+  * documentation: internally, docstrings now conform to a subset of Markdown,
+    but DOCUMENTATION (and thus DESCRIBE) strips some of this markup. The
+    official manual is still generated from Texinfo, but the Texinfo files are
+    generated from SB-MANUAL.
+  * documentation: the manual now has a separate index for declarations.
+
+changes in sbcl-2.6.6 relative to sbcl-2.6.5:
+  * minor incompatible change: FDEFINITION now returns the outermost wrapper
+    (added e.g. by TRACE, PROFILE) like SYMBOL-FUNCTION.  (lp#799533)
+  * minor incompatible change: in unsafe code, C strings with :EXTERNAL-FORMAT
+    :ASCII are copied directly as byte-sized quantities without checking
+    whether the top bit of the byte is set.
+  * platform support:
+    ** fix the build on big-endian 64-bit PowerPC with ELFv2.  (thanks to
+       Piotr Kubaj)
+    ** move the static space address for macOS 27 on ARM64.  (lp#2156072,
+       reported by Gary Palter)
+    ** optimizations to SB-THREAD:BARRIER for ARM64.  (thanks to Sahil Kang)
+    ** fix a compiler crash in MULTIPLE-VALUE-LIST in argument forms on ARM64.
+       (lp#2155788, reported by Gary Palter)
+  * bug fix: TRACE no longer fails when trying to print a return value that
+    cannot be printed readably and *PRINT-READABLY* is true.
+  * optimization: the compiler is more precise in its type derivation of
+    COERCE given constraints on its inputs.
+  * optimization: the compiler is better able to derive the return types of
+    AREF and ELT.
+  * optimization: faster encoding and decoding of UTF-8 C strings.
+  * optimization: (length (intersection a b)) doesn't cons an intermediate
+    list.
+  * documentation: the manual now includes a section for SB-INTROSPECT, which
+    has also seen improvement in its documentation strings and comments.
+  * documentation: fixed many typesetting problems and typos in the user
+    manual.
+
+changes in sbcl-2.6.5 relative to sbcl-2.6.4:
+  * minor incompatible change: the condition signalled when an accessed slot
+    is missing from an object is no longer a TYPE-ERROR.
+  * minor incompatible change: the condition signalled when accessing an
+    uninitialized structure slot is no longer a TYPE-ERROR.
+  * minor incompatible change: the implementations of standardized functions
+    treating lists as sets, such as INTERSECTION and UNION, take more
+    advantage of the freedom to return the elements of the result in any
+    order.
+  * platform support:
+    ** add low-level support for floating point state manipulation on
+       PPC64/FreeBSD.  (thanks to Piotr Kubaj)
+    ** improve the software emulation of displaced instructions on ARM64.
+    ** restore building the system using the musl C library.  (lp#2153432,
+       reported by Tom Gillespie)
+    ** fix some SB-SIMD shifting instructions on AVX2.  (lp#2152791, reported
+       by Willem Broekema)
+  * enhancement: definition sources for alien callbacks are now findable by
+    name in SB-INTROSPECT.
+  * bug fix: the SYMBOL restart for finding packages now actually performs a
+    non-local exit.  (lp#2153092, reported by Zach Shaftel)
+  * bug fix: TYPEP on large disjoint numeric union types compiles faster using
+    fewer resources.  (lp#2151818, reported by James McDonald)
+  * bug fix: strings of arbitrary size with fill-pointer set to 1 are
+    character designators.  (reported by _death)
+  * bug fix: the KEEP-OLD restart established by ADD-PACKAGE-LOCAL-NICKNAME
+    keeps the old nickname instead of going ahead with the change (and the
+    restart report function no longer returns from ADD-PACKAGE-LOCAL-NICKNAME).
+  * bug fix: when EXPORT results in a conflict between symbols exported by
+    different used packages, the TAKE-NEW restart now shadowing-imports the
+    new symbol instead of doing nothing and leaving the package in an
+    inconsistent state.
+  * bug fix: the SB-EVAL interpreter checks program syntax more thoroughly.
+  * bug fix: compiler cross-reference data is decoded correctly for a
+    functional with more than one entry for a given name.
+  * bug fix: TYPE-ERRORs signalled by SBCL are more likely to have a DATUM
+    that is not of the condition's EXPECTED-TYPE.
+  * bug fix: the code walker no longer uses the stack to walk PROGN bodies.
+  * optimization: in various situations, INTERSECTION and UNION will use
+    hash-tables to perform the operation more quickly.
+
+changes in sbcl-2.6.4 relative to sbcl-2.6.3:
+  * minor incompatible change: when DEFSETF is called on a name that was
+    previously used as a (presumed) call to a function, it issues a single
+    STYLE-WARNING (like DEFMACRO).
+  * minor incompatible change: SB-EXT:PROCESS-KILL no longer supports the
+    :PTY-PROCESS-GROUP option (which was never correctly implemented).
+  * minor incompatible change: the :INITIAL-OFFSET argument for typed
+    DEFSTRUCT, if given, no longer accepts NIL.
+  * platform support:
+    ** more likely to support 32-bit linux's struct timeval with 64-bit time_t.
+    ** the runtime's main function is now defined as a weak symbol for
+       platform/compiler combinations that support it.
+    ** on Windows, individual empty arguments for RUN-PROGRAM are escaped.
+    ** add input/output speed fields for our definition of the termios
+       structure, addressing a crash in sb-posix:tcsetattr on OpenBSD.
+       (lp#2150499, thanks to Robert Palm)
+  * bug fix: address infinite loops in the compiler.  (lp#2144911, lp#2148056)
+  * bug fix: if an FTYPE has been proclaimed for a function, don't mix NULL
+    with explicitly-typed keyword arguments.  (lp#2147050, reported by Vasily
+    Postnicov)
+  * bug fix: compiler error from treatment of unused results.  (lp#2147383)
+  * bug fix: compiler error from invalid dimension arguments to MAKE-ARRAY.
+    (lp#2147384)
+  * bug fix: compiler error arising from continuing to optimize deleted nodes.
+    (lp#2147385)
+  * bug fix: make sure linkage-table alien entries have base-string names.
+    (lp#2147646, thanks to Seokjun Lee)
+  * bug fix: make sure CHECK-TYPE's expansion does not include internal
+    non-externalizable objects.  (lp#2148777, reported by Willem Broekema)
+  * bug fix: alien calls involving passing structs by value are less likely to
+    read or write wrong areas of memory.  (thanks to Jesse Bouwman)
+  * bug fix: lowering EQUALP to EQUAL handles constant dotted lists correctly.
+  * bug fix: a number of standard functions perform more explicit type checks
+    on their arguments.
+  * bug fix: only return the primary value from (LIST*/APPEND/NCONC <values>).
+  * bug fix: correct treatment of escaped closing brackets in pathname
+    patterns.
+  * bug fix: escape dots in pathnames more correctly.
+  * bug fix: the hash set for function names will incur collisions less
+    frequently.  (reported by Andrew Wolven)
+  * bug fix: the system is now capable of expressing select() on the whole
+    range of possible file descriptors.
+  * optimization: compiler optimizations for (REPLACE vector list) now apply
+    given :START1 and/or :END1 keyword arguments.
+  * optimization: CONCATENATE is faster for concatenating list arguments to a
+    VECTOR.
+  * optimization: ROUND for integers is more compact.
+  * optimization: on 64-bit x86, implement TRUNCATE using the Lemire, Kaser,
+    Kurz transform.
+
+changes in sbcl-2.6.3 relative to sbcl-2.6.2:
+  * minor incompatible change: (MAKE-ARRAY X :ELEMENT-TYPE 'UNDEFINED) now
+    signals an error, consistent with (UPGRADED-ARRAY-ELEMENT-TYPE
+    'UNDEFINED).
+  * platform support:
+    ** fix disassembler on ppc for the MFLR and ISEL instructions
+    ** the Lisp Return Address object (as part of the Lisp calling convention)
+       is no longer needed or supported on PPC, SPARC, MIPS or ARM.  (This
+       also frees up a widetag slot previously held by return-pc-widetag)
+    ** remove sensitivity to SBCL init files when building embedcore-sbcl.
+       (thanks to Robert Brown)
+    ** add support for the ADCX and ADOX instructions on x86-64.  (thanks to
+       Robert Smith)
+    ** on PPC64, indicate the number of return values through flags, making
+       function calls four times faster.
+    ** fix FFI involving int128 arguments on x86-64.  (thanks to Andreas
+       Franke)
+    ** fix build on OpenIndiana/x86-64.  (thanks to Andreas Wacknitz)
+    ** fix build on Haiku/x86-64.
+  * bug fix: improved stability of (particularly) the mark-region garbage
+    collector.  (lp#2142711)
+  * bug fix: compiler type error in complicated expression involving BOOLE and
+    conditionals.  (lp#2142949)
+  * bug fix: dotted lists involving symbols whose names begins with "DEF" are
+    not definitions.  (lp#2143114, reported by Glenn Thompson)
+  * bug fix: STABLE-SORT miscompiled on declared union types involving both
+    LIST and VECTOR.  (lp#2143163, reported by akater, thanks to Vasily
+    Postnicov)
+  * bug fix: more consistent results between compiler and interpreter in
+    numerical computations involving negative zeros.  (lp#2143383)
+  * bug fix: USE-PACKAGE once again signals the correct error if an attempt is
+    made to use the KEYWORD package.
+  * bug fix: EQUALP on hash tables is no longer sensitive to irrelevant
+    aspects of the hash table.
+  * bug fix: SB-INTROSPECT:DEFTYPE-LAMBDA-LIST is more robust against types
+    defined in low debug situations.
+  * bug fix: ENSURE-GENERIC-FUNCTION ensures that the allocation of a generic
+    function does not happen in an arena.  (Thanks to Andreas Franke)
+  * optimization: actually return early when we hit the cache for a
+    :MAKUNBOUND slot access.  (thanks to John Mallery)
+  * optimization: streams opened with WITH-OPEN-FILE avoid having finalizers.
+  * optimization: improvement of COUNT on non-simple bit-vectors, or simple
+    ones with :START/:END arguments.  (lp#2142062, thanks to Andrew Berkley)
+  * optimization: SIMD routines for checking strings for ASCII-only content
+    are more compact.
+  * optimization: the reader prefers base-string symbol-names of uninterned
+    symbols if possible.
+  * optimization: (length (remove-duplicates a s)) doesn't cons an
+    intermediate sequence.
+  * optimization: REMOVE-DUPLICATES uses hash-tables in more situations.
+  * optimization: UPGRADED-ARRAY-ELEMENT-TYPE is now faster.
+
 changes in sbcl-2.6.2 relative to sbcl-2.6.1:
   * minor incompatible change: IMAGPART of a negative float returns 0.0, not
     -0.0, consistent with a treatment of reals as complexes with an imaginary

README

@@ -9,19 +9,17 @@ To find out more about who created the system, see the "CREDITS" file.
 If you'd like information about the legalities of copying the system,
 see the "COPYING" file.
 
-If you'd like more information about using the system, see the man
-page, "sbcl.1", or the user manual in the "doc/manual" subdirectory of
-the distribution. (The user manual is maintained as Texinfo in the
-source distribution; HTML version is available for download, and
-"INSTALL" describes how to build the Texinfo version in HTML and PDF.)
+The "DOCUMENTATION" file describes the various formats and ways to
+access the documentation.
 
 The system is a work in progress. See the "TODO" file in the source
 distribution for some highlights.
 
 See the "BUGS" file for how to view or report bugs.
 
-If you'd like to make suggestions, report a bug, or help to improve the
-system, please send mail to one of the mailing lists:
+If you'd like to make suggestions or help to improve the system,
+please send mail to one of the mailing lists:
+
   sbcl-help@lists.sourceforge.net
   sbcl-devel@lists.sourceforge.net
 Note that as a spam reduction measure you must subscribe to the lists

TLA

@@ -3,8 +3,8 @@
 abbreviations we try to use pervasively in the system
 
 As Dan Barlow pointed out long ago on the mailing list, unabbreviated
-names are easier to deal with than abbreviated names, because you
-never need to remember what abbreviation to use. That's true, but in a
+names are easier to deal with than abbreviated names because you never
+need to remember what abbreviation to use. That's true, but in a
 language like Lisp which depends on compound names for important
 things like structure accessors, that can lead to painful names like
 MAKE-EXTERNAL-ENTRY-POINT-LAMBDA-EXPRESSION and associated indenting
@@ -30,8 +30,8 @@ making them more consistent.
 	SB	storage base (in compiler IR2)
 	SC	storage class (in compiler IR2)
 	TN	temporary name (?) (in compiler IR2)
-	VAR	variable (in the lisp entity "noun" sense, not in the 
+	VAR	variable (in the lisp entity "noun" sense, not in the
 			adjectival sense)
 	XEP	external entry point
-Making them even more consistent (within the limits of ANSI and 
+Making them even more consistent (within the limits of ANSI and
 MOP compatibility) would probably be good.

TODO

@@ -65,10 +65,7 @@ ADVICE/FWRAP
   SBCL has an internal function encapsulation mechanism, and is able to
   install breakpoint to function start/end -- this is used to implement
   the instrumentation based profiler and tracing. It would be good to
-  have this as an exported interface, and it would be good if the
-  SYMBOL-FUNCTION / FDEFINITION confusion was fixed: currently the
-  latter returns the underlying definition, whereas the first returns
-  the encapsulation.
+  have this as an exported interface.
 
 POLICY MADNESS
 

benchmarks/rwlbench2.lisp

@@ -0,0 +1,102 @@
+(defpackage :sbcl-lock-bench
+  (:use :cl :sb-thread)
+  (:shadow #:spinlock #:with-spinlock)
+  (:export :compare-locks))
+
+(in-package :sbcl-lock-bench)
+
+(defparameter *time-wasting-iterations-read* 100)
+(defparameter *time-wasting-iterations-write* 150)
+
+;;; spinlock
+(defstruct (spinlock)
+  (state 0 :type sb-ext:word)) ; 0 = unlocked, 1 = locked
+
+(defun spin-lock (lock)
+  (declare (optimize (speed 3) (safety 0)))
+  (loop (if (eql 0 (sb-ext:cas (spinlock-state lock) 0 1))
+            (return t)
+            (sb-thread:thread-yield))))
+
+(declaim (inline spin-unlock))
+(defun spin-unlock (lock)
+  (declare (optimize (speed 3) (safety 0)))
+  (setf (spinlock-state lock) 0))
+
+(defmacro with-spinlock ((lock) &body body)
+  `(progn
+     (spin-lock ,lock)
+     (unwind-protect (progn ,@body)
+       (spin-unlock ,lock))))
+
+(defun run-spinlock-worker (lock operations write-percent)
+  (let* ((my-random-state (make-random-state t))
+         (r (random 100 my-random-state)))
+    (dotimes (i (the fixnum operations))
+      ;; readers and writers use the same lock mechanism
+      (with-spinlock (lock)
+        (if (< r write-percent)
+            (loop repeat *time-wasting-iterations-write*)
+            (loop repeat *time-wasting-iterations-read*))
+        ;; and do some some work inside the lock
+        (setq r (random 100 my-random-state))))))
+
+;;; spinlock-based rwlock
+(defmacro with-rwlock-read ((lock) &body body)
+  `(progn (rwspinlock-rdlock ,lock)
+          (multiple-value-prog1 (progn ,@body) (rwspinlock-rdunlock ,lock))))
+
+(defmacro with-rwlock-write ((lock) &body body)
+  `(progn (rwspinlock-wrlock ,lock)
+          (multiple-value-prog1 (progn ,@body) (rwspinlock-wrunlock ,lock))))
+
+(defun run-rwlock-worker (lock operations write-percent)
+  (let* ((my-random-state (make-random-state t))
+        (r (random 100 my-random-state)))
+    (dotimes (i (the fixnum operations))
+      (if (< r write-percent)
+          (with-rwlock-write (lock)
+            (loop repeat *time-wasting-iterations-write*)
+            (setq r (random 100 my-random-state)))
+          (with-rwlock-read (lock)
+            (loop repeat *time-wasting-iterations-read*)
+            (setq r (random 100 my-random-state)))))))
+
+;; runner
+(defun execute-bench (lock-type operations threads write-percent)
+  (let ((threads-list '())
+        (start-time (get-internal-real-time))
+        (lock (if (eq lock-type :rwlock)
+                  (sb-thread::make-rw-spinlock)
+                  (make-spinlock))))
+
+    (dotimes (i threads)
+      (push (sb-thread:make-thread
+             (lambda ()
+               (if (eq lock-type :rwlock)
+                   (run-rwlock-worker lock operations write-percent)
+                   (run-spinlock-worker lock operations write-percent))))
+            threads-list))
+
+    (mapc #'sb-thread:join-thread threads-list)
+
+    (let* ((end-time (get-internal-real-time))
+           (elapsed (/ (- end-time start-time) internal-time-units-per-second)))
+      elapsed)))
+
+(defun compare-locks (&key (threads 4) (ops-per-thread 100000))
+  (format t "~%================================================~%")
+  (format t "Comparing spinlock-based RWLOCK vs MUTEX~%")
+  (format t "Threads: ~D | Ops/Thread: ~D~%" threads ops-per-thread)
+  (format t "================================================~%~%")
+  (format t "~10A | ~15A | ~15A | ~A~%" "Write %" "RW-Lock Time" "Mutex Time" "Speedup (RW / Mutex)")
+  (format t "------------------------------------------------------------------~%")
+
+  (dolist (pct '(0 1 2 3 5 10 20 30 40 50 60 70 80 90 100))
+    (let ((rw-time (execute-bench :rwlock ops-per-thread threads pct))
+          (mutex-time (execute-bench :mutex ops-per-thread threads pct)))
+      (format t "~10D | ~14,3Fs | ~14,3Fs | ~,2Fx~%"
+              pct
+              rw-time
+              mutex-time
+              (/ mutex-time rw-time))))) ; Speedup > 1.0x means RWLock was faster

benchmarks/utf8-to-string.lisp

@@ -0,0 +1,125 @@
+(defun calc-et (from-sec from-ns to-sec to-ns)
+  (let ((from (floor (+ (* 1000000000 from-sec) from-ns) 1000))
+        (to   (floor (+ (* 1000000000 to-sec)   to-ns)   1000)))
+    (- to from)))
+
+(defmacro my-timing (form)
+  (let ((clockid #+linux sb-unix:clock-thread-cputime-id
+                 #+darwin sb-unix:clock-process-cputime-id))
+    `(sb-int:binding* (((sec-before nsec-before) (sb-unix:clock-gettime ,clockid))
+                       (nil ,form)
+                       ((sec-after nsec-after)  (sb-unix:clock-gettime ,clockid)))
+       (calc-et sec-before nsec-before sec-after nsec-after))))
+
+(defparameter *validate* nil)
+(defun compare (testcases &optional (validate *validate*))
+  (declare (simple-vector testcases))
+  (declare (optimize speed))
+  (declare (notinline sb-ext:octets-to-string)) ; because unsafely flushable
+  (when validate ; make sure they agree
+    (sb-int:dovector (x testcases)
+      (let ((way1 (sb-unicode:utf8-decode-from-octets x))
+            (way2 (sb-ext:octets-to-string x)))
+        (assert (string= way1 way2)))))
+  ;; These tests cons a lot and can be easily skewed by having very random
+  ;; places at which they GC.  We get fairly consistent results when both the
+  ;; baseline and experiment have a manual GC done beforehand.
+  (gc)
+  ;; run the new way first to give the benefit-of-doubt to the old way
+  ;; in terms of bringing memory into L1 cache
+  (let ((et-new (my-timing
+                 (sb-int:dovector (x testcases) (sb-unicode:utf8-decode-from-octets x)))))
+    (gc)
+    (let ((et-old (my-timing
+                   (sb-int:dovector (x testcases) (sb-ext:octets-to-string x)))))
+      (format t "~D ~D (~f%)~%" et-old et-new (* 100 (/ (- et-new et-old) et-old))))))
+
+(defun random-string (stringlen percent-ascii &aux (unicode (- 100 percent-ascii)))
+  (let ((s (make-string stringlen)))
+    (dotimes (i stringlen s)
+      (setf (char s i)
+            (code-char (if (< (random 100.0) unicode)
+                           (loop (let ((c (max 1 (random char-code-limit))))
+                                   (when (sb-unicode:scalar-p c) (return c))))
+                           (max 1 (random 128))))))))
+
+(defun bench ()
+  (dolist  (stringlen '(5 100 1000 10000 1000000))
+    (dolist (percent-ascii '(100 99 98 97 96 95 90 80 70 60 50 40))
+      (let ((n-trials (ceiling 10000000 stringlen)))
+        (format t "~&~3D% ASCII, length=~d [~d iterations]: " percent-ascii stringlen n-trials)
+        (force-output)
+        (let ((testcases
+               (coerce
+                (loop repeat n-trials
+                      collect
+                      (string-to-octets (random-string stringlen percent-ascii)))
+                'vector)))
+          (compare testcases))))))
+
+#|
+My results on an x86-64 macbook (negative percent diff means new is is better)
+and the deltas are either very similar or show slightly less of an improvement
+for arm64 macbook, but still always an improvement over the baseline.
+
+100% ASCII, length=5 [2000000 iterations]: 426481 89626 (-78.984764%)
+ 99% ASCII, length=5 [2000000 iterations]: 469787 60676 (-87.08436%)
+ 98% ASCII, length=5 [2000000 iterations]: 471085 65415 (-86.11397%)
+ 97% ASCII, length=5 [2000000 iterations]: 488598 68924 (-85.89352%)
+ 96% ASCII, length=5 [2000000 iterations]: 482445 74787 (-84.49834%)
+ 95% ASCII, length=5 [2000000 iterations]: 481117 77478 (-83.896225%)
+ 90% ASCII, length=5 [2000000 iterations]: 491657 87709 (-82.16053%)
+ 80% ASCII, length=5 [2000000 iterations]: 515252 103540 (-79.904976%)
+ 70% ASCII, length=5 [2000000 iterations]: 539165 116004 (-78.484505%)
+ 60% ASCII, length=5 [2000000 iterations]: 569675 136532 (-76.033356%)
+ 50% ASCII, length=5 [2000000 iterations]: 589356 143996 (-75.56723%)
+ 40% ASCII, length=5 [2000000 iterations]: 613356 144063 (-76.51234%)
+100% ASCII, length=100 [100000 iterations]: 140443 3974 (-97.17038%)
+ 99% ASCII, length=100 [100000 iterations]: 149191 50945 (-65.8525%)
+ 98% ASCII, length=100 [100000 iterations]: 139345 77090 (-44.67688%)
+ 97% ASCII, length=100 [100000 iterations]: 154108 74434 (-51.700108%)
+ 96% ASCII, length=100 [100000 iterations]: 144111 87651 (-39.178135%)
+ 95% ASCII, length=100 [100000 iterations]: 162271 79519 (-50.996174%)
+ 90% ASCII, length=100 [100000 iterations]: 157988 92868 (-41.21832%)
+ 80% ASCII, length=100 [100000 iterations]: 196840 91016 (-53.76143%)
+ 70% ASCII, length=100 [100000 iterations]: 209938 108707 (-48.219475%)
+ 60% ASCII, length=100 [100000 iterations]: 248108 109260 (-55.962727%)
+ 50% ASCII, length=100 [100000 iterations]: 254467 121921 (-52.087696%)
+ 40% ASCII, length=100 [100000 iterations]: 285208 114461 (-59.867535%)
+100% ASCII, length=1000 [10000 iterations]: 105570 1546 (-98.53557%)
+ 99% ASCII, length=1000 [10000 iterations]: 112052 87494 (-21.91661%)
+ 98% ASCII, length=1000 [10000 iterations]: 120597 77042 (-36.116154%)
+ 97% ASCII, length=1000 [10000 iterations]: 133759 75267 (-43.729393%)
+ 96% ASCII, length=1000 [10000 iterations]: 124279 76452 (-38.483574%)
+ 95% ASCII, length=1000 [10000 iterations]: 145278 76784 (-47.14685%)
+ 90% ASCII, length=1000 [10000 iterations]: 141478 81126 (-42.658222%)
+ 80% ASCII, length=1000 [10000 iterations]: 177988 89134 (-49.921345%)
+ 70% ASCII, length=1000 [10000 iterations]: 190698 99088 (-48.039307%)
+ 60% ASCII, length=1000 [10000 iterations]: 240096 108468 (-54.82307%)
+ 50% ASCII, length=1000 [10000 iterations]: 241091 113381 (-52.9717%)
+ 40% ASCII, length=1000 [10000 iterations]: 274307 113896 (-58.478638%)
+100% ASCII, length=10000 [1000 iterations]: 116469 1521 (-98.69408%)
+ 99% ASCII, length=10000 [1000 iterations]: 122827 84726 (-31.020052%)
+ 98% ASCII, length=10000 [1000 iterations]: 125854 85905 (-31.742336%)
+ 97% ASCII, length=10000 [1000 iterations]: 144160 76104 (-47.208656%)
+ 96% ASCII, length=10000 [1000 iterations]: 149835 76644 (-48.847733%)
+ 95% ASCII, length=10000 [1000 iterations]: 149988 79170 (-47.21578%)
+ 90% ASCII, length=10000 [1000 iterations]: 143812 98798 (-31.300587%)
+ 80% ASCII, length=10000 [1000 iterations]: 188148 89944 (-52.19508%)
+ 70% ASCII, length=10000 [1000 iterations]: 214511 129084 (-39.824066%)
+ 60% ASCII, length=10000 [1000 iterations]: 240322 110871 (-53.865646%)
+ 50% ASCII, length=10000 [1000 iterations]: 237560 122167 (-48.574253%)
+ 40% ASCII, length=10000 [1000 iterations]: 272658 114349 (-58.061382%)
+100% ASCII, length=1000000 [10 iterations]: 122355 3950 (-96.77169%)
+ 99% ASCII, length=1000000 [10 iterations]: 127108 70778 (-44.316643%)
+ 98% ASCII, length=1000000 [10 iterations]: 135022 80389 (-40.462296%)
+ 97% ASCII, length=1000000 [10 iterations]: 126778 72913 (-42.487656%)
+ 96% ASCII, length=1000000 [10 iterations]: 144430 74448 (-48.453922%)
+ 95% ASCII, length=1000000 [10 iterations]: 138535 78872 (-43.067097%)
+ 90% ASCII, length=1000000 [10 iterations]: 145037 77884 (-46.3006%)
+ 80% ASCII, length=1000000 [10 iterations]: 190233 89723 (-52.83521%)
+ 70% ASCII, length=1000000 [10 iterations]: 197363 95052 (-51.838997%)
+ 60% ASCII, length=1000000 [10 iterations]: 216470 115180 (-46.791702%)
+ 50% ASCII, length=1000000 [10 iterations]: 244138 108852 (-55.413742%)
+ 40% ASCII, length=1000000 [10 iterations]: 247799 123019 (-50.355328%)
+|#

binary-distribution.sh

@@ -19,6 +19,7 @@ tar -cf $b-binary.tar \
     $b/src/runtime/sbcl.mk \
     `grep '^LIBSBCL=' $b/src/runtime/sbcl.mk | cut -d= -f2- | while read lib; do echo $b/src/runtime/$lib; done` \
     $b/BUGS $b/COPYING $b/CREDITS $b/INSTALL $b/NEWS $b/README \
+    $b/DOCUMENTATION \
     $b/install.sh $b/find-gnumake.sh $b/sbcl-pwd.sh $b/run-sbcl.sh \
     $b/doc/sbcl.1 \
     $b/pubring.pgp \

build-all-cores.sh

@@ -32,7 +32,8 @@
 ;;; This list is: arch-name and ((configuation-name feature ...) ...)
 (defparameter *all-configurations*
   '(("arm"    ("arm" :little-endian :largefile))
-    ("arm64"  ("arm64" :little-endian :sb-thread)
+    ("arm64"  ("arm64" :little-endian :sb-thread :tls-based-mv-return)
+              ("arm64+simd" :little-endian :sb-thread :sb-simd-pack)
               ("arm64-darwin" :darwin :bsd :unix :mach-o :little-endian :sb-thread :darwin-jit)
               ("arm64-reloc"  :little-endian :sb-thread :relocatable-static-space :immobile-space)
               ("arm64-immobile-space" :little-endian :sb-thread :immobile-space))
@@ -49,15 +50,15 @@
               ("x86-ascii" :little-endian :largefile (not :sb-unicode))
               ("x86-thread" :little-endian :largefile :sb-thread)
               ("x86-linux" :little-endian :largefile :sb-thread :linux :unix :elf :sb-thread))
-    ("x86-64" ("x86-64" :little-endian :avx2 :gencgc :sb-simd-pack :sb-simd-pack-256)
-              ("x86-64-linux" :linux :unix :elf :little-endian :avx2 :gencgc :sb-simd-pack :sb-simd-pack-256
-                              (not :sb-eval) :sb-fasteval)
-              ("x86-64-darwin" :darwin :bsd :unix :mach-o :little-endian :avx2 :gencgc
-                               :sb-simd-pack :sb-simd-pack-256)
-              ("x86-64-imm" :little-endian :avx2 :gencgc :sb-simd-pack :sb-simd-pack-256
-                            :immobile-space (not :sb-unicode))
-              ("x86-64-permgen" :little-endian :avx2 :gencgc :sb-simd-pack :sb-simd-pack-256
-                                :permgen))))
+    ("x86-64" ("x86-64" :little-endian :avx2 :avx512 :gencgc :sb-simd-pack :sb-simd-pack-256 :sb-simd-pack-512)
+              ("x86-64-linux" :linux :unix :elf :little-endian :avx2 :avx512 :gencgc :tls-based-mv-return
+                               :sb-simd-pack :sb-simd-pack-256 :sb-simd-pack-512 (not :sb-eval) :sb-fasteval)
+              ("x86-64-darwin" :darwin :bsd :unix :mach-o :little-endian :avx2 :avx512 :gencgc
+                               :sb-simd-pack :sb-simd-pack-256 :sb-simd-pack-512)
+              ("x86-64-imm" :little-endian :avx2 :avx512 :gencgc :sb-simd-pack :sb-simd-pack-256
+                            :sb-simd-pack-512 :immobile-space (not :sb-unicode))
+              ("x86-64-permgen" :little-endian :avx2 :avx512 :gencgc :sb-simd-pack
+                                :sb-simd-pack-256 :sb-simd-pack-512 :permgen))))
 
 (setq sb-ext:*evaluator-mode* :compile)
 (defun interpolate (string substitutions)
@@ -92,7 +93,10 @@
   ((new
     (with-output-to-string (makefile)
       (format makefile
-          "SBCL=src/runtime/sbcl
+          "###################################################
+### THIS FILE IS MACHINE-GENERATED. DO NOT EDIT ###
+###################################################
+SBCL=src/runtime/sbcl
 ARGS=--core output/sbcl.core --noinform --disable-debugger --noprint --no-userinit --no-sysinit
 SCRIPT1=crossbuild-runner/pass-1.lisp
 SCRIPT2=crossbuild-runner/pass-2.lisp
@@ -134,7 +138,7 @@ obj/xbuild/{cfg}.core: obj/xbuild/{cfg}/xc.core
                  "-k"
                  "-fcrossbuild-runner/Makefile"
                  ,@(corefiles))
-               :output t :error t
+               :input t :output t :error t
                :search t))
 (when (= (process-exit-code *process*) 0)
   (load "src/cold/shared" :verbose t)

contrib/Makefile

@@ -3,7 +3,7 @@ vpath %.fasl ../obj/sbcl-home/contrib/
 contribs = sb-posix sb-bsd-sockets sb-introspect sb-cltl2 sb-aclrepl \
      sb-sprof sb-capstone sb-md5 sb-capstone sb-executable sb-gmp sb-mpfr \
      sb-queue sb-rotate-byte sb-rt sb-simple-streams sb-concurrency sb-cover \
-     sb-simd sb-grovel sb-perf asdf
+     sb-simd sb-grovel sb-perf sb-manual asdf
 
 active_contribs = $(filter-out $(SBCL_CONTRIB_BLOCKLIST),$(contribs))
 
@@ -21,6 +21,13 @@ sb-queue.fasl: sb-concurrency.fasl
 sb-simple-streams.fasl: sb-posix.fasl sb-bsd-sockets.fasl
 sb-grovel.fasl: asdf.fasl # for building the module, not for our build
 
+sb_manual_module_deps = sb-aclrepl sb-bsd-sockets sb-cltl2 sb-concurrency \
+    sb-cover sb-grovel sb-introspect sb-md5 sb-posix sb-rotate-byte \
+    sb-queue sb-sprof sb-simd sb-simple-streams
+sb_manual_unblocked_module_deps = \
+    $(filter-out $(SBCL_CONTRIB_BLOCKLIST), $(sb_manual_module_deps))
+sb-manual.fasl: $(sb_manual_unblocked_module_deps:=.fasl)
+
 # Notes:
 # 1. Invoking $(MAKE) for nested make eliminates a warning about jobserver mode.
 # 2. This could be made to 'tee' into output/{something} as it used to,

contrib/README

@@ -3,7 +3,7 @@ core SBCL functionality, or else they'd be built into the main SBCL
 binary automatically. And they're not portable Common Lisp, or they'd
 be put elsewhere (see http://sbcl.sf.net/libs.php for pointers)
 
-There are two kinds of contrib module in this directory:
+There are two kinds of contrib modules in this directory:
 
   * Newer contrib modules conform to the contrib standard (see 
     STANDARDS) and are automatically built and installed along with

contrib/STANDARDS

@@ -1,4 +1,4 @@
-Proposed contrib standard, $Revision$
+Proposed contrib standard
 
 The SBCL contrib mechanism provides a mechanism to
 manage code which does not form part of SBCL itself, but which is
@@ -71,23 +71,16 @@ good place to test that they still exist, etc.
 
 * Documentation
 
-Each package should provide documentation in Texinfo format.  For the
-documentation to be included in the sbcl manual, the following must
-hold:
+Each package should provide documentation in SB-MANUAL format.  For the
+documentation to be included in the SBCL manual, you must
 
-- Each Texinfo file must have the extension `.texinfo' so the
-  automatic manual builder will find it.
+- add the relevant file to contrib/sb-manual/sb-manual.asd,
 
-- It must contain one @node - @section pair at the top and only
-  @subsection (or lower) sectioning commands within, e.g.
+- modify SB-MANUAL::*PAGES*.
 
-@node Sample Contrib
-@section Sample Contrib
-...
-
-  so that the contrib menu can be created automatically.
-
-Take care to choose unique node names.
+Take care to choose globally unique and meaningful section names, as
+the names are exported from SB-MANUAL and also visible to the user as
+HTML anchors.
 
 [ make install should copy the documentation somewhere that the user
 can find it ]

contrib/asdf-module.mk

@@ -12,7 +12,7 @@ FASL=$(DEST)/$(SYSTEM).fasl
 ASD=$(DEST)/$(SYSTEM).asd
 
 ifeq (SunOS,$(UNAME))
-  EXTRA_CFLAGS+=-D_XOPEN_SOURCE=500 -D__EXTENSIONS__
+  CFLAGS+=-D_XOPEN_SOURCE=500 -D__EXTENSIONS__
   PATH:=/usr/xpg4/bin:${PATH}
 endif
 ifeq (CYGWIN,$(findstring CYGWIN,$(UNAME)))
@@ -22,10 +22,10 @@ ifeq (CYGWIN,$(findstring CYGWIN,$(UNAME)))
   CC:=$(shell cygpath -m $(shell readlink -fn $(shell which $(CC))))
 endif
 ifeq (Linux,$(UNAME))
-  EXTRA_CFLAGS+=-D_GNU_SOURCE
+  CFLAGS+=-D_GNU_SOURCE
 endif
 
-export CC SBCL EXTRA_CFLAGS
+export CC SBCL CFLAGS
 
 all: $(FASL)
 

contrib/make-contrib.lisp

@@ -46,6 +46,15 @@
           (unless (= result 0) (error "C execution failed")))))))
 
 (defparameter +genfile+ "generated-constants")
+
+(defun resolve-up-directory-components (pathname-directory)
+  (let ((result ()))
+    (dolist (c pathname-directory)
+      (if (eq c :up)
+          (pop result)
+          (push c result)))
+    (nreverse result)))
+
 (defun logicalize (path generated)
   (make-pathname :host "SYS"
                  :directory
@@ -53,8 +62,9 @@
                          (if generated
                              (list "OBJ" "FROM-SELF" "CONTRIB" *system*)
                              (list* "CONTRIB"
-                                    (append (last (pathname-directory *default-pathname-defaults*))
-                                            (cdr (pathname-directory path))))))
+                                    (resolve-up-directory-components
+                                     (append (last (pathname-directory *default-pathname-defaults*))
+                                             (cdr (pathname-directory path)))))))
                  :name (pathname-name path)
                  :type (pathname-type path)))
 
@@ -134,6 +144,8 @@
         ;; foreign-glue contains macros needed to compile the generated file
         (let ((*evaluator-mode* :compile)) (load "../sb-grovel/foreign-glue")))
       (let (wcu-warnings)
+        ;; SETQ is fine, we're going to exit this image soon enough
+        (setq sb-ext:*derive-function-types* t)
         (handler-bind (((and warning (not style-warning))
                         (lambda (c)
                           (unless (ignorable-warning-p c)
@@ -141,13 +153,15 @@
           (with-compilation-unit ()
             (loop for (generated-p stem) in (flattened-sources)
              do (let ((fasl
-                       (if (string= (pathname-type stem) "fasl")
-                           stem
-                           (multiple-value-bind (output warnings errors)
-                               (compile-file (logicalize stem generated-p)
-                                             :output-file (format nil "~A~A.fasl" objdir stem))
-                             (when (or warnings errors) (sb-sys:os-exit 1))
-                             output))))
+                        (if (string= (pathname-type stem) "fasl")
+                            stem
+                            (multiple-value-bind (output warnings errors)
+                                (compile-file
+                                 (logicalize stem generated-p)
+                                 :output-file (ensure-directories-exist
+                                               (merge-pathnames stem objdir)))
+                              (when (or warnings errors) (sb-sys:os-exit 1))
+                              output))))
                   (fasls fasl)
                   (load fasl)))))
         ;; Deferred warnings occur *after* exiting the W-C-U body.

contrib/sb-aclrepl/manual.lisp

@@ -0,0 +1,51 @@
+(in-package :sb-manual)
+
+(defsection @sb-aclrepl (:title "sb-aclrepl" :concepts (~repl))
+  "The `SB-ACLREPL` module offers an Allegro CL-style
+  Read-Eval-Print Loop for SBCL, with integrated inspector. Adding a
+  debugger interface is planned.
+
+  Allegro CL is a registered trademark of Franz Inc."
+  (@sb-aclrepl-usage section)
+  (@sb-aclrepl-customization section)
+  (@sb-aclrepl-example-initialization section))
+
+(defsection @sb-aclrepl-usage (:title "Usage")
+  "To start `SB-ACLREPL` as your read-eval-print loop, put the form
+
+      (require 'sb-aclrepl)
+
+  in your `~/.sbclrc`, one of your @INITIALIZATION-FILES.")
+
+(defsection @sb-aclrepl-customization (:title "Customization")
+  "The following customization variables are available:"
+  (sb-aclrepl:*command-char* variable)
+  (sb-aclrepl:*prompt* variable)
+  (sb-aclrepl:*exit-on-eof* variable)
+  (sb-aclrepl:*use-short-package-name* variable)
+  (sb-aclrepl:*max-history* variable))
+
+(defsection @sb-aclrepl-example-initialization (:title "Example Initialization")
+  "Here's a longer example of a `~/.sbclrc` file that shows off
+  some of the features of sb-aclrepl:
+
+      (ignore-errors (require 'sb-aclrepl))
+
+      (when (find-package 'sb-aclrepl)
+        (push :aclrepl cl:*features*))
+      #+aclrepl
+      (progn
+        (setq sb-aclrepl:*max-history* 100)
+        (setf (sb-aclrepl:alias \"asdc\")
+             #'(lambda (sys) (asdf:operate 'asdf:compile-op sys)))
+        (sb-aclrepl:alias \"l\" (sys) (asdf:operate 'asdf:load-op sys))
+        (sb-aclrepl:alias \"t\" (sys) (asdf:operate 'asdf:test-op sys))
+        ;; The 1 below means that two characaters (\"up\") are required
+        (sb-aclrepl:alias (\"up\" 1 \"Use package\") (package) (use-package package))
+        ;; The 0 below means only the first letter (\"r\") is required,
+        ;; such as \":r base64\"
+        (sb-aclrepl:alias (\"require\" 0 \"Require module\") (sys) (require sys))
+        (setq cl:*features* (delete :aclrepl cl:*features*)))
+
+  Questions, comments, or bug reports should be sent to Kevin Rosenberg
+  (kevin@rosenberg.net).")

contrib/sb-aclrepl/repl.lisp

@@ -33,7 +33,7 @@
 (defparameter *prompt* #.*default-prompt*
   "The current prompt string or formatter function.")
 (defparameter *use-short-package-name* t
-  "when T, use the shortnest package nickname in a prompt")
+  "When T, use the shortest package nickname in a prompt.")
 (defparameter *dir-stack* nil
   "The top-level directory stack")
 (defparameter *command-char* #\:

contrib/sb-aclrepl/sb-aclrepl.texinfo

@@ -1,39 +1,70 @@
-@node sb-aclrepl
-@section sb-aclrepl
+@c Generated by the sb-manual contrib. Do not edit.
+
+@node sb aclrepl
 @cindex Read-Eval-Print Loop
 @cindex REPL
+@section sb-aclrepl
+
+@menu
+* Usage: sb aclrepl usage.
+* Customization: sb aclrepl customization.
+* Example Initialization: sb aclrepl example initialization.
+@end menu
 
-@c FIXME: I wanted to use @registeredsymbol{}, but that's
-@c only available in Texinfo 4.7.  sigh.
 The @code{sb-aclrepl} module offers an Allegro CL-style
-Read-Eval-Print Loop for SBCL, with integrated inspector.  Adding a
+Read-Eval-Print Loop for SBCL, with integrated inspector. Adding a
 debugger interface is planned.
 
+Allegro CL is a registered trademark of Franz Inc.
+
+@node sb aclrepl usage
 @subsection Usage
 
 To start @code{sb-aclrepl} as your read-eval-print loop, put the form
-@lisp
+
+@example
 (require 'sb-aclrepl)
-@end lisp
+@end example
 
-in your @file{~/.sbclrc} initialization file.
+in your @code{~/.sbclrc}, one of your @ref{initialization files}.
 
+@node sb aclrepl customization
 @subsection Customization
 
 The following customization variables are available:
 
-@include var-sb-aclrepl-star-command-char-star.texinfo
-@include var-sb-aclrepl-star-prompt-star.texinfo
-@include var-sb-aclrepl-star-exit-on-eof-star.texinfo
-@include var-sb-aclrepl-star-use-short-package-name-star.texinfo
-@include var-sb-aclrepl-star-max-history-star.texinfo
-
+@anchor{Variable sb-aclrepl *command-char*}
+@vvindex @sortas{command-char* sb-aclrepl} *command-char* [sb-aclrepl]
+@deffn{Variable} sb-aclrepl:*command-char*
+Prefix character for a top-level command
+@end deffn
+@anchor{Variable sb-aclrepl *prompt*}
+@vvindex @sortas{prompt* sb-aclrepl} *prompt* [sb-aclrepl]
+@deffn{Variable} sb-aclrepl:*prompt*
+The current prompt string or formatter function.
+@end deffn
+@anchor{Variable sb-aclrepl *exit-on-eof*}
+@vvindex @sortas{exit-on-eof* sb-aclrepl} *exit-on-eof* [sb-aclrepl]
+@deffn{Variable} sb-aclrepl:*exit-on-eof*
+If @code{t}, then exit when the EOF character is entered.
+@end deffn
+@anchor{Variable sb-aclrepl *use-short-package-name*}
+@vvindex @sortas{use-short-package-name* sb-aclrepl} *use-short-package-name* [sb-aclrepl]
+@deffn{Variable} sb-aclrepl:*use-short-package-name*
+When @code{t}, use the shortest package nickname in a prompt.
+@end deffn
+@anchor{Variable sb-aclrepl *max-history*}
+@vvindex @sortas{max-history* sb-aclrepl} *max-history* [sb-aclrepl]
+@deffn{Variable} sb-aclrepl:*max-history*
+Maximum number of history commands to remember
+@end deffn
+@node sb aclrepl example initialization
 @subsection Example Initialization
 
-Here's a longer example of a @file{~/.sbclrc} file that shows off
-some of the features of @code{sb-aclrepl}:
+Here's a longer example of a @code{~/.sbclrc} file that shows off
+some of the features of sb-aclrepl:
 
-@lisp
+@example
 (ignore-errors (require 'sb-aclrepl))
 
 (when (find-package 'sb-aclrepl)
@@ -51,11 +82,8 @@ some of the features of @code{sb-aclrepl}:
   ;; such as ":r base64"
   (sb-aclrepl:alias ("require" 0 "Require module") (sys) (require sys))
   (setq cl:*features* (delete :aclrepl cl:*features*)))
-@end lisp
+@end example
 
 Questions, comments, or bug reports should be sent to Kevin Rosenberg
-(@email{kevin@@rosenberg.net}).
+(kevin@@rosenberg.net).
 
-@subsection Credits
-
-Allegro CL is a registered trademark of Franz Inc.

contrib/sb-bsd-sockets/inet.lisp

@@ -22,7 +22,7 @@
 ;;; getprotobyname if it's available.
 (defun get-protocol-by-name (name)
   "Given a protocol name, return the protocol number, the protocol name, and
-a list of protocol aliases"
+a list of protocol aliases."
   (let ((result (cdr (if (keywordp name)
                          (assoc name *protocols*)
                          (assoc name *protocols* :test #'string-equal)))))

contrib/sb-bsd-sockets/inet4.lisp

@@ -9,9 +9,9 @@
 
 Examples:
 
- (make-instance 'sb-bsd-sockets:inet-socket :type :stream :protocol :tcp)
+    (make-instance 'sb-bsd-sockets:inet-socket :type :stream :protocol :tcp)
 
- (make-instance 'sb-bsd-sockets:inet-socket :type :datagram :protocol :udp)
+    (make-instance 'sb-bsd-sockets:inet-socket :type :datagram :protocol :udp)
 ")))
 
 (defun address-numbers/v4 (address)

contrib/sb-bsd-sockets/inet6.lisp

@@ -9,9 +9,9 @@
 
 Examples:
 
- (make-instance 'sb-bsd-sockets:inet6-socket :type :stream :protocol :tcp)
+    (make-instance 'sb-bsd-sockets:inet6-socket :type :stream :protocol :tcp)
 
- (make-instance 'sb-bsd-sockets:inet6-socket :type :datagram :protocol :udp)
+    (make-instance 'sb-bsd-sockets:inet6-socket :type :datagram :protocol :udp)
 ")))
 
 (defun address-numbers/v6 (address)

contrib/sb-bsd-sockets/local.lisp

@@ -6,8 +6,8 @@
 (defclass local-socket (socket)
   ((family :initform sockint::af-local))
   (:documentation
-   "Class representing local domain (AF_LOCAL) sockets,
-also known as unix-domain sockets."))
+   "Class representing local domain (`\\\\AF_LOCAL`) sockets,
+also known as Unix-domain sockets."))
 
 (defmethod socket-namestring ((socket local-socket))
   (ignore-errors (socket-name socket)))
@@ -41,8 +41,8 @@ also known as unix-domain sockets."))
 
 (defclass local-abstract-socket (local-socket) ()
   (:documentation
-   "Class representing local domain (AF_LOCAL) sockets with addresses
-in the abstract namespace."))
+   "Class representing local domain (`\\\\AF_LOCAL`) sockets with
+addresses in the abstract namespace."))
 
 (defmethod make-sockaddr-for ((socket local-abstract-socket)
                               &optional sockaddr &rest address)

contrib/sb-bsd-sockets/manual.lisp

@@ -0,0 +1,127 @@
+(in-package :sb-manual)
+
+(defsection @networking (:title "Networking")
+  "The `SB-BSD-SOCKETS` module provides a thinly disguised BSD
+  socket API for SBCL. Ideas have been stolen from the BSD socket API
+  for C and Graham Barr's `IO::Socket` classes for Perl.
+
+  Sockets are represented as CLOS objects, and the API naming
+  conventions attempt to balance between the BSD names and good lisp
+  style."
+  (@sockets-overview section)
+  (@general-sockets section)
+  (@socket-options section)
+  (@inet-domain-sockets section)
+  #-win32
+  (@local-domain-sockets section)
+  (@name-service section))
+
+(defsection @sockets-overview (:title "Sockets Overview")
+  "Most of the functions are modelled on the BSD socket API. BSD sockets
+  are widely supported, portably (by Unix standards, at least)
+  available on a variety of systems, and documented. There are some
+  differences in approach where we have taken advantage of some of the
+  more useful features of Common Lisp -- briefly:
+
+  - Where the C API would typically return -1 and set `errno`,
+    `SB-BSD-SOCKETS` signals an error. All the errors are subclasses
+    of SB-BSD-SOCKETS:SOCKET-ERROR and generally correspond one for
+    one with possible `errno` values.
+
+  - We use multiple return values in many places where the C API would
+    use pass-by-reference values.
+
+  - We can often avoid supplying an explicit length argument to
+    functions because we already know how long the argument is.
+
+  - IP addresses and ports are represented in slightly friendlier
+    fashion than \"network-endian integers\".")
+
+(defsection @general-sockets (:title "General Sockets")
+  (sb-bsd-sockets:socket class)
+  (sb-bsd-sockets:socket-bind function)
+  (sb-bsd-sockets:socket-accept function)
+  (sb-bsd-sockets:socket-connect function)
+  (sb-bsd-sockets:socket-peername function)
+  (sb-bsd-sockets:socket-name function)
+  (sb-bsd-sockets:socket-receive function)
+  (sb-bsd-sockets:socket-send function)
+  (sb-bsd-sockets:socket-listen function)
+  (sb-bsd-sockets:socket-open-p function)
+  (sb-bsd-sockets:socket-close function)
+  (sb-bsd-sockets:socket-shutdown function)
+  (sb-bsd-sockets:socket-make-stream function)
+  (sb-bsd-sockets:socket-error function)
+  (sb-bsd-sockets:non-blocking-mode function))
+
+(defsection @socket-options (:title "Socket Options")
+  "A subset of socket options are supported, using a fairly general
+  framework which should make it simple to add more as required -- see
+  `\\\\SYS:CONTRIB;SB-BSD-SOCKETS:SOCKOPT.LISP` for details. The name
+  mapping from C is fairly straightforward: `\\\\SO_RCVLOWAT` becomes
+  SB-BSD-SOCKETS:SOCKOPT-RECEIVE-LOW-WATER and `(SETF
+  SB-BSD-SOCKETS:SOCKOPT-RECEIVE-LOW-WATER)`."
+  (sb-bsd-sockets:sockopt-reuse-address function)
+  (sb-bsd-sockets:sockopt-keep-alive function)
+  (sb-bsd-sockets:sockopt-oob-inline function)
+  (sb-bsd-sockets:sockopt-bsd-compatible function)
+  (sb-bsd-sockets:sockopt-pass-credentials function)
+  (sb-bsd-sockets:sockopt-debug function)
+  (sb-bsd-sockets:sockopt-dont-route function)
+  (sb-bsd-sockets:sockopt-broadcast function)
+  (sb-bsd-sockets:sockopt-tcp-nodelay function))
+
+(defsection @inet-domain-sockets (:title "INET Domain Sockets")
+  "The TCP and UDP sockets that you know and love. Some representation
+  issues:
+
+  - IPv4 Internet addresses are represented by vectors of
+    `(UNSIGNED-BYTE 8)` (e.g. `#(127 0 0 1)`). Ports are just
+    integers. No conversion between network- and host-order data is
+    needed from the user of this package.
+
+  - IPv6 Internet addresses are represented by length 16 vectors of
+    `(UNSIGNED-BYTE 8)` (e.g. `#(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1)`.
+    Ports are just integers. As for IPv4 addresses, no conversion
+    between network- and host-order data is needed from the user of
+    this package.
+
+  - Socket addresses are represented by the two values for address and
+    port, so for example, `(SB-BSD-SOCKETS:SOCKET-CONNECT SOCKET #(192
+    168 1 1) 80)` for IPv4 and `(SB-BSD-SOCKETS:SOCKET-CONNECT SOCKET
+    #(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1) 80)` for IPv6."
+  (sb-bsd-sockets:inet-socket class)
+  (sb-bsd-sockets:inet6-socket class)
+  (sb-bsd-sockets:make-inet-address function)
+  (sb-bsd-sockets:make-inet6-address function)
+  (sb-bsd-sockets:get-protocol-by-name function))
+
+#-win32
+(defsection @local-domain-sockets (:title "Local Domain Sockets")
+  "Local domain (`\\\\AF_LOCAL`) sockets are also known as Unix-domain
+  sockets but were renamed by POSIX presumably on the basis that they
+  may be available on other systems too.
+
+  A local socket address is a string, which is used to create a node
+  in the local filesystem. This means of course that they cannot be
+  used across a network."
+  (sb-bsd-sockets:local-socket class)
+  "A local abstract socket address is also a string the scope of which is
+  the local machine. However, in contrast to a local socket address, there
+  is no corresponding filesystem node."
+  (sb-bsd-sockets:local-abstract-socket class))
+
+(defsection @name-service (:title "Name Service")
+  "Presently name service is implemented by calling out to the
+  `getaddrinfo(3)` and `gethostinfo(3)`, or to `gethostbyname(3)` and
+  `gethostbyaddr(3)` on platforms where the preferred functions are
+  not available. The exact details of the name resolving process (for
+  example the choice of whether DNS or a hosts file is used for
+  lookup) are platform dependent."
+  ;; Direct links to the asynchronous `resolver(3)` routines would be
+  ;; nice to have eventually, so that we can do DNS lookups in
+  ;; parallel with other things.
+  (sb-bsd-sockets:host-ent class)
+  (sb-bsd-sockets:get-host-by-name function)
+  (sb-bsd-sockets:get-host-by-address function)
+  (sb-bsd-sockets:host-ent-address function))

contrib/sb-bsd-sockets/name-service.lisp

@@ -67,14 +67,14 @@
   (defun get-host-by-name (host-name)
     "Returns a HOST-ENT instance for HOST-NAME or signals a NAME-SERVICE-ERROR.
 HOST-NAME may also be an IP address in dotted quad notation or some other
-weird stuff - see gethostbyname(3) for the details."
+weird stuff -- see `gethostbyname(3)` for the details."
     (sb-int:with-system-mutex (**gethostby-lock** :allow-with-interrupts t)
       (make-host-ent (sockint::gethostbyname host-name))))
 
   (defun get-host-by-address (address)
     "Returns a HOST-ENT instance for ADDRESS, which should be a vector of
- (integer 0 255), or signals a NAME-SERVICE-ERROR. See gethostbyaddr(3)
- for details."
+ `(INTEGER 0 255)`, or signals a NAME-SERVICE-ERROR. See
+ `gethostbyaddr(3)` for details."
     (sb-int:with-system-mutex (**gethostby-lock** :allow-with-interrupts t)
       (sockint::with-in-addr packed-addr ()
         (let ((addr-vector (coerce address 'vector)))

contrib/sb-bsd-sockets/protocol.lisp

@@ -58,23 +58,22 @@ peer."))
    "Close SOCKET, unless it was already closed.
 
 If SOCKET-MAKE-STREAM has been called, calls CLOSE using ABORT on that
-stream.  Otherwise closes the socket file descriptor using
-close(2)."))
+stream. Otherwise closes the socket file descriptor using `close(2)`."))
 
 (defgeneric socket-bind (socket &rest address)
   (:documentation
    "Bind SOCKET to ADDRESS, which may vary according to socket family.
-For the INET family, pass ADDRESS and PORT as two arguments; for FILE
-address family sockets, pass the filename string.  See also bind(2)"))
+For the INET family, pass ADDRESS and PORT as two arguments; for local
+address family sockets, pass the filename string. See also `bind(2)`."))
 
 (defgeneric socket-accept (socket)
   (:documentation
-   "Perform the accept(2) call, returning a newly-created connected
+   "Perform the `accept(2)` call, returning a newly-created connected
 socket and the peer address as multiple values"))
 
 (defgeneric socket-connect (socket &rest address)
   (:documentation
-   "Perform the connect(2) call to connect SOCKET to a remote PEER.
+   "Perform the `connect(2)` call to connect SOCKET to a remote PEER.
 No useful return value."))
 
 (defgeneric socket-receive (socket buffer length
@@ -82,13 +81,13 @@ No useful return value."))
                             oob peek waitall dontwait element-type)
   (:documentation
    "Read LENGTH octets from SOCKET into BUFFER (or a freshly-consed
-buffer if NIL), using recvfrom(2). If LENGTH is NIL, the length of
+buffer if NIL), using `recvfrom(2)`. If LENGTH is NIL, the length of
 BUFFER is used, so at least one of these two arguments must be
 non-NIL. If BUFFER is supplied, it had better be of an element type
 one octet wide. Returns the buffer, its length, and the address of the
 peer that sent it, as multiple values. On datagram sockets, sets
-MSG_TRUNC so that the actual packet length is returned even if the
-buffer was too small."))
+`\\\\MSG_TRUNC` so that the actual packet length is returned even if
+the buffer was too small."))
 
 (defgeneric socket-send (socket buffer length
                          &key
@@ -97,12 +96,12 @@ buffer was too small."))
                          oob eor dontroute dontwait nosignal
                          #+linux confirm #+linux more)
   (:documentation
-   "Send LENGTH octets from BUFFER into SOCKET, using sendto(2). If
+   "Send LENGTH octets from BUFFER into SOCKET, using `sendto(2)`. If
 BUFFER is a string, it will converted to octets according to
 EXTERNAL-FORMAT. If LENGTH is NIL, the length of the octet buffer is
 used. The format of ADDRESS depends on the socket type (for example
 for INET domain sockets it would be a list of an IP address and a
-port). If no socket address is provided, send(2) will be called
+port). If no socket address is provided, `send(2)` will be called
 instead. Returns the number of octets written."))
 
 (defgeneric socket-listen (socket backlog)
@@ -110,7 +109,7 @@ instead. Returns the number of octets written."))
    "Mark SOCKET as willing to accept incoming connections.  The
 integer BACKLOG defines the maximum length that the queue of pending
 connections may grow to before new connection attempts are refused.
-See also listen(2)"))
+See also `listen(2)`."))
 
 (defgeneric socket-shutdown (socket &key direction)
   (:documentation

contrib/sb-bsd-sockets/sb-bsd-sockets.texinfo

@@ -1,192 +1,368 @@
-@node Networking
-@comment  node-name,  next,  previous,  up
+@c Generated by the sb-manual contrib. Do not edit.
+
+@node networking
 @chapter Networking
-@cindex Sockets, Networking
+
+@menu
+* Sockets Overview: sockets overview.
+* General Sockets: general sockets.
+* Socket Options: socket options.
+* INET Domain Sockets: inet domain sockets.
+* Local Domain Sockets: local domain sockets.
+* Name Service: name service.
+@end menu
 
 The @code{sb-bsd-sockets} module provides a thinly disguised BSD
 socket API for SBCL. Ideas have been stolen from the BSD socket API
-for C and Graham Barr's IO::Socket classes for Perl.
+for C and Graham Barr's @code{IO::Socket} classes for Perl.
 
 Sockets are represented as CLOS objects, and the API naming
-conventions attempt to balance between the BSD names and good lisp style.
+conventions attempt to balance between the BSD names and good lisp
+style.
 
-@menu
-* Sockets Overview::
-* General Sockets::      Methods applicable to all sockets
-* Socket Options::
-* INET Domain Sockets::
-* Local (Unix) Domain Sockets::
-* Name Service::
-@end menu
-
-@node Sockets Overview
+@node sockets overview
 @section Sockets Overview
 
-Most of the functions are modelled on the BSD socket API.  BSD sockets
-are widely supported, portably @emph{(``portable'' by Unix standards, at least)}
-available on a variety of systems, and documented.  There are some
+Most of the functions are modelled on the BSD socket API. BSD sockets
+are widely supported, portably (by Unix standards, at least)
+available on a variety of systems, and documented. There are some
 differences in approach where we have taken advantage of some of the
-more useful features of Common Lisp - briefly:
+more useful features of Common Lisp -- briefly:
 
 @itemize
+@item Where the C API would typically return -1 and set @code{errno},
+  @code{sb-bsd-sockets} signals an error. All the errors are subclasses
+  of @code{sb-bsd-sockets:socket-error} and generally correspond one for
+  one with possible @code{errno} values.
 
-@item
-Where the C API would typically return -1 and set @code{errno},
-@code{sb-bsd-sockets} signals an error. All the errors are subclasses
-of @code{sb-bsd-sockets:socket-condition} and generally correspond one
-for one with possible @code{errno} values.
+@item We use multiple return values in many places where the C API would
+  use pass-by-reference values.
 
-@item
-We use multiple return values in many places where the C API would use
-pass-by-reference values.
-
-@item
-We can often avoid supplying an explicit @emph{length} argument to
-functions because we already know how long the argument is.
-
-@item
-IP addresses and ports are represented in slightly friendlier fashion
-than "network-endian integers".
+@item We can often avoid supplying an explicit length argument to
+  functions because we already know how long the argument is.
 
+@item IP addresses and ports are represented in slightly friendlier
+  fashion than "network-endian integers".
 @end itemize
 
-@node General Sockets
+@node general sockets
 @section General Sockets
 
-@include class-sb-bsd-sockets-socket.texinfo
+@anchor{Class sb-bsd-sockets socket}
+@ttindex @sortas{socket sb-bsd-sockets} socket [sb-bsd-sockets]
+@deffn{Class} sb-bsd-sockets:socket
+Common superclass of all sockets, not meant to be
+directly instantiated.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-bind}
+@ffindex @sortas{socket-bind sb-bsd-sockets} socket-bind [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-bind socket &rest address
+Bind @code{socket} to @code{address}, which may vary according to socket family.
+For the INET family, pass @code{address} and @code{port} as two arguments; for local
+address family sockets, pass the filename string. See also @code{bind(2)}.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-accept}
+@ffindex @sortas{socket-accept sb-bsd-sockets} socket-accept [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-accept socket
+Perform the @code{accept(2)} call, returning a newly-created connected
+socket and the peer address as multiple values
+@end deffn
+@anchor{Function sb-bsd-sockets socket-connect}
+@ffindex @sortas{socket-connect sb-bsd-sockets} socket-connect [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-connect socket &rest address
+Perform the @code{connect(2)} call to connect @code{socket} to a remote @code{peer}.
+No useful return value.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-peername}
+@ffindex @sortas{socket-peername sb-bsd-sockets} socket-peername [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-peername socket
+Return @code{socket}'s peer; depending on the address family this may
+return multiple values
+@end deffn
+@anchor{Function sb-bsd-sockets socket-name}
+@ffindex @sortas{socket-name sb-bsd-sockets} socket-name [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-name socket
+Return the address (as vector of bytes) and port that @code{socket} is
+bound to, as multiple values.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-receive}
+@ffindex @sortas{socket-receive sb-bsd-sockets} socket-receive [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-receive socket buffer length &key oob peek waitall dontwait element-type
+Read @code{length} octets from @code{socket} into @code{buffer} (or a freshly-consed
+buffer if @code{nil}), using @code{recvfrom(2)}. If @code{length} is @code{nil}, the length of
+@code{buffer} is used, so at least one of these two arguments must be
+non-@code{nil}. If @code{buffer} is supplied, it had better be of an element type
+one octet wide. Returns the buffer, its length, and the address of the
+peer that sent it, as multiple values. On datagram sockets, sets
+@code{MSG_TRUNC} so that the actual packet length is returned even if
+the buffer was too small.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-send}
+@ffindex @sortas{socket-send sb-bsd-sockets} socket-send [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-send socket buffer length &key address external-format oob eor dontroute dontwait nosignal confirm more
+Send @code{length} octets from @code{buffer} into @code{socket}, using @code{sendto(2)}. If
+@code{buffer} is a string, it will converted to octets according to
+@code{external-format}. If @code{length} is @code{nil}, the length of the octet buffer is
+used. The format of @code{address} depends on the socket type (for example
+for INET domain sockets it would be a list of an IP address and a
+port). If no socket address is provided, @code{send(2)} will be called
+instead. Returns the number of octets written.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-listen}
+@ffindex @sortas{socket-listen sb-bsd-sockets} socket-listen [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-listen socket backlog
+Mark @code{socket} as willing to accept incoming connections.  The
+integer @code{backlog} defines the maximum length that the queue of pending
+connections may grow to before new connection attempts are refused.
+See also @code{listen(2)}.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-open-p}
+@ffindex @sortas{socket-open-p sb-bsd-sockets} socket-open-p [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-open-p socket
+Return true if @code{socket} is open; otherwise, return false.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-close}
+@ffindex @sortas{socket-close sb-bsd-sockets} socket-close [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-close socket &key abort
+Close @code{socket}, unless it was already closed.
 
-@include fun-sb-bsd-sockets-socket-bind.texinfo
+If @code{socket-make-stream} has been called, calls @code{close} using @code{abort} on that
+stream. Otherwise closes the socket file descriptor using @code{close(2)}.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-shutdown}
+@ffindex @sortas{socket-shutdown sb-bsd-sockets} socket-shutdown [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-shutdown socket &key direction
+Indicate that no communication in @code{direction} will be performed on
+@code{socket}.
 
-@include fun-sb-bsd-sockets-socket-accept.texinfo
+@code{direction} has to be one of @code{:input}, @code{:output} or @code{:io}.
 
-@include fun-sb-bsd-sockets-socket-connect.texinfo
+After a shutdown, no input and/or output of the indicated @code{direction}
+can be performed on @code{socket}.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-make-stream}
+@ffindex @sortas{socket-make-stream sb-bsd-sockets} socket-make-stream [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-make-stream socket &key input output element-type external-format buffering timeout auto-close serve-events
+Find or create a @code{stream} that can be used for IO on @code{socket} (which
+must be connected).  Specify whether the stream is for @code{input}, @code{output},
+or both (it is an error to specify neither).
 
-@include fun-sb-bsd-sockets-socket-peername.texinfo
+@code{element-type} and @code{external-format} are as per @code{open}.
 
-@include fun-sb-bsd-sockets-socket-name.texinfo
+@code{timeout} specifies a read timeout for the stream.
+@end deffn
+@anchor{Function sb-bsd-sockets socket-error}
+@ffindex @sortas{socket-error sb-bsd-sockets} socket-error [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:socket-error where &optional errno
+Signal an appropriate error for syscall @code{where} and @code{errno}.
 
-@include fun-sb-bsd-sockets-socket-receive.texinfo
+@code{where} should be a string naming the failed function.
 
-@include fun-sb-bsd-sockets-socket-send.texinfo
-
-@include fun-sb-bsd-sockets-socket-listen.texinfo
-
-@include fun-sb-bsd-sockets-socket-open-p.texinfo
-
-@include fun-sb-bsd-sockets-socket-close.texinfo
-
-@include fun-sb-bsd-sockets-socket-shutdown.texinfo
-
-@include fun-sb-bsd-sockets-socket-make-stream.texinfo
-
-@include fun-sb-bsd-sockets-socket-error.texinfo
-
-@include fun-sb-bsd-sockets-non-blocking-mode.texinfo
-
-@node Socket Options
+When supplied, @code{errno} should be the UNIX error number associated to the
+failed call. The default behavior is to use the current value of the
+errno variable.
+@end deffn
+@anchor{Function sb-bsd-sockets non-blocking-mode}
+@ffindex @sortas{non-blocking-mode sb-bsd-sockets} non-blocking-mode [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:non-blocking-mode socket
+Is @code{socket} in non-blocking mode?
+@end deffn
+@node socket options
 @section Socket Options
 
 A subset of socket options are supported, using a fairly general
-framework which should make it simple to add more as required - see
-@file{SYS:CONTRIB;SB-BSD-SOCKETS:SOCKOPT.LISP} for details. The name
+framework which should make it simple to add more as required -- see
+@code{SYS:CONTRIB;SB-BSD-SOCKETS:SOCKOPT.LISP} for details. The name
 mapping from C is fairly straightforward: @code{SO_RCVLOWAT} becomes
-@code{sockopt-receive-low-water} and @code{(setf
-sockopt-receive-low-water)}.
+@code{sb-bsd-sockets:sockopt-receive-low-water} and @code{(setf
+sb-bsd-sockets:sockopt-receive-low-water)}.
 
-@include fun-sb-bsd-sockets-sockopt-reuse-address.texinfo
-
-@include fun-sb-bsd-sockets-sockopt-keep-alive.texinfo
-
-@include fun-sb-bsd-sockets-sockopt-oob-inline.texinfo
-
-@include fun-sb-bsd-sockets-sockopt-bsd-compatible.texinfo
-
-@include fun-sb-bsd-sockets-sockopt-pass-credentials.texinfo
-
-@include fun-sb-bsd-sockets-sockopt-debug.texinfo
-
-@include fun-sb-bsd-sockets-sockopt-dont-route.texinfo
-
-@include fun-sb-bsd-sockets-sockopt-broadcast.texinfo
-
-@include fun-sb-bsd-sockets-sockopt-tcp-nodelay.texinfo
-
-@node INET Domain Sockets
+@anchor{Function sb-bsd-sockets sockopt-reuse-address}
+@ffindex @sortas{sockopt-reuse-address sb-bsd-sockets} sockopt-reuse-address [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-reuse-address socket
+Return the value of the SO-REUSEADDR socket option for @code{socket}. This can also be
+updated with @code{setf}.
+@end deffn
+@anchor{Function sb-bsd-sockets sockopt-keep-alive}
+@ffindex @sortas{sockopt-keep-alive sb-bsd-sockets} sockopt-keep-alive [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-keep-alive socket
+Return the value of the SO-KEEPALIVE socket option for @code{socket}. This can also be
+updated with @code{setf}.
+@end deffn
+@anchor{Function sb-bsd-sockets sockopt-oob-inline}
+@ffindex @sortas{sockopt-oob-inline sb-bsd-sockets} sockopt-oob-inline [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-oob-inline socket
+Return the value of the SO-OOBINLINE socket option for @code{socket}. This can also be
+updated with @code{setf}.
+@end deffn
+@anchor{Function sb-bsd-sockets sockopt-bsd-compatible}
+@ffindex @sortas{sockopt-bsd-compatible sb-bsd-sockets} sockopt-bsd-compatible [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-bsd-compatible socket
+Return the value of the SO-BSDCOMPAT socket option for @code{socket}. This can also be
+updated with @code{setf}. Available only on Linux.
+@end deffn
+@anchor{Function sb-bsd-sockets sockopt-pass-credentials}
+@ffindex @sortas{sockopt-pass-credentials sb-bsd-sockets} sockopt-pass-credentials [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-pass-credentials socket
+Return the value of the SO-PASSCRED socket option for @code{socket}. This can also be
+updated with @code{setf}. Available only on Linux.
+@end deffn
+@anchor{Function sb-bsd-sockets sockopt-debug}
+@ffindex @sortas{sockopt-debug sb-bsd-sockets} sockopt-debug [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-debug socket
+Return the value of the SO-DEBUG socket option for @code{socket}. This can also be
+updated with @code{setf}.
+@end deffn
+@anchor{Function sb-bsd-sockets sockopt-dont-route}
+@ffindex @sortas{sockopt-dont-route sb-bsd-sockets} sockopt-dont-route [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-dont-route socket
+Return the value of the SO-DONTROUTE socket option for @code{socket}. This can also be
+updated with @code{setf}.
+@end deffn
+@anchor{Function sb-bsd-sockets sockopt-broadcast}
+@ffindex @sortas{sockopt-broadcast sb-bsd-sockets} sockopt-broadcast [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-broadcast socket
+Return the value of the SO-BROADCAST socket option for @code{socket}. This can also be
+updated with @code{setf}.
+@end deffn
+@anchor{Function sb-bsd-sockets sockopt-tcp-nodelay}
+@ffindex @sortas{sockopt-tcp-nodelay sb-bsd-sockets} sockopt-tcp-nodelay [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:sockopt-tcp-nodelay socket
+Return the value of the TCP-NODELAY socket option for @code{socket}. This can also be
+updated with @code{setf}.
+@end deffn
+@node inet domain sockets
 @section INET Domain Sockets
 
 The TCP and UDP sockets that you know and love. Some representation
 issues:
 
 @itemize
+@item IPv4 Internet addresses are represented by vectors of
+  @code{(unsigned-byte 8)} (e.g. @code{#(127 0 0 1)}). Ports are just
+  integers. No conversion between network- and host-order data is
+  needed from the user of this package.
 
-@item
-IPv4 Internet addresses are represented by vectors of
-@code{(unsigned-byte 8)} - viz. @code{#(127 0 0 1)}. Ports are just
-integers: 6010. No conversion between network- and host-order data is
-needed from the user of this package.
-
-@item
-IPv6 Internet addresses are represented by vectors of 16
-@code{(unsigned-byte 8)} - viz. @code{#(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
-1)}. Ports are just integers. As for IPv4 addresses, no conversion
-between network- and host-order data is needed from the user of this
-package.
-
-@item
-Socket addresses are represented by the two values for address and port,
-so for example, @code{(socket-connect socket #(192 168 1 1) 80)} for
-IPv4 and @code{(socket-connect socket #(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1)
-80)} for IPv6.
+@item IPv6 Internet addresses are represented by length 16 vectors of
+  @code{(unsigned-byte 8)} (e.g. @code{#(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1)}.
+  Ports are just integers. As for IPv4 addresses, no conversion
+  between network- and host-order data is needed from the user of
+  this package.
 
+@item Socket addresses are represented by the two values for address and
+  port, so for example, @code{(sb-bsd-sockets:socket-connect socket #(192
+  168 1 1) 80)} for IPv4 and @code{(sb-bsd-sockets:socket-connect socket
+  #(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1) 80)} for IPv6.
 @end itemize
 
-@include class-sb-bsd-sockets-inet-socket.texinfo
+@anchor{Class sb-bsd-sockets inet-socket}
+@ttindex @sortas{inet-socket sb-bsd-sockets} inet-socket [sb-bsd-sockets]
+@deffn{Class} sb-bsd-sockets:inet-socket
+Class representing TCP and UDP over IPv4 sockets.
 
-@include class-sb-bsd-sockets-inet6-socket.texinfo
+Examples:
 
-@include fun-sb-bsd-sockets-make-inet-address.texinfo
+@example
+(make-instance 'sb-bsd-sockets:inet-socket :type :stream :protocol :tcp)
 
-@include fun-sb-bsd-sockets-make-inet6-address.texinfo
+(make-instance 'sb-bsd-sockets:inet-socket :type :datagram :protocol :udp)
+@end example
+@end deffn
+@anchor{Class sb-bsd-sockets inet6-socket}
+@ttindex @sortas{inet6-socket sb-bsd-sockets} inet6-socket [sb-bsd-sockets]
+@deffn{Class} sb-bsd-sockets:inet6-socket
+Class representing TCP and UDP over IPv6 sockets.
 
-@include fun-sb-bsd-sockets-get-protocol-by-name.texinfo
+Examples:
 
-@node Local (Unix) Domain Sockets
-@section Local (Unix) Domain Sockets
+@example
+(make-instance 'sb-bsd-sockets:inet6-socket :type :stream :protocol :tcp)
+
+(make-instance 'sb-bsd-sockets:inet6-socket :type :datagram :protocol :udp)
+@end example
+@end deffn
+@anchor{Function sb-bsd-sockets make-inet-address}
+@ffindex @sortas{make-inet-address sb-bsd-sockets} make-inet-address [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:make-inet-address dotted-quads
+Return a vector of octets given a string @code{dotted-quads} in the format
+"127.0.0.1". Signals an error if the string is malformed.
+@end deffn
+@anchor{Function sb-bsd-sockets make-inet6-address}
+@ffindex @sortas{make-inet6-address sb-bsd-sockets} make-inet6-address [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:make-inet6-address colon-separated-integers
+Return a vector of octets given a string representation of an IPv6
+address @code{colon-separated-integers}. Signal an error if the string is
+malformed.
+@end deffn
+@anchor{Function sb-bsd-sockets get-protocol-by-name}
+@ffindex @sortas{get-protocol-by-name sb-bsd-sockets} get-protocol-by-name [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:get-protocol-by-name name
+Given a protocol name, return the protocol number, the protocol name, and
+a list of protocol aliases.
+@end deffn
+@node local domain sockets
+@section Local Domain Sockets
 
 Local domain (@code{AF_LOCAL}) sockets are also known as Unix-domain
-sockets, but were renamed by POSIX presumably on the basis that they
+sockets but were renamed by POSIX presumably on the basis that they
 may be available on other systems too.
 
-A local socket address is a string, which is used to create a node in
-the local filesystem. This means of course that they cannot be used
-across a network.
-
-@include class-sb-bsd-sockets-local-socket.texinfo
+A local socket address is a string, which is used to create a node
+in the local filesystem. This means of course that they cannot be
+used across a network.
 
+@anchor{Class sb-bsd-sockets local-socket}
+@ttindex @sortas{local-socket sb-bsd-sockets} local-socket [sb-bsd-sockets]
+@deffn{Class} sb-bsd-sockets:local-socket
+Class representing local domain (@code{AF_LOCAL}) sockets,
+also known as Unix-domain sockets.
+@end deffn
 A local abstract socket address is also a string the scope of which is
 the local machine. However, in contrast to a local socket address, there
 is no corresponding filesystem node.
 
-@include class-sb-bsd-sockets-local-abstract-socket.texinfo
-
-@node Name Service
+@anchor{Class sb-bsd-sockets local-abstract-socket}
+@ttindex @sortas{local-abstract-socket sb-bsd-sockets} local-abstract-socket [sb-bsd-sockets]
+@deffn{Class} sb-bsd-sockets:local-abstract-socket
+Class representing local domain (@code{AF_LOCAL}) sockets with
+addresses in the abstract namespace.
+@end deffn
+@node name service
 @section Name Service
 
 Presently name service is implemented by calling out to the
-@code{getaddrinfo(3)} and @code{gethostinfo(3)}, or to
-@code{gethostbyname(3)} @code{gethostbyaddr(3)} on platforms where
-the preferred functions are not available. The exact details of
-the name resolving process (for example the choice of whether
-DNS or a hosts file is used for lookup) are platform dependent.
+@code{getaddrinfo(3)} and @code{gethostinfo(3)}, or to @code{gethostbyname(3)} and
+@code{gethostbyaddr(3)} on platforms where the preferred functions are
+not available. The exact details of the name resolving process (for
+example the choice of whether DNS or a hosts file is used for
+lookup) are platform dependent.
 
-@c Direct links to the asynchronous @code{resolver(3)} routines would be
-@c nice to have eventually, so that we can do DNS lookups in parallel
-@c with other things.
+@anchor{Class sb-bsd-sockets host-ent}
+@ttindex @sortas{host-ent sb-bsd-sockets} host-ent [sb-bsd-sockets]
+@deffn{Class} sb-bsd-sockets:host-ent
+This class represents the results of an address lookup.
+@end deffn
+@anchor{Function sb-bsd-sockets get-host-by-name}
+@ffindex @sortas{get-host-by-name sb-bsd-sockets} get-host-by-name [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:get-host-by-name host-name
+Returns a @code{host-ent} instance for @code{host-name} or signals a @code{name-service-error}.
 
-@include class-sb-bsd-sockets-host-ent.texinfo
+Another @code{host-ent} instance containing zero, one or more IPv6 addresses
+may be returned as a second return value.
 
-@include fun-sb-bsd-sockets-get-host-by-name.texinfo
-
-@include fun-sb-bsd-sockets-get-host-by-address.texinfo
-
-@include fun-sb-bsd-sockets-host-ent-address.texinfo
+@code{host-name} may also be an IP address in dotted quad notation or some other
+weird stuff - see getaddrinfo(3) for the details.
+@end deffn
+@anchor{Function sb-bsd-sockets get-host-by-address}
+@ffindex @sortas{get-host-by-address sb-bsd-sockets} get-host-by-address [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:get-host-by-address address
+Returns a @code{host-ent} instance for @code{address}, which should be a vector of
+(integer 0 255) with 4 elements in case of an IPv4 address and 16
+elements in case of an IPv6 address, or signals a @code{name-service-error}.
+See gethostbyaddr(3) for details.
+@end deffn
+@anchor{Function sb-bsd-sockets host-ent-address}
+@ffindex @sortas{host-ent-address sb-bsd-sockets} host-ent-address [sb-bsd-sockets]
+@deffn{Function} sb-bsd-sockets:host-ent-address host-ent
+Return some valid address for @code{host-ent}.
+@end deffn

contrib/sb-bsd-sockets/sockopt.lisp

@@ -52,7 +52,10 @@ Code for options that not every system has should be conditionalised:
     `(progn
        (export ',lisp-name)
        (defun ,lisp-name (socket)
-         ,@(when documentation (list (concatenate 'string documentation " " info)))
+         ,@(when documentation
+             (list (if (plusp (length info))
+                       (format nil "~A ~A" documentation info)
+                       documentation)))
          ,@(if supportedp
                `((sb-alien:with-alien ((size sb-alien:int)
                                        (buffer ,buffer-type))
@@ -135,9 +138,9 @@ Code for options that not every system has should be conditionalised:
 
 (defmacro define-socket-option-bool (name level c-name &optional features (info ""))
   `(define-socket-option ,name
-    ,(format nil "~@<Return the value of the ~A socket option for SOCKET. ~
+       ,(format nil "~@<Return the value of the ~A socket option for SOCKET. ~
                  This can also be updated with SETF.~:@>"
-             (symbol-name c-name))
+                (symbol-name c-name))
     ,level ,c-name
     sb-alien:int bool-to-foreign-int foreign-int-to-bool sb-alien:addr
     ,features ,info))

contrib/sb-cltl2/env.lisp

@@ -180,18 +180,14 @@ Note that the global function binding may differ from the local one.
 This function returns three values. The first indicates the type of
 function definition or binding:
 
-  NIL
-    There is no apparent definition for NAME.
+- NIL: There is no apparent definition for NAME.
 
-  :FUNCTION
-    NAME refers to a function.
+- :FUNCTION: NAME refers to a function.
 
-  :MACRO
-    NAME refers to a macro.
+- :MACRO: NAME refers to a macro.
 
-  :SPECIAL-FORM
-    NAME refers to a special operator. If the name refers to both a
-    macro and a special operator, the macro takes precedence.
+- :SPECIAL-FORM: NAME refers to a special operator. If the name refers
+  to both a macro and a special operator, the macro takes precedence.
 
 The second value is true if NAME is bound locally.
 
@@ -199,26 +195,23 @@ The third value is an alist describing the declarations that apply to
 the function NAME. Standard declaration specifiers that may appear in
 CARS of the alist include:
 
-  DYNAMIC-EXTENT
-    If the CDR is T, NAME has been declared DYNAMIC-EXTENT. If the CDR
-    is NIL, the alist element may be omitted.
+- DYNAMIC-EXTENT: If the CDR is T, NAME has been declared
+  DYNAMIC-EXTENT. If the CDR is NIL, the alist element may be omitted.
 
-  INLINE
-    The CDR is one of the symbols INLINE, NOTINLINE, or NIL, to
-    indicate if the function has been declared INLINE or NOTINLINE. If
-    the CDR is NIL the alist element may be omitted.
+- INLINE: The CDR is one of the symbols INLINE, NOTINLINE, or NIL, to
+  indicate if the function has been declared INLINE or NOTINLINE. If
+  the CDR is NIL the alist element may be omitted.
 
-  FTYPE
-    The CDR is the type specifier associated with NAME, or the symbol
-    FUNCTION if there is functional type declaration or proclamation
-    associated with NAME. If the CDR is FUNCTION the alist element may
-    be omitted.
+- FTYPE: The CDR is the type specifier associated with NAME, or the
+  symbol FUNCTION if there is functional type declaration or
+  proclamation associated with NAME. If the CDR is FUNCTION the alist
+  element may be omitted.
 
-  SB-EXT:DEPRECATED
-    \(SBCL specific)
-    The CDR is a plist containing the following properties
+- SB-EXT:DEPRECATED: (SBCL specific) The CDR is a plist containing the
+  following properties:
+
+    - :STATE ( :EARLY | :LATE | :FINAL )
 
-      :STATE ( :EARLY | :LATE | :FINAL )
         Use of :EARLY deprecated functions signals a STYLE-WARNING at
         compile-time.
 
@@ -228,14 +221,16 @@ CARS of the alist include:
         Use of :FINAL deprecated functions signals a full WARNING at
         compile-time and an error at runtime.
 
-      :SINCE (SOFTWARE VERSION)
-        VERSION is a string designating the version since which the
-        function has been deprecated. SOFTWARE is NIL or the name of
-        the software to which VERSION refers, e.g. \"SBCL\" for
+    - :SINCE `(SOFTWARE VERSION)`
+
+        `VERSION` is a string designating the version since which the
+        function has been deprecated. `SOFTWARE` is NIL or the name of
+        the software to which `VERSION` refers, e.g. `\"SBCL\"` for
         deprecated functions in SBCL.
 
-      :REPLACEMENTS REPLACEMENTS
-        When this property is present, REPLACEMENTS is a list of
+    - :REPLACEMENTS `REPLACEMENTS`
+
+        When this property is present, `REPLACEMENTS` is a list of
         symbols naming functions that should be used instead of the
         deprecated function.
 
@@ -314,27 +309,20 @@ Note that the global binding may differ from the local one.
 This function returns three values. The first indicated the type of the variable
 binding:
 
-  NIL
-    There is no apparent binding for NAME.
+- NIL: There is no apparent binding for NAME.
 
-  :SPECIAL
-    NAME refers to a special variable.
+- :SPECIAL: NAME refers to a special variable.
 
-  :LEXICAL
-    NAME refers to a lexical variable.
+- :LEXICAL: NAME refers to a lexical variable.
 
-  :SYMBOL-MACRO
-    NAME refers to a symbol macro.
+- :SYMBOL-MACRO: NAME refers to a symbol macro.
 
-  :CONSTANT
-    NAME refers to a named constant defined using DEFCONSTANT, or NAME
-    is a keyword.
+- :CONSTANT: NAME refers to a named constant defined using
+  DEFCONSTANT, or NAME is a keyword.
 
-  :GLOBAL
-    NAME refers to a global variable. (SBCL specific extension.)
+- :GLOBAL: NAME refers to a global variable. (SBCL specific extension.)
 
-  :ALIEN
-    NAME refers to an alien variable. (SBCL specific extension.)
+- :ALIEN NAME refers to an alien variable. (SBCL specific extension.)
 
 The second value is true if NAME is bound locally. This is currently
 always NIL for special variables, although arguably it should be T
@@ -344,30 +332,26 @@ The third value is an alist describing the declarations that apply to
 the function NAME. Standard declaration specifiers that may appear in
 CARS of the alist include:
 
-  DYNAMIC-EXTENT
-    If the CDR is T, NAME has been declared DYNAMIC-EXTENT. If the CDR
-    is NIL, the alist element may be omitted.
+- DYNAMIC-EXTENT: If the CDR is T, NAME has been declared
+  DYNAMIC-EXTENT. If the CDR is NIL, the alist element may be omitted.
 
-  IGNORE
-    If the CDR is T, NAME has been declared IGNORE. If the CDR is NIL,
-    the alist element may be omitted.
+- IGNORE: If the CDR is T, NAME has been declared IGNORE. If the CDR
+  is NIL, the alist element may be omitted.
 
-  TYPE
-    The CDR is the type specifier associated with NAME, or the symbol
-    T if there is explicit type declaration or proclamation associated
-    with NAME. The type specifier may be equivalent to or a supertype
-    of the original declaration. If the CDR is T the alist element may
-    be omitted.
+- TYPE: The CDR is the type specifier associated with NAME, or the
+  symbol T if there is explicit type declaration or proclamation
+  associated with NAME. The type specifier may be equivalent to or a
+  supertype of the original declaration. If the CDR is T the alist
+  element may be omitted.
 
-  SB-EXT:ALWAYS-BOUND
-    \(SBCL specific)
-    If CDR is T, NAME has been declared as SB-EXT:ALWAYS-BOUND
+- SB-EXT:ALWAYS-BOUND: (SBCL specific) If CDR is T, NAME has been
+  declared as SB-EXT:ALWAYS-BOUND.
 
-  SB-EXT:DEPRECATED
-    \(SBCL specific)
-    The CDR is a plist containing the following properties
+- SB-EXT:DEPRECATED: (SBCL specific) The CDR is a plist containing the
+  following properties:
+
+    - :STATE ( :EARLY | :LATE | :FINAL )
 
-      :STATE ( :EARLY | :LATE | :FINAL )
         Use of :EARLY deprecated variables signals a STYLE-WARNING at
         compile-time.
 
@@ -377,14 +361,16 @@ CARS of the alist include:
         Use of :FINAL deprecated variables signals a full WARNING at
         compile-time and an error at runtime.
 
-      :SINCE (SOFTWARE VERSION)
-        VERSION is a string designating the version since which the
-        variable has been deprecated. SOFTWARE is NIL or the name of
-        the software to which VERSION refers, e.g. \"SBCL\" for
+    - :SINCE `(SOFTWARE VERSION)`
+
+        `VERSION` is a string designating the version since which the
+        variable has been deprecated. `SOFTWARE` is NIL or the name of
+        the software to which `VERSION` refers, e.g. `\"SBCL\"` for
         deprecated variables in SBCL.
 
-      :REPLACEMENTS REPLACEMENTS
-        When this property is present, REPLACEMENTS is a list of
+    - :REPLACEMENTS `REPLACEMENTS`
+
+        When this property is present, `REPLACEMENTS` is a list of
         symbols naming variables that should be used instead of the
         deprecated variable.
 

contrib/sb-concurrency/frlock.lisp

@@ -101,31 +101,30 @@ Readers gain entry to protected regions without waiting, but need to retry if
 a writer operated inside the region while they were reading. This makes frlocks
 very efficient when readers are much more common than writers.
 
-FRlocks are NOT suitable when it is not safe at all for readers and writers to
-operate on the same data in parallel: they provide consistency, not exclusion
-between readers and writers. Hence using an frlock to eg. protect an SBCL
-hash-table is unsafe. If multiple readers operating in parallel with a writer
-would be safe but inconsistent without a lock, frlocks are suitable.
+FRlocks are _not_ suitable when it is not safe at all for readers and writers
+to operate on the same data in parallel: they provide consistency, not
+exclusion between readers and writers. Hence using an frlock to e.g. protect
+an SBCL hash-table is unsafe. If multiple readers operating in parallel with
+a writer would be safe but inconsistent without a lock, frlocks are suitable.
 
 The recommended interface to use is FRLOCK-READ and FRLOCK-WRITE, but those
 needing it can also use a lower-level interface.
 
 Example:
 
-  ;; Values returned by FOO are always consistent so that
-  ;; the third value is the sum of the two first ones.
-  (let ((a 0)
-        (b 0)
-        (c 0)
-        (lk (make-frlock)))
-    (defun foo ()
-       (frlock-read (lk) a b c))
-    (defun bar (x y)
-       (frlock-write (lk)
-         (setf a x
-               b y
-               c (+ x y)))))
-"
+    ;; Values returned by FOO are always consistent so that
+    ;; the third value is the sum of the two first ones.
+    (let ((a 0)
+          (b 0)
+          (c 0)
+          (lk (make-frlock)))
+      (defun foo ()
+         (frlock-read (lk) a b c))
+      (defun bar (x y)
+         (frlock-write (lk)
+           (setf a x
+                 b y
+                 c (+ x y)))))"
   .
   #-new-frlock (
   (mutex (make-mutex :name "FRLock mutex") :type mutex :read-only t)
@@ -212,14 +211,13 @@ Using FRLOCK-READ instead is recommended.
 
 Example:
 
-  (multiple-value-bind (t0 e0) (frlock-read-begin *fr*)
-    (let ((a (get-a))
-          (b (get-b)))
-      (multiple-value-bind (t1 e1) (frlock-read-end *fr*)
-        (if (and (eql t0 t1) (eql e0 e1))
-            (list :a a :b b)
-            :aborted))))
-"
+    (multiple-value-bind (t0 e0) (frlock-read-begin *fr*)
+      (let ((a (get-a))
+            (b (get-b)))
+        (multiple-value-bind (t1 e1) (frlock-read-end *fr*)
+          (if (and (eql t0 t1) (eql e0 e1))
+              (list :a a :b b)
+              :aborted))))"
   (barrier (:read))
   (values (frlock-pre-counter frlock)
           (frlock-epoch frlock)))
@@ -247,7 +245,7 @@ set, and returns that as multiple values."
       ;; 0, they will still be holding on to the old epoch. While it is
       ;; extremely unlikely, it isn't quite "not before heath death of the
       ;; universe" stuff: a 30 bit counter can roll over in a couple of
-      ;; seconds -- and a thread can easily be interrupted by eg. a timer for
+      ;; seconds -- and a thread can easily be interrupted by e.g. a timer for
       ;; that long, so a pathological system could be have a thread in a
       ;; danger-zone every second. Run that system for a year, and it would
       ;; have a 1 in 3 chance of hitting the incipient bug. Adding an epoch
@@ -268,7 +266,7 @@ set, and returns that as multiple values."
 (defun grab-frlock-write-lock (frlock &key (wait-p t) timeout)
   "Acquires FRLOCK for writing, invalidating existing and future read-tokens
 for the duration. Returns T on success, and NIL if the lock wasn't acquired
-due to eg. a timeout. Using FRLOCK-WRITE instead is recommended."
+due to e.g. a timeout. Using FRLOCK-WRITE instead is recommended."
   (without-interrupts
     (allow-with-interrupts (%%grab-frlock-write-lock frlock wait-p timeout))))
 

contrib/sb-concurrency/mailbox.lisp

@@ -22,7 +22,7 @@ SEND-MESSAGE adds a message to the mailbox, RECEIVE-MESSAGE waits till
 a message becomes available, whereas RECEIVE-MESSAGE-NO-HANG is a non-blocking
 variant, and RECEIVE-PENDING-MESSAGES empties the entire mailbox in one go.
 
-Messages can be arbitrary objects"
+Messages can be arbitrary objects."
   (queue (missing-arg) :type queue)
   (semaphore (missing-arg) :type semaphore)
   (name nil))
@@ -56,7 +56,7 @@ Messages can be arbitrary objects"
 
 (declaim (ftype (sfunction (mailbox) unsigned-byte) mailbox-count))
 (defun mailbox-count (mailbox)
-  "Returns the number of messages currently in the mailbox."
+  "Returns the number of messages currently in MAILBOX."
   (semaphore-count (mailbox-semaphore mailbox)))
 
 (declaim (ftype (sfunction (mailbox) boolean) mailbox-empty-p))
@@ -66,8 +66,8 @@ Messages can be arbitrary objects"
 
 (declaim (ftype (sfunction (mailbox) list) list-mailbox-messages))
 (defun list-mailbox-messages (mailbox)
-  "Returns a fresh list containing all the messages in the
-mailbox. Does not remove messages from the mailbox."
+  "Returns a fresh list containing all the messages in MAILBOX. Does not
+remove messages from the mailbox."
   (list-queue-contents (mailbox-queue mailbox)))
 
 (declaim (ftype (sfunction (mailbox t) null) send-message))
@@ -126,10 +126,10 @@ message could be received."
   "Removes and returns all (or at most N) currently pending messages
 from MAILBOX, or returns NIL if no messages are pending.
 
-Note: Concurrent threads may be snarfing messages during the run of
-this function, so even though X,Y appear right next to each other in
-the result, does not necessarily mean that Y was the message sent
-right after X."
+> _Note_: Concurrent threads may be snarfing messages during the run
+> of this function, so even `X` and `Y` appearing right next to each
+> other in the result does not necessarily mean that `Y` was the
+> message sent right after `X`."
   (prog* ((msgs  '())
           (sem   (mailbox-semaphore mailbox))
           (queue (mailbox-queue mailbox))

contrib/sb-concurrency/manual.lisp

@@ -0,0 +1,72 @@
+(in-package :sb-manual)
+
+(defsection @sb-concurrency (:title "sb-concurrency")
+  "Additional data structures, synchronization primitives and tools for
+  concurrent programming. Similiar to Java's `java.util.concurrent`
+  package."
+  (@sb-concurrency-queue section)
+  (@sb-concurrency-mailbox section)
+  (@sb-concurrency-gates section)
+  (@sb-concurrency-frlocks section))
+
+(defsection @sb-concurrency-queue (:title "Queue")
+  "SB-CONCURRENCY:QUEUE is a lock-free, thread-safe FIFO queue
+  datatype.
+
+  The implementation is based on _An Optimistic Approach to Lock-Free
+  FIFO Queues_ by Edya Ladan-Mozes and Nir Shavit.
+
+  Before SBCL 1.0.38, this implementation resided in its own contrib
+  (see @SB-QUEUE), which is still provided for
+  backwards-compatibility, but which has since been deprecated."
+  (sb-concurrency:queue structure)
+  (sb-concurrency:dequeue function)
+  (sb-concurrency:enqueue function)
+  (sb-concurrency:list-queue-contents function)
+  (sb-concurrency:make-queue function)
+  (sb-concurrency:queue-count function)
+  (sb-concurrency:queue-empty-p function)
+  (sb-concurrency:queue-name function)
+  (sb-concurrency:queuep function))
+
+(defsection @sb-concurrency-mailbox (:title "Mailbox (lock-free)")
+  "SB-CONCURRENCY:MAILBOX is a lock-free message queue where one or
+  multiple ends can send messages to one or multiple receivers. The
+  difference to @SB-CONCURRENCY-QUEUE is that the receiving end may
+  block until a message arrives.
+
+  Built on top of the @SB-CONCURRENCY-QUEUE implementation."
+  (sb-concurrency:mailbox structure)
+  (sb-concurrency:list-mailbox-messages function)
+  (sb-concurrency:mailbox-count function)
+  (sb-concurrency:mailbox-empty-p function)
+  (sb-concurrency:mailbox-name function)
+  (sb-concurrency:mailboxp function)
+  (sb-concurrency:make-mailbox function)
+  (sb-concurrency:receive-message function)
+  (sb-concurrency:receive-message-no-hang function)
+  (sb-concurrency:receive-pending-messages function)
+  (sb-concurrency:send-message function))
+
+(defsection @sb-concurrency-gates (:title "Gates")
+  "SB-CONCURRENCY:GATE is a synchronization object suitable for when
+  multiple threads must wait for a single event before proceeding."
+  (sb-concurrency:gate structure)
+  (sb-concurrency:close-gate function)
+  (sb-concurrency:gate-name function)
+  (sb-concurrency:gate-open-p function)
+  (sb-concurrency:gatep function)
+  (sb-concurrency:make-gate function)
+  (sb-concurrency:open-gate function)
+  (sb-concurrency:wait-on-gate function))
+
+(defsection @sb-concurrency-frlocks (:title "Frlocks, aka Fast Read Locks")
+  (sb-concurrency:frlock structure)
+  (sb-concurrency:frlock-read macro)
+  (sb-concurrency:frlock-write macro)
+  (sb-concurrency:make-frlock function)
+  (sb-concurrency:frlock-name function)
+  (sb-concurrency:frlock-read-begin function)
+  (sb-concurrency:frlock-read-end function)
+  (sb-concurrency:grab-frlock-write-lock function)
+  (sb-concurrency:release-frlock-write-lock function))

contrib/sb-concurrency/sb-concurrency.texinfo

@@ -1,95 +1,336 @@
-@node sb-concurrency
+@c Generated by the sb-manual contrib. Do not edit.
+
+@node sb concurrency
 @section sb-concurrency
-@cindex Concurrency
-@cindex Sb-concurrency
+
+@menu
+* Queue: sb concurrency queue.
+* Mailbox (lock-free): sb concurrency mailbox.
+* Gates: sb concurrency gates.
+* Frlocks, aka Fast Read Locks: sb concurrency frlocks.
+@end menu
 
 Additional data structures, synchronization primitives and tools for
 concurrent programming. Similiar to Java's @code{java.util.concurrent}
 package.
 
-@page
-@anchor{Section sb-concurrency:queue}
+@node sb concurrency queue
 @subsection Queue
-@cindex Queue, lock-free
 
 @code{sb-concurrency:queue} is a lock-free, thread-safe FIFO queue
 datatype.
-@*@*
-The implementation is based on @cite{An Optimistic Approach to
-Lock-Free FIFO Queues} by Edya Ladan-Mozes and Nir Shavit.
-@*@*
+
+The implementation is based on @emph{An Optimistic Approach to Lock-Free
+FIFO Queues} by Edya Ladan-Mozes and Nir Shavit.
+
 Before SBCL 1.0.38, this implementation resided in its own contrib
-(@pxref{sb-queue}) which is still provided for backwards-compatibility
-but which has since been deprecated.
+(see @ref{sb queue}), which is still provided for
+backwards-compatibility, but which has since been deprecated.
 
-@include struct-sb-concurrency-queue.texinfo
+@anchor{Structure sb-concurrency queue}
+@ttindex @sortas{queue sb-concurrency} queue [sb-concurrency]
+@deffn{Structure} sb-concurrency:queue
+Lock-free thread safe FIFO queue.
 
-@include fun-sb-concurrency-dequeue.texinfo
-@include fun-sb-concurrency-enqueue.texinfo
-@include fun-sb-concurrency-list-queue-contents.texinfo
-@include fun-sb-concurrency-make-queue.texinfo
-@include fun-sb-concurrency-queue-count.texinfo
-@include fun-sb-concurrency-queue-empty-p.texinfo
-@include fun-sb-concurrency-queue-name.texinfo
-@include fun-sb-concurrency-queuep.texinfo
-
-@page
+Use @code{enqueue} to add objects to the queue, and @code{dequeue} to remove them.
+@end deffn
+@anchor{Function sb-concurrency dequeue}
+@ffindex @sortas{dequeue sb-concurrency} dequeue [sb-concurrency]
+@deffn{Function} sb-concurrency:dequeue queue
+Retrieves the oldest value in @code{queue} and returns it as the primary value,
+and @code{t} as secondary value. If the queue is empty, returns @code{nil} as both primary
+and secondary value.
+@end deffn
+@anchor{Function sb-concurrency enqueue}
+@ffindex @sortas{enqueue sb-concurrency} enqueue [sb-concurrency]
+@deffn{Function} sb-concurrency:enqueue value queue
+Adds @code{value} to the end of @code{queue}. Returns @code{value}.
+@end deffn
+@anchor{Function sb-concurrency list-queue-contents}
+@ffindex @sortas{list-queue-contents sb-concurrency} list-queue-contents [sb-concurrency]
+@deffn{Function} sb-concurrency:list-queue-contents queue
+Returns the contents of @code{queue} as a list without removing them from the
+@code{queue}. Mainly useful for manual examination of queue state, as the list may be
+out of date by the time it is returned, and concurrent dequeue operations may
+in the worse case force the queue-traversal to be restarted several times.
+@end deffn
+@anchor{Function sb-concurrency make-queue}
+@ffindex @sortas{make-queue sb-concurrency} make-queue [sb-concurrency]
+@deffn{Function} sb-concurrency:make-queue &key name initial-contents
+Returns a new @code{queue} with @code{name} and contents of the @code{initial-contents}
+sequence enqueued.
+@end deffn
+@anchor{Function sb-concurrency queue-count}
+@ffindex @sortas{queue-count sb-concurrency} queue-count [sb-concurrency]
+@deffn{Function} sb-concurrency:queue-count queue
+Returns the number of objects in @code{queue}. Mainly useful for manual
+examination of queue state, and in @code{print-object} methods: inefficient as it
+must walk the entire queue.
+@end deffn
+@anchor{Function sb-concurrency queue-empty-p}
+@ffindex @sortas{queue-empty-p sb-concurrency} queue-empty-p [sb-concurrency]
+@deffn{Function} sb-concurrency:queue-empty-p queue
+Returns @code{t} if @code{queue} is empty, @code{nil} otherwise.
+@end deffn
+@anchor{Function sb-concurrency queue-name}
+@ffindex @sortas{queue-name sb-concurrency} queue-name [sb-concurrency]
+@deffn{Function} sb-concurrency:queue-name instance
+Name of a @code{queue}. Can be assigned to using @code{setf}. Queue names
+can be arbitrary printable objects, and need not be unique.
+@end deffn
+@anchor{Function sb-concurrency queuep}
+@ffindex @sortas{queuep sb-concurrency} queuep [sb-concurrency]
+@deffn{Function} sb-concurrency:queuep object
+Returns true if argument is a @code{queue}, @code{nil} otherwise.
+@end deffn
+@node sb concurrency mailbox
 @subsection Mailbox (lock-free)
-@cindex Mailbox, lock-free
 
-@code{sb-concurrency:mailbox} is a lock-free message queue where one
-or multiple ends can send messages to one or multiple receivers. The
-difference to @ref{Section sb-concurrency:queue, queues} is that the receiving
-end may block until a message arrives.
-@*@*
-Built on top of the @ref{Structure sb-concurrency queue, queue} implementation.
+@code{sb-concurrency:mailbox} is a lock-free message queue where one or
+multiple ends can send messages to one or multiple receivers. The
+difference to @ref{sb concurrency queue} is that the receiving end may
+block until a message arrives.
 
-@include struct-sb-concurrency-mailbox.texinfo
+Built on top of the @ref{sb concurrency queue} implementation.
 
-@include fun-sb-concurrency-list-mailbox-messages.texinfo
-@include fun-sb-concurrency-mailbox-count.texinfo
-@include fun-sb-concurrency-mailbox-empty-p.texinfo
-@include fun-sb-concurrency-mailbox-name.texinfo
-@include fun-sb-concurrency-mailboxp.texinfo
-@include fun-sb-concurrency-make-mailbox.texinfo
-@include fun-sb-concurrency-receive-message.texinfo
-@include fun-sb-concurrency-receive-message-no-hang.texinfo
-@include fun-sb-concurrency-receive-pending-messages.texinfo
-@include fun-sb-concurrency-send-message.texinfo
+@anchor{Structure sb-concurrency mailbox}
+@ttindex @sortas{mailbox sb-concurrency} mailbox [sb-concurrency]
+@deffn{Structure} sb-concurrency:mailbox
+Mailbox aka message queue.
 
-@page
-@anchor{Section sb-concurrency:gate}
+@code{send-message} adds a message to the mailbox, @code{receive-message} waits till
+a message becomes available, whereas @code{receive-message-no-hang} is a non-blocking
+variant, and @code{receive-pending-messages} empties the entire mailbox in one go.
+
+Messages can be arbitrary objects.
+@end deffn
+@anchor{Function sb-concurrency list-mailbox-messages}
+@ffindex @sortas{list-mailbox-messages sb-concurrency} list-mailbox-messages [sb-concurrency]
+@deffn{Function} sb-concurrency:list-mailbox-messages mailbox
+Returns a fresh list containing all the messages in @code{mailbox}. Does not
+remove messages from the mailbox.
+@end deffn
+@anchor{Function sb-concurrency mailbox-count}
+@ffindex @sortas{mailbox-count sb-concurrency} mailbox-count [sb-concurrency]
+@deffn{Function} sb-concurrency:mailbox-count mailbox
+Returns the number of messages currently in @code{mailbox}.
+@end deffn
+@anchor{Function sb-concurrency mailbox-empty-p}
+@ffindex @sortas{mailbox-empty-p sb-concurrency} mailbox-empty-p [sb-concurrency]
+@deffn{Function} sb-concurrency:mailbox-empty-p mailbox
+Returns true if @code{mailbox} is currently empty, @code{nil} otherwise.
+@end deffn
+@anchor{Function sb-concurrency mailbox-name}
+@ffindex @sortas{mailbox-name sb-concurrency} mailbox-name [sb-concurrency]
+@deffn{Function} sb-concurrency:mailbox-name instance
+Name of a @code{mailbox}. @code{setf}able.
+@end deffn
+@anchor{Function sb-concurrency mailboxp}
+@ffindex @sortas{mailboxp sb-concurrency} mailboxp [sb-concurrency]
+@deffn{Function} sb-concurrency:mailboxp object
+Returns true if argument is a @code{mailbox}, @code{nil} otherwise.
+@end deffn
+@anchor{Function sb-concurrency make-mailbox}
+@ffindex @sortas{make-mailbox sb-concurrency} make-mailbox [sb-concurrency]
+@deffn{Function} sb-concurrency:make-mailbox &key name initial-contents
+Returns a new @code{mailbox} with messages in @code{initial-contents} enqueued.
+@end deffn
+@anchor{Function sb-concurrency receive-message}
+@ffindex @sortas{receive-message sb-concurrency} receive-message [sb-concurrency]
+@deffn{Function} sb-concurrency:receive-message mailbox &key timeout
+Removes the oldest message from @code{mailbox} and returns it as the primary
+value, and a secondary value of @code{t}. If @code{mailbox} is empty waits until a message
+arrives.
+
+If @code{timeout} is provided, and no message arrives within the specified interval,
+returns primary and secondary value of @code{nil}.
+@end deffn
+@anchor{Function sb-concurrency receive-message-no-hang}
+@ffindex @sortas{receive-message-no-hang sb-concurrency} receive-message-no-hang [sb-concurrency]
+@deffn{Function} sb-concurrency:receive-message-no-hang mailbox
+The non-blocking variant of @code{receive-message}. Returns two values,
+the message removed from @code{mailbox}, and a flag specifying whether a
+message could be received.
+@end deffn
+@anchor{Function sb-concurrency receive-pending-messages}
+@ffindex @sortas{receive-pending-messages sb-concurrency} receive-pending-messages [sb-concurrency]
+@deffn{Function} sb-concurrency:receive-pending-messages mailbox &optional n
+Removes and returns all (or at most @code{n}) currently pending messages
+from @code{mailbox}, or returns @code{nil} if no messages are pending.
+
+@quotation
+@emph{Note}: Concurrent threads may be snarfing messages during the run
+of this function, so even @code{x} and @code{y} appearing right next to each
+other in the result does not necessarily mean that @code{y} was the
+message sent right after @code{x}.
+@end quotation
+@end deffn
+@anchor{Function sb-concurrency send-message}
+@ffindex @sortas{send-message sb-concurrency} send-message [sb-concurrency]
+@deffn{Function} sb-concurrency:send-message mailbox message
+Adds a @code{message} to @code{mailbox}. Message can be any object.
+@end deffn
+@node sb concurrency gates
 @subsection Gates
-@cindex Gate
 
 @code{sb-concurrency:gate} is a synchronization object suitable for when
 multiple threads must wait for a single event before proceeding.
 
-@include struct-sb-concurrency-gate.texinfo
+@anchor{Structure sb-concurrency gate}
+@ttindex @sortas{gate sb-concurrency} gate [sb-concurrency]
+@deffn{Structure} sb-concurrency:gate
+@code{gate} type. Gates are synchronization constructs suitable for making
+multiple threads wait for single event before proceeding.
 
-@include fun-sb-concurrency-close-gate.texinfo
-@include fun-sb-concurrency-gate-name.texinfo
-@include fun-sb-concurrency-gate-open-p.texinfo
-@include fun-sb-concurrency-gatep.texinfo
-@include fun-sb-concurrency-make-gate.texinfo
-@include fun-sb-concurrency-open-gate.texinfo
-@include fun-sb-concurrency-wait-on-gate.texinfo
-
-@page
-@anchor{Section sb-concurrency:frlock}
+Use @code{wait-on-gate} to wait for a gate to open, @code{open-gate} to open one,
+and @code{close-gate} to close an open gate. @code{gate-open-p} can be used to test
+the state of a gate without blocking.
+@end deffn
+@anchor{Function sb-concurrency close-gate}
+@ffindex @sortas{close-gate sb-concurrency} close-gate [sb-concurrency]
+@deffn{Function} sb-concurrency:close-gate gate
+Closes @code{gate}. Returns @code{t} if the gate was previously open, and @code{nil}
+if the gate was already closed.
+@end deffn
+@anchor{Function sb-concurrency gate-name}
+@ffindex @sortas{gate-name sb-concurrency} gate-name [sb-concurrency]
+@deffn{Function} sb-concurrency:gate-name instance
+Name of a @code{gate}. @code{setf}able.
+@end deffn
+@anchor{Function sb-concurrency gate-open-p}
+@ffindex @sortas{gate-open-p sb-concurrency} gate-open-p [sb-concurrency]
+@deffn{Function} sb-concurrency:gate-open-p gate
+Returns true if @code{gate} is open.
+@end deffn
+@anchor{Function sb-concurrency gatep}
+@ffindex @sortas{gatep sb-concurrency} gatep [sb-concurrency]
+@deffn{Function} sb-concurrency:gatep object
+Returns true if the argument is a @code{gate}.
+@end deffn
+@anchor{Function sb-concurrency make-gate}
+@ffindex @sortas{make-gate sb-concurrency} make-gate [sb-concurrency]
+@deffn{Function} sb-concurrency:make-gate &key name open
+Makes a new gate. Gate will be initially open if @code{open} is true, and closed if @code{open}
+is @code{nil} (the default.) @code{name}, if provided, is the name of the gate, used when printing
+the gate.
+@end deffn
+@anchor{Function sb-concurrency open-gate}
+@ffindex @sortas{open-gate sb-concurrency} open-gate [sb-concurrency]
+@deffn{Function} sb-concurrency:open-gate gate
+Opens @code{gate}. Returns @code{t} if the gate was previously closed, and @code{nil}
+if the gate was already open.
+@end deffn
+@anchor{Function sb-concurrency wait-on-gate}
+@ffindex @sortas{wait-on-gate sb-concurrency} wait-on-gate [sb-concurrency]
+@deffn{Function} sb-concurrency:wait-on-gate gate &key timeout
+Waits for @code{gate} to open, or @code{timeout} seconds to pass. Returns @code{t}
+if the gate was opened in time, and @code{nil} otherwise.
+@end deffn
+@node sb concurrency frlocks
 @subsection Frlocks, aka Fast Read Locks
-@cindex Frlock
-@cindex Fast Read Lock
 
-@include struct-sb-concurrency-frlock.texinfo
+@anchor{Structure sb-concurrency frlock}
+@ttindex @sortas{frlock sb-concurrency} frlock [sb-concurrency]
+@deffn{Structure} sb-concurrency:frlock
+FRlock, aka Fast Read Lock.
 
-@include macro-sb-concurrency-frlock-read.texinfo
-@include macro-sb-concurrency-frlock-write.texinfo
+Fast Read Locks allow multiple readers and one potential writer to operate in
+parallel while providing for consistency for readers and mutual exclusion for
+writers.
 
-@include fun-sb-concurrency-make-frlock.texinfo
-@include fun-sb-concurrency-frlock-name.texinfo
+Readers gain entry to protected regions without waiting, but need to retry if
+a writer operated inside the region while they were reading. This makes frlocks
+very efficient when readers are much more common than writers.
 
-@include fun-sb-concurrency-frlock-read-begin.texinfo
-@include fun-sb-concurrency-frlock-read-end.texinfo
-@include fun-sb-concurrency-grab-frlock-write-lock.texinfo
-@include fun-sb-concurrency-release-frlock-write-lock.texinfo
+FRlocks are @emph{not} suitable when it is not safe at all for readers and writers
+to operate on the same data in parallel: they provide consistency, not
+exclusion between readers and writers. Hence using an frlock to e.g. protect
+an SBCL hash-table is unsafe. If multiple readers operating in parallel with
+a writer would be safe but inconsistent without a lock, frlocks are suitable.
+
+The recommended interface to use is @code{frlock-read} and @code{frlock-write}, but those
+needing it can also use a lower-level interface.
+
+Example:
+
+@example
+;; Values returned by FOO are always consistent so that
+;; the third value is the sum of the two first ones.
+(let ((a 0)
+      (b 0)
+      (c 0)
+      (lk (make-frlock)))
+  (defun foo ()
+     (frlock-read (lk) a b c))
+  (defun bar (x y)
+     (frlock-write (lk)
+       (setf a x
+             b y
+             c (+ x y)))))
+@end example
+@end deffn
+@anchor{Macro sb-concurrency frlock-read}
+@ffindex @sortas{frlock-read sb-concurrency} frlock-read [sb-concurrency]
+@deffn{Macro} sb-concurrency:frlock-read (frlock) &body value-forms
+Evaluates @code{value-forms} under @code{frlock} till it obtains a consistent
+set, and returns that as multiple values.
+@end deffn
+@anchor{Macro sb-concurrency frlock-write}
+@ffindex @sortas{frlock-write sb-concurrency} frlock-write [sb-concurrency]
+@deffn{Macro} sb-concurrency:frlock-write (frlock &key wait-p timeout) &body body
+Executes @code{body} while holding @code{frlock} for writing.
+@end deffn
+@anchor{Function sb-concurrency make-frlock}
+@ffindex @sortas{make-frlock sb-concurrency} make-frlock [sb-concurrency]
+@deffn{Function} sb-concurrency:make-frlock &key name
+Returns a new @code{frlock} with @code{name}.
+@end deffn
+@anchor{Function sb-concurrency frlock-name}
+@ffindex @sortas{frlock-name sb-concurrency} frlock-name [sb-concurrency]
+@deffn{Function} sb-concurrency:frlock-name instance
+Name of an @code{frlock}. @code{setf}able.
+@end deffn
+@anchor{Function sb-concurrency frlock-read-begin}
+@ffindex @sortas{frlock-read-begin sb-concurrency} frlock-read-begin [sb-concurrency]
+@deffn{Function} sb-concurrency:frlock-read-begin frlock
+Start a read sequence on @code{frlock}. Returns a read-token and an epoch to be
+validated later.
+
+Using @code{frlock-read} instead is recommended.
+@end deffn
+@anchor{Function sb-concurrency frlock-read-end}
+@ffindex @sortas{frlock-read-end sb-concurrency} frlock-read-end [sb-concurrency]
+@deffn{Function} sb-concurrency:frlock-read-end frlock
+Ends a read sequence on @code{frlock}. Returns a token and an epoch. If the token
+and epoch are @code{eql} to the read-token and epoch returned by @code{frlock-read-begin},
+the values read under the @code{frlock} are consistent and can be used: if the values
+differ, the values are inconsistent and the read must be restated.
+
+Using @code{frlock-read} instead is recommended.
+
+Example:
+
+@example
+(multiple-value-bind (t0 e0) (frlock-read-begin *fr*)
+  (let ((a (get-a))
+        (b (get-b)))
+    (multiple-value-bind (t1 e1) (frlock-read-end *fr*)
+      (if (and (eql t0 t1) (eql e0 e1))
+          (list :a a :b b)
+          :aborted))))
+@end example
+@end deffn
+@anchor{Function sb-concurrency grab-frlock-write-lock}
+@ffindex @sortas{grab-frlock-write-lock sb-concurrency} grab-frlock-write-lock [sb-concurrency]
+@deffn{Function} sb-concurrency:grab-frlock-write-lock frlock &key wait-p timeout
+Acquires @code{frlock} for writing, invalidating existing and future read-tokens
+for the duration. Returns @code{t} on success, and @code{nil} if the lock wasn't acquired
+due to e.g. a timeout. Using @code{frlock-write} instead is recommended.
+@end deffn
+@anchor{Function sb-concurrency release-frlock-write-lock}
+@ffindex @sortas{release-frlock-write-lock sb-concurrency} release-frlock-write-lock [sb-concurrency]
+@deffn{Function} sb-concurrency:release-frlock-write-lock frlock
+Releases @code{frlock} after writing, allowing valid read-tokens to be acquired again.
+Signals an error if the current thread doesn't hold @code{frlock} for writing. Using @code{frlock-write}
+instead is recommended.
+@end deffn

contrib/sb-cover/cover.lisp

@@ -229,7 +229,7 @@
   (prog1 (sb-c::integer-from-octets octets) (setf (fill-pointer octets) 0)))
 
 (defun unpack-pair (packed-pair)
-  (let ((list (sb-c::unpack-code-fixup-locs packed-pair)))
+  (let ((list (sb-c:unpack-code-fixup-locs packed-pair)))
     (values (1- (car list)) ; 1 as encoded means 0, etc
             ;; If START and END were =, then the delta is 0, which can't be encoded,
             ;; so the pair reads back as only one integer, which we just repeat.

contrib/sb-cover/genhtml.lisp

@@ -102,11 +102,11 @@ If DIRECTORY does not exist, it will be created. The main report will be
 printed to the file cover-index.html. The external format of the source
 files can be specified with the EXTERNAL-FORMAT parameter.
 
-If the keyword argument FORM-MODE has the value :CAR, the annotations in
-the coverage report will be placed on the CARs of any cons-forms, while if
-it has the value :WHOLE the whole form will be annotated (the default).
-The former mode shows explicitly which forms were instrumented, while the
-latter mode is generally easier to read.
+If the keyword argument :FORM-MODE has the value :CAR, the annotations
+in the coverage report will be placed on the CARs of any cons-forms,
+while if it has the value :WHOLE the whole form will be annotated (the
+default). The former mode shows explicitly which forms were
+instrumented, while the latter mode is generally easier to read.
 
 The keyword argument IF-MATCHES should be a designator for a function
 of one argument, called for the namestring of each file with code

contrib/sb-cover/manual.lisp

@@ -0,0 +1,42 @@
+(in-package :sb-manual)
+
+;;; FIXME: Write some documentation about how to interpret the results.
+(defsection @sb-cover (:title "sb-cover")
+  "The `SB-COVER` module provides a code coverage tool for SBCL. The
+  tool has support for expression coverage, and for some branch
+  coverage. Coverage reports are only generated for code compiled
+  using COMPILE-FILE with the value of the
+  SB-COVER:STORE-COVERAGE-DATA optimization quality set to 3.
+
+  As of SBCL 1.0.6, `SB-COVER` is still experimental, and the
+  interfaces documented here might change in later versions.
+
+  How to use it:
+
+      ;;; Load SB-COVER
+      (require :sb-cover)
+
+      ;;; Turn on generation of code coverage instrumentation in the compiler
+      (declaim (optimize sb-cover:store-coverage-data))
+
+      ;;; Load some code, ensuring that it's recompiled with the new optimization
+      ;;; policy.
+      (asdf:oos 'asdf:load-op :cl-ppcre-test :force t)
+
+      ;;; Run the test suite.
+      (cl-ppcre-test:test)
+
+      ;;; Produce a coverage report
+      (sb-cover:report \"/tmp/report/\")
+
+      ;;; Turn off instrumentation
+      (declaim (optimize (sb-cover:store-coverage-data 0)))"
+  (sb-cover:report function)
+  (sb-cover:reset-coverage function)
+  (sb-cover:clear-coverage function)
+  (sb-cover:save-coverage function)
+  (sb-cover:save-coverage-in-file function)
+  (sb-cover:restore-coverage function)
+  (sb-cover:restore-coverage-from-file function)
+  (sb-cover:merge-coverage function)
+  (sb-cover:merge-coverage-from-file function))

contrib/sb-cover/sb-cover.texinfo

@@ -1,19 +1,20 @@
-@node sb-cover
+@c Generated by the sb-manual contrib. Do not edit.
+
+@node sb cover
 @section sb-cover
-@cindex Code Coverage
 
 The @code{sb-cover} module provides a code coverage tool for SBCL. The
-tool has support for expression coverage, and for some branch coverage.
-Coverage reports are only generated for code compiled using
-@code{compile-file} with the value of the
+tool has support for expression coverage, and for some branch
+coverage. Coverage reports are only generated for code compiled
+using @code{compile-file} with the value of the
 @code{sb-cover:store-coverage-data} optimization quality set to 3.
 
-As of SBCL 1.0.6 @code{sb-cover} is still experimental, and the
+As of SBCL 1.0.6, @code{sb-cover} is still experimental, and the
 interfaces documented here might change in later versions.
 
-@subsection Example Usage
+How to use it:
 
-@lisp
+@example
 ;;; Load SB-COVER
 (require :sb-cover)
 
@@ -32,27 +33,76 @@ interfaces documented here might change in later versions.
 
 ;;; Turn off instrumentation
 (declaim (optimize (sb-cover:store-coverage-data 0)))
-@end lisp
+@end example
 
-@c @subsection Output
-@c Write some documentation about how to interpret the results
+@anchor{Function sb-cover report}
+@ffindex @sortas{report sb-cover} report [sb-cover]
+@deffn{Function} sb-cover:report directory &key form-mode if-matches external-format
+Print a code coverage report of all instrumented files into @code{directory}.
+If @code{directory} does not exist, it will be created. The main report will be
+printed to the file cover-index.html. The external format of the source
+files can be specified with the @code{external-format} parameter.
 
-@subsection Functions
+If the keyword argument @code{:form-mode} has the value @code{:car}, the annotations
+in the coverage report will be placed on the @code{car}s of any cons-forms,
+while if it has the value @code{:whole} the whole form will be annotated (the
+default). The former mode shows explicitly which forms were
+instrumented, while the latter mode is generally easier to read.
 
-@include fun-sb-cover-report.texinfo
-
-@include fun-sb-cover-reset-coverage.texinfo
-
-@include fun-sb-cover-clear-coverage.texinfo
-
-@include fun-sb-cover-save-coverage.texinfo
-
-@include fun-sb-cover-save-coverage-in-file.texinfo
-
-@include fun-sb-cover-restore-coverage.texinfo
-
-@include fun-sb-cover-restore-coverage-from-file.texinfo
-
-@include fun-sb-cover-merge-coverage.texinfo
-
-@include fun-sb-cover-merge-coverage-from-file.texinfo
+The keyword argument @code{if-matches} should be a designator for a function
+of one argument, called for the namestring of each file with code
+coverage info. If it returns true, the file's info is included in the
+report, otherwise ignored. The default value is @code{cl:identity}.
+@end deffn
+@anchor{Function sb-cover reset-coverage}
+@ffindex @sortas{reset-coverage sb-cover} reset-coverage [sb-cover]
+@deffn{Function} sb-cover:reset-coverage &optional object
+Reset all coverage data back to the @code{Not executed} state.
+@end deffn
+@anchor{Function sb-cover clear-coverage}
+@ffindex @sortas{clear-coverage sb-cover} clear-coverage [sb-cover]
+@deffn{Function} sb-cover:clear-coverage
+Clear all files from the coverage database. The files will be re-entered
+into the database when the FASL files (produced by compiling
+@code{store-coverage-data} optimization policy set to 3) are loaded again into the
+image.
+@end deffn
+@anchor{Function sb-cover save-coverage}
+@ffindex @sortas{save-coverage sb-cover} save-coverage [sb-cover]
+@deffn{Function} sb-cover:save-coverage
+Returns an opaque representation of the current code coverage state.
+The only operation that may be done on the state is passing it to
+@code{restore-coverage}. The representation is guaranteed to be readably printable.
+A representation that has been printed and read back will work identically
+in @code{restore-coverage}.
+@end deffn
+@anchor{Function sb-cover save-coverage-in-file}
+@ffindex @sortas{save-coverage-in-file sb-cover} save-coverage-in-file [sb-cover]
+@deffn{Function} sb-cover:save-coverage-in-file pathname
+Call @code{save-coverage} and write the results of that operation into the
+file designated by @code{pathname}.
+@end deffn
+@anchor{Function sb-cover restore-coverage}
+@ffindex @sortas{restore-coverage sb-cover} restore-coverage [sb-cover]
+@deffn{Function} sb-cover:restore-coverage coverage-state
+Restore the code coverage data back to an earlier state produced by
+@code{save-coverage}.
+@end deffn
+@anchor{Function sb-cover restore-coverage-from-file}
+@ffindex @sortas{restore-coverage-from-file sb-cover} restore-coverage-from-file [sb-cover]
+@deffn{Function} sb-cover:restore-coverage-from-file pathname
+@code{read} the contents of the file designated by @code{pathname} and pass the
+result to @code{restore-coverage}.
+@end deffn
+@anchor{Function sb-cover merge-coverage}
+@ffindex @sortas{merge-coverage sb-cover} merge-coverage [sb-cover]
+@deffn{Function} sb-cover:merge-coverage coverage-state
+Merge the code coverage data to include covered code from an earlier
+state produced by @code{save-coverage}.
+@end deffn
+@anchor{Function sb-cover merge-coverage-from-file}
+@ffindex @sortas{merge-coverage-from-file sb-cover} merge-coverage-from-file [sb-cover]
+@deffn{Function} sb-cover:merge-coverage-from-file pathname
+@code{read} the contents of the file designated by @code{pathname} and pass the
+result to @code{merge-coverage}.
+@end deffn

contrib/sb-gmp/gmp.lisp

@@ -62,6 +62,7 @@
   (setf (sb-int:system-package-p *package*) t))
 
 (defvar *gmp-disabled* nil)
+(declaim (sb-ext:always-bound *gmp-disabled*))
 
 (defconstant +bignum-raw-area-offset+
   (- (* sb-vm:bignum-digits-offset sb-vm:n-word-bytes)
@@ -960,21 +961,26 @@ pre-allocated bignum. The allocated bignum-length must be (1+ COUNT)."
   (declare (inline mpz-mul-2exp mpz-pow)
            (optimize (sb-c:verify-arg-count 0)))
   (cond
-    ((or (and (integerp base)
-              (< (abs power) 1000)
-              (< (blength base) 4))
-         (member base '(0 1 -1))
+    ((or (not (typep power '(integer #.(1+ most-negative-fixnum) #.most-positive-fixnum)))
+         (if (integerp base)
+             (and
+              (< (blength base) 4)
+              (typep power '(signed-byte 10)))
+             ;; EXPT dispatches to INTEXP for a (COMPLEX RATIONAL) base as well,
+             ;; and MPZ-POW below only takes an integer.
+             (not (typep base 'ratio)))
          *gmp-disabled*)
      (orig-intexp base power))
     (t
-     (check-type power (integer #.(1+ most-negative-fixnum) #.most-positive-fixnum))
      (cond ((minusp power)
-            (/ (gmp-intexp base (- power))))
+            (let ((abs-power (- power)))
+              (sb-kernel:build-ratio (sb-ext:truly-the integer (gmp-intexp (denominator base) abs-power))
+                                     (sb-ext:truly-the integer (gmp-intexp (numerator base) abs-power)))))
            ((eql base 2)
             (mpz-mul-2exp 1 power))
            ((typep base 'ratio)
-            (sb-kernel::%make-ratio (gmp-intexp (numerator base) power)
-                                    (gmp-intexp (denominator base) power)))
+            (sb-kernel::%make-ratio (sb-ext:truly-the integer (gmp-intexp (numerator base) power))
+                                    (sb-ext:truly-the integer (gmp-intexp (denominator base) power))))
            (t
             (mpz-pow base power))))))
 

contrib/sb-gmp/tests.lisp

@@ -207,6 +207,14 @@
     (sb-gmp::gmp-intexp 113/355 -1)
   355/113)
 
+;; EXPT dispatches to INTEXP for a (COMPLEX RATIONAL) base as well.
+(deftest intexp-complex-rational
+    (list (sb-gmp::gmp-intexp #c(1/5 1) 1)
+          (sb-gmp::gmp-intexp #c(1/5 1) 2)
+          (sb-gmp::gmp-intexp #c(2 3) 5)
+          (sb-gmp::gmp-intexp #c(1/5 1) -1))
+  (#c(1/5 1) #c(-24/25 2/5) #c(122 -597) #c(5/26 -25/26)))
+
 (deftest remove-1
     (multiple-value-list (mpz-remove 28 2))
   (7 2))

contrib/sb-grovel/def-to-lisp.lisp

@@ -80,25 +80,39 @@ code:
   (destructuring-bind (cname &rest elements) cstruct
     (printf "(sb-grovel::define-c-struct ~A %ld" lispname
             (word-cast (format nil "sizeof(~A)" cname)))
-    (dolist (e elements)
-      (destructuring-bind (lisp-type lisp-el-name c-type c-el-name &key distrust-length) e
-        (printf " (~A ~A \"~A\"" lisp-el-name lisp-type c-type)
-        ;; offset
-        (as-c "{" cname "t;")
-        (printf "  %lu"
-                (format nil "((unsigned long~A)&(t.~A)) - ((unsigned long~A)&(t))"
-                        #+(and win32 64-bit) " long" #-(and win32 64-bit) ""
-                        c-el-name
-                        #+(and win32 64-bit) " long" #-(and win32 64-bit) ""))
-        (as-c "}")
-        ;; length
-        (if distrust-length
-            (printf "  0)")
-            (progn
-              (as-c "{" cname "t;")
-              (printf "  %ld)"
-                      (word-cast (format nil "sizeof(t.~A)" c-el-name)))
-              (as-c "}")))))
+    (when elements
+      (as-c "{" cname "t;")
+      (dolist (e elements)
+        (destructuring-bind (lisp-type lisp-el-name c-type c-el-name &key distrust-length) e
+          (printf " (~A ~A \"~A\"" lisp-el-name lisp-type c-type)
+          ;; offset
+          (flet ((ifdef (fun c-el-name)
+                   (cond ((typep c-el-name '(cons (eql :if)))
+                          (destructuring-bind (cond then else) (cdr c-el-name)
+                            (as-c "#if" cond)
+                            (funcall fun then)
+                            (as-c "#else")
+                            (funcall fun else)
+                            (as-c "#endif")))
+                         (t
+                          (funcall fun c-el-name)))))
+            (ifdef (lambda (c-el-name)
+                     (printf "  %lu"
+                             (format nil "((unsigned long~A)&(t.~A)) - ((unsigned long~A)&(t))"
+                                     #+(and win32 64-bit) " long" #-(and win32 64-bit) ""
+                                     c-el-name
+                                     #+(and win32 64-bit) " long" #-(and win32 64-bit) "")))
+                   c-el-name)
+
+            ;; length
+            (if distrust-length
+                (printf "  0)")
+                (ifdef (lambda (c-el-name)
+                         (printf "  %ld)"
+
+                                 (word-cast (format nil "sizeof(t.~A)" c-el-name))))
+                       c-el-name)))))
+      (as-c "}"))
     (printf ")")))
 
 (defun print-c-source (stream headers definitions package-name)
@@ -195,7 +209,7 @@ int main(int argc, char *argv[]) {
      (sb-ext:run-program
       cc
       (append
-       (split-cflags (sb-ext:posix-getenv "EXTRA_CFLAGS"))
+       (split-cflags (sb-ext:posix-getenv "CFLAGS"))
        #+(and linux largefile)
        '("-D_LARGEFILE_SOURCE" "-D_LARGEFILE64_SOURCE" "-D_FILE_OFFSET_BITS=64")
        #+64-bit-time

contrib/sb-grovel/foreign-glue.lisp

@@ -339,47 +339,46 @@ deeply nested structures."
 
 (defmacro define-c-struct (name size &rest elements)
   (multiple-value-bind (struct-elements accessors)
-      (let* ((root (make-instance 'struct :name name :children nil :offset 0)))
+      (let ((root (make-instance 'struct :name name :children nil :offset 0)))
         (loop for e in (sort (copy-list elements) #'< :key #'fourth)
-              do (insert-element root (apply 'mk-val e))
-              finally (return root))
+              do (insert-element root (apply 'mk-val e)))
         (setf (children root)
               (nconc (children root)
-                     (list
-                      (mk-padding (max 0 (- size
-                                            (size root)))
-                                  (size root)))))
+                     (let ((pad (- size (size root))))
+                       (when (> pad 0)
+                         (list
+                          (mk-padding pad (size root)))))))
         (generate-struct-definition name root nil))
     `(progn
-           (sb-alien:define-alien-type ,@(first struct-elements))
-           ,@accessors
-           ;; This macro's lambda vars are uninterned so that they don't refer to
-           ;; the SB-GROVEL package, but they don't need to be GENSYMed.
-           (defmacro ,(sb-int:symbolicate "WITH-" name)
-               (alien (&rest #1=#:initializers) &body #2=#:body)
-             `(sb-alien:with-alien ((,alien ,',name))
-                (alien-funcall (extern-alien "memset"
-                                (function void system-area-pointer int sb-kernel::os-vm-size-t))
-                               (sb-alien:alien-sap ,alien) 0 ,,size)
-                (let ((,alien (cast ,alien (* ,',name))))
-                  (setf ,@(mapcan
-                           ;; The symbol CONS is not in the SB-GROVEL package, making it more
-                           ;; clear that this expander works fine in the absence of sb-grovel.
-                           (lambda (cons)
-                             `((,(sb-int:package-symbolicate ,(package-name (symbol-package name))
-                                                             ,(concatenate 'string (string name) "-")
-                                                             (car cons)) ,alien)
-                               ,(cadr cons)))
-                           #1#))
-                  ,@#2#)))
-           (defconstant ,(sb-int:symbolicate "SIZE-OF-" name) ,size) ; why does this exist?
-           (defun ,(sb-int:symbolicate "ALLOCATE-" name) () ; and this?
-             (let ((sb-kernel:instance (sb-alien:make-alien ,name)))
-               ;; The allocator returns 0-filled aliens. It's unknowable whether anyone cares.
-               (alien-funcall (extern-alien "memset"
-                               (function void system-area-pointer int sb-kernel::os-vm-size-t))
-                              (sb-alien:alien-sap sb-kernel:instance) 0 ,size)
-               sb-kernel:instance)))))
+       (sb-alien:define-alien-type ,@(first struct-elements))
+       ,@accessors
+       ;; This macro's lambda vars are uninterned so that they don't refer to
+       ;; the SB-GROVEL package, but they don't need to be GENSYMed.
+       (defmacro ,(sb-int:symbolicate "WITH-" name)
+           (alien (&rest #1=#:initializers) &body #2=#:body)
+         `(sb-alien:with-alien ((,alien ,',name))
+            (alien-funcall (extern-alien "memset"
+                                         (function void system-area-pointer int sb-kernel::os-vm-size-t))
+                           (sb-alien:alien-sap ,alien) 0 ,,size)
+            (let ((,alien (cast ,alien (* ,',name))))
+              (setf ,@(mapcan
+                       ;; The symbol CONS is not in the SB-GROVEL package, making it more
+                       ;; clear that this expander works fine in the absence of sb-grovel.
+                       (lambda (cons)
+                         `((,(sb-int:package-symbolicate ,(package-name (symbol-package name))
+                                                         ,(concatenate 'string (string name) "-")
+                                                         (car cons)) ,alien)
+                           ,(cadr cons)))
+                       #1#))
+              ,@#2#)))
+       (defconstant ,(sb-int:symbolicate "SIZE-OF-" name) ,size) ; why does this exist?
+       (defun ,(sb-int:symbolicate "ALLOCATE-" name) () ; and this?
+         (let ((sb-kernel:instance (sb-alien:make-alien ,name)))
+           ;; The allocator returns 0-filled aliens. It's unknowable whether anyone cares.
+           (alien-funcall (extern-alien "memset"
+                                        (function void system-area-pointer int sb-kernel::os-vm-size-t))
+                          (sb-alien:alien-sap sb-kernel:instance) 0 ,size)
+           sb-kernel:instance)))))
 
 ;; FIXME: Nothing in SBCL uses this, but kept it around in case there
 ;; are third-party sb-grovel clients.  It should go away eventually,

contrib/sb-grovel/manual.lisp

@@ -0,0 +1,208 @@
+(in-package :sb-manual)
+
+(defsection @sb-grovel (:title "sb-grovel")
+  "The `SB-GROVEL` module helps in generation of foreign function
+  interfaces. It aids in extracting constants' values from the C
+  compiler and in generating sb-alien structure and union types,
+  @DEFINING-FOREIGN-TYPES.
+
+  The ASDF (<http://www.cliki.net/ASDF>) component type
+  GROVEL-CONSTANTS-FILE has its ASDF:PERFORM operation defined to
+  write out a C source file, compile it, and run it. The output from
+  this program is Lisp, which is then itself compiled and loaded.
+
+  `SB-GROVEL` is used in a few contributed modules, and it is
+  currently compatible only to SBCL. However, if you want to use it,
+  here are a few directions."
+  (@using-sb-grovel section)
+  (@sb-grovel-constants-file section)
+  (@sb-grovel-structures section)
+  (@sb-grovel-traps section))
+
+(defsection @using-sb-grovel (:title "Using sb-grovel in your own ASDF System")
+  "- Create a Lisp package for the foreign constants/functions to go
+  into.
+
+  - Make your system depend on the `SB-GROVEL` system.
+
+  - Create a grovel-constants data file -- for an example, see
+    `example-constants.lisp` in the `contrib/sb-grovel/` directory in
+    the SBCL source distribution.
+
+  - Add it as a component in your system. For example:
+
+          (eval-when (:compile-toplevel :load-toplevel :execute)
+            (require :sb-grovel))
+
+          (defpackage :example-package.system
+                      (:use :cl :asdf :sb-grovel :sb-alien))
+
+          (in-package :example-package.system)
+
+          (defsystem example-system
+              :depends-on (sb-grovel)
+              :components
+              ((:module \"sbcl\"
+                        :components
+                        ((:file \"defpackage\")
+                         (grovel-constants-file \"example-constants\"
+                                                :package :example-package)))))
+
+  Make sure to specify the package you chose in step 1.
+
+  - Build stuff.")
+
+(defsection @sb-grovel-constants-file
+    (:title "Contents of a grovel-constants-file")
+  "The grovel-constants-file, typically named `constants.lisp`,
+  comprises lisp expressions describing the foreign things that you
+  want to grovel for. A `constants.lisp` file contains two sections:
+
+  - a list of headers to include in the C program, for example:
+
+          (\"sys/types.h\" \"sys/socket.h\" \"sys/stat.h\" \"unistd.h\" \"sys/un.h\"
+           \"netinet/in.h\" \"netinet/in_systm.h\" \"netinet/ip.h\" \"net/if.h\"
+           \"netdb.h\" \"errno.h\" \"netinet/tcp.h\" \"fcntl.h\" \"signal.h\")
+
+  - A list of sb-grovel clauses describing the things you want to
+    grovel from the C compiler, for example:
+
+          ((:integer af-local
+                     #+(or sunos solaris) \"AF_UNIX\"
+                     #-(or sunos solaris) \"AF_LOCAL\"
+                     \"Local to host (pipes and file-domain).\")
+           (:structure stat (\"struct stat\"
+                             (integer dev \"dev_t\" \"st_dev\")
+                             (integer atime \"time_t\" \"st_atime\")))
+           (:function getpid (\"getpid\" int )))
+
+  There are two types of things that sb-grovel can sensibly extract
+  from the C compiler: constant integers and structure layouts. It is
+  also possible to define foreign functions in the constants.lisp
+  file, but these definitions don't use any information from the C
+  program; they expand directly to SB-ALIEN:DEFINE-ALIEN-ROUTINE
+  forms.
+
+  Here's how to use the grovel clauses:
+
+  - :INTEGER: constant expressions in C. Used in this form:
+
+           (:integer lisp-variable-name \"C expression\" &optional doc export)
+
+      `\"C expression\"` will be typically be the name of a constant,
+      but other forms are possible.
+
+  - :ENUM:
+
+           (:enum lisp-type-name ((lisp-enumerated-name c-enumerated-name) ...)))
+
+      An SB-ALIEN:ENUM type with name `LISP-TYPE-NAME` will be
+      defined. The symbols are the `LISP-ENUMERATED-NAME`s, and the
+      values are grovelled from the `C-ENUMERATED-NAME`s.
+
+  - :STRUCTURE: alien structure definitions look like this:
+
+           (:structure lisp-struct-name (\"struct c_structure\"
+                                         (type-designator lisp-element-name
+                                          \"c_element_type\" \"c_element_name\"
+                                          :distrust-length nil)
+                                         ; ...
+                                         ))
+
+      `TYPE-DESIGNATOR` is a reference to a type whose size (and type
+      constraints) will be groveled for. sb-grovel accepts a form of
+      type designator that doesn't quite conform to either lisp nor
+      sb-alien's type specifiers. Here's a list of type designators
+      that sb-grovel currently accepts:
+
+      - `\\INTEGER`: a C integral type; sb-grovel will infer the exact
+        type from size information extracted from the C program. All
+        common C integer types can be grovelled for with this type
+        designator, but it is not possible to grovel for bit fields
+        yet.
+
+      - `(UNSIGNED N)`: an unsigned integer variable that is `N` bytes
+        long. No size information from the C program will be used.
+
+      - `(SIGNED N)`: an signed integer variable that is `N` bytes
+        long. No size information from the C program will be used.
+
+      - `\\C-STRING`: an array of `\\char` in the structure. sb-grovel
+        will use the array's length from the C program, unless you
+        pass it the :DISTRUST-LENGTH keyword argument with non-`NIL`
+        value (this might be required for structures such as solaris's
+        `struct dirent`).
+
+      - SB-GROVEL::C-STRING-POINTER: a pointer to a C string,
+        corresponding to the SB-ALIEN:C-STRING type (see
+        @FOREIGN-TYPE-SPECIFIERS).
+
+      - `(ARRAY ALIEN-TYPE)`: an array of the previously-declared
+        `ALIEN-TYPE`. The array's size will be determined from the
+        output of the C program and the alien type's size.
+
+      - `(ARRAY ALIEN-TYPE N):` an array of the previously-declared
+        `ALIEN-TYPE`. The array's size will be assumed as being `N`.
+
+  Note that `\\C-STRING` and SB-GROVEL::C-STRING-POINTER do not have
+  the same meaning. If you declare that an element is of type
+  C-STRING, it will be treated as if the string is a part of the
+  structure, whereas if you declare that the element is of type
+  SB-GROVEL::C-STRING-POINTER, a _pointer to a string_ will be the
+  structure member.
+
+  - :FUNCTION: alien function definitions are similar to
+    DEFINE-ALIEN-ROUTINE definitions, because they expand to such
+    forms when the lisp program is loaded. See
+    @FOREIGN-FUNCTION-CALLS.
+
+          (:function lisp-function-name
+                     (\"alien_function_name\" alien-return-type
+                                            (argument alien-type)
+                                            (argument2 alien-type)))")
+
+(defsection @sb-grovel-structures
+    (:title "Programming with sb-grovel's structure types")
+  "Let us assume that you have a grovelled structure definition:
+
+      (:structure mystruct (\"struct my_structure\"
+                            (integer myint \"int\" \"st_int\")
+                            (c-string mystring \"char[]\" \"st_str\")))
+
+  What can you do with it? Here's a short interface document:
+
+  - Creating and destroying objects:
+
+      - Function `(ALLOCATE-MYSTRUCT)` allocates an object of type
+        `mystruct` and returns a system area pointer to it.
+
+      - Macro `(WITH-MYSTRUCT VAR ((MEMBER INIT) [...]) &BODY BODY)`
+        allocates an object of type `MYSTRUCT` that is valid in
+        `BODY`. If `BODY` terminates or performs an non-local exit,
+        the object pointed to by `VAR` will be deallocated.
+
+  - Accessing structure members:
+
+      - `(MYSTRUCT-MYINT VAR)` and `(MYSTRUCT-MYSTRING VAR)` return
+        the value of the respective fields in `MYSTRUCT`.
+
+      - `(SETF (MYSTRUCT-MYINT VAR) NEW-VAL)` and
+        `(SETF (MYSTRUCT-MYSTRING VAR) NEW-VAL)` sets the value of the
+        respective structure member to the value of `NEW-VAL`. Notice
+        that in `(SETF (MYSTRUCT-MYSTRING VAR) NEW-VAL)`'s case,
+        `NEW-VAL` is a lisp string.")
+
+(defsection @sb-grovel-traps (:title "Traps and Pitfalls")
+  "Basically, you can treat functions and data structure definitions that
+  sb-grovel spits out as if they were alien routines and types. This has
+  a few implications that might not be immediately obvious (especially
+  if you have programmed in a previous version of sb-grovel that didn't
+  use alien types):
+
+  - You must take care of grovel-allocated structures yourself. They
+    are alien types, so the garbage collector will not collect them
+    when you drop the last reference.
+
+  - If you use the `WITH-MYSTRUCT` macro, be sure that no references
+    to the variable thus allocated leaks out. It will be deallocated
+    when the block exits.")

contrib/sb-grovel/sb-grovel.texinfo

@@ -1,42 +1,47 @@
-@node sb-grovel
+@c Generated by the sb-manual contrib. Do not edit.
+
+@node sb grovel
 @section sb-grovel
-@cindex Foreign Function Interface, generation
+
+@menu
+* Using sb-grovel in your own ASDF System: using sb grovel.
+* Contents of a grovel-constants-file: sb grovel constants file.
+* Programming with sb-grovel's structure types: sb grovel structures.
+* Traps and Pitfalls: sb grovel traps.
+@end menu
 
 The @code{sb-grovel} module helps in generation of foreign function
 interfaces. It aids in extracting constants' values from the C
-compiler and in generating SB-ALIEN structure and union types,
-@pxref{Defining Foreign Types}.
+compiler and in generating sb-alien structure and union types,
+@ref{defining foreign types}.
 
-The ASDF(@uref{http://www.cliki.net/ASDF}) component type
-GROVEL-CONSTANTS-FILE has its PERFORM
-@c @xref for PERFORM when asdf manual is included?
-operation defined to write out a C source file, compile it, and run
-it.  The output from this program is Lisp, which is then itself
-compiled and loaded.
+The ASDF (@url{http://www.cliki.net/ASDF}) component type
+GROVEL-CONSTANTS-FILE has its @code{asdf:perform} operation defined to
+write out a C source file, compile it, and run it. The output from
+this program is Lisp, which is then itself compiled and loaded.
 
-sb-grovel is used in a few contributed modules, and it is currently
-compatible only to SBCL. However, if you want to use it, here are a
-few directions.
+@code{sb-grovel} is used in a few contributed modules, and it is
+currently compatible only to SBCL. However, if you want to use it,
+here are a few directions.
 
-@subsection Using sb-grovel in your own ASDF system
+@node using sb grovel
+@subsection Using sb-grovel in your own ASDF System
 
-@enumerate
+@itemize
+@item Create a Lisp package for the foreign constants/functions to go
+@end itemize
+into.
 
-@item
-Create a Lisp package for the foreign constants/functions to go into.
+@itemize
+@item Make your system depend on the @code{sb-grovel} system.
 
-@item
-Make your system depend on the 'sb-grovel system.
+@item Create a grovel-constants data file -- for an example, see
+  @code{example-constants.lisp} in the @code{contrib/sb-grovel/} directory in
+  the SBCL source distribution.
 
-@item
-Create a grovel-constants data file - for an example, see
-example-constants.lisp in the contrib/sb-grovel/ directory in the SBCL
-source distribution.
+@item Add it as a component in your system. For example:
 
-@item
-Add it as a component in your system. e.g.
-
-@lisp
+@example
 (eval-when (:compile-toplevel :load-toplevel :execute)
   (require :sb-grovel))
 
@@ -53,34 +58,35 @@ Add it as a component in your system. e.g.
               ((:file "defpackage")
                (grovel-constants-file "example-constants"
                                       :package :example-package)))))
-@end lisp
+@end example
+@end itemize
 
-Make sure to specify the package you chose in step 1
+Make sure to specify the package you chose in step 1.
 
-@item
-Build stuff.
-
-@end enumerate
+@itemize
+@item Build stuff.
+@end itemize
 
+@node sb grovel constants file
 @subsection Contents of a grovel-constants-file
 
 The grovel-constants-file, typically named @code{constants.lisp},
-comprises lisp expressions describing the foreign things that you want
-to grovel for. A @code{constants.lisp} file contains two sections:
+comprises lisp expressions describing the foreign things that you
+want to grovel for. A @code{constants.lisp} file contains two sections:
 
 @itemize
-@item
-a list of headers to include in the C program, for example:
-@lisp
+@item a list of headers to include in the C program, for example:
+
+@example
 ("sys/types.h" "sys/socket.h" "sys/stat.h" "unistd.h" "sys/un.h"
  "netinet/in.h" "netinet/in_systm.h" "netinet/ip.h" "net/if.h"
- "netdb.h" "errno.h" "netinet/tcp.h" "fcntl.h" "signal.h" )
-@end lisp
+ "netdb.h" "errno.h" "netinet/tcp.h" "fcntl.h" "signal.h")
+@end example
 
-@item
-A list of sb-grovel clauses describing the things you want to grovel
-from the C compiler, for example:
-@lisp
+@item A list of sb-grovel clauses describing the things you want to
+  grovel from the C compiler, for example:
+
+@example
 ((:integer af-local
            #+(or sunos solaris) "AF_UNIX"
            #-(or sunos solaris) "AF_LOCAL"
@@ -89,149 +95,151 @@ from the C compiler, for example:
                    (integer dev "dev_t" "st_dev")
                    (integer atime "time_t" "st_atime")))
  (:function getpid ("getpid" int )))
-@end lisp
+@end example
 @end itemize
 
-There are two types of things that sb-grovel can sensibly extract from
-the C compiler: constant integers and structure layouts. It is also
-possible to define foreign functions in the constants.lisp file, but
-these definitions don't use any information from the C program; they
-expand directly to @code{sb-alien:define-alien-routine}
-(@pxref{The define-alien-routine Macro}) forms.
+There are two types of things that sb-grovel can sensibly extract
+from the C compiler: constant integers and structure layouts. It is
+also possible to define foreign functions in the constants.lisp
+file, but these definitions don't use any information from the C
+program; they expand directly to @code{sb-alien:define-alien-routine}
+forms.
 
 Here's how to use the grovel clauses:
 
 @itemize
-@item
-@code{:integer} - constant expressions in C. Used in this form:
-@lisp
+@item @code{:integer}: constant expressions in C. Used in this form:
+
+@example
  (:integer lisp-variable-name "C expression" &optional doc export)
-@end lisp
+@end example
 
-@code{"C expression"} will be typically be the name of a constant. But
-other forms are possible.
+@code{"C expression"} will be typically be the name of a constant,
+but other forms are possible.
 
-@item
-@code{:enum}
-@lisp
+@item @code{:enum}:
+
+@example
  (:enum lisp-type-name ((lisp-enumerated-name c-enumerated-name) ...)))
-@end lisp
+@end example
 
-An @code{sb-alien:enum} type with name @code{lisp-type-name} will be defined.
-The symbols are the @code{lisp-enumerated-name}s, and the values
-are grovelled from the @code{c-enumerated-name}s.
+An @code{sb-alien:enum} type with name @code{lisp-type-name} will be
+defined. The symbols are the @code{lisp-enumerated-name}s, and the
+values are grovelled from the @code{c-enumerated-name}s.
 
-@item
-@code{:structure} - alien structure definitions look like this:
-@lisp
+@item @code{:structure}: alien structure definitions look like this:
+
+@example
  (:structure lisp-struct-name ("struct c_structure"
                                (type-designator lisp-element-name
                                 "c_element_type" "c_element_name"
                                 :distrust-length nil)
                                ; ...
                                ))
-@end lisp
+@end example
 
 @code{type-designator} is a reference to a type whose size (and type
-constraints) will be groveled for. sb-grovel accepts a form of type
-designator that doesn't quite conform to either lisp nor sb-alien's
-type specifiers. Here's a list of type designators that sb-grovel
-currently accepts:
+constraints) will be groveled for. sb-grovel accepts a form of
+type designator that doesn't quite conform to either lisp nor
+sb-alien's type specifiers. Here's a list of type designators
+that sb-grovel currently accepts:
+
 @itemize
-@item
-@code{integer} - a C integral type; sb-grovel will infer the exact
-type from size information extracted from the C program. All common C
-integer types can be grovelled for with this type designator, but it
-is not possible to grovel for bit fields yet.
+@item @code{integer}: a C integral type; sb-grovel will infer the exact
+      type from size information extracted from the C program. All
+      common C integer types can be grovelled for with this type
+      designator, but it is not possible to grovel for bit fields
+      yet.
 
-@item
-@code{(unsigned n)} - an unsigned integer variable that is @code{n}
-bytes long. No size information from the C program will be used.
-@item
-@code{(signed n)} - an signed integer variable that is @code{n} bytes
-long. No size information from the C program will be used.
+@item @code{(unsigned n)}: an unsigned integer variable that is @code{n} bytes
+      long. No size information from the C program will be used.
 
-@item
-@code{c-string} - an array of @code{char} in the structure. sb-grovel
-will use the array's length from the C program, unless you pass it the
-@code{:distrust-length} keyword argument with non-@code{nil} value
-(this might be required for structures such as solaris's @code{struct
-dirent}).
+@item @code{(signed n)}: an signed integer variable that is @code{n} bytes
+      long. No size information from the C program will be used.
 
-@item
-@code{c-string-pointer} - a pointer to a C string, corresponding to
-the @code{sb-alien:c-string} type (@pxref{Foreign Type Specifiers}).
-@item
-@code{(array alien-type)} - An array of the previously-declared alien
-type. The array's size will be determined from the output of the C
-program and the alien type's size.
-@item
-@code{(array alien-type n)} - An array of the previously-declared alien
-type. The array's size will be assumed as being @code{n}.
+@item @code{c-string}: an array of @code{char} in the structure. sb-grovel
+      will use the array's length from the C program, unless you
+      pass it the @code{:distrust-length} keyword argument with non-@code{nil}
+      value (this might be required for structures such as solaris's
+      @code{struct dirent}).
+
+@item @code{sb-grovel::c-string-pointer}: a pointer to a C string,
+      corresponding to the @code{sb-alien:c-string} type (see
+      @ref{foreign type specifiers}).
+
+@item @code{(array alien-type)}: an array of the previously-declared
+      @code{alien-type}. The array's size will be determined from the
+      output of the C program and the alien type's size.
+
+@item @code{(array alien-type n):} an array of the previously-declared
+      @code{alien-type}. The array's size will be assumed as being @code{n}.
+@end itemize
 @end itemize
 
-
-Note that @code{c-string} and @code{c-string-pointer} do not have the
-same meaning. If you declare that an element is of type
+Note that @code{c-string} and @code{sb-grovel::c-string-pointer} do not have
+the same meaning. If you declare that an element is of type
 @code{c-string}, it will be treated as if the string is a part of the
 structure, whereas if you declare that the element is of type
-@code{c-string-pointer}, a @emph{pointer to a string} will be the
+@code{sb-grovel::c-string-pointer}, a @emph{pointer to a string} will be the
 structure member.
 
-@item
-@code{:function} - alien function definitions are similar to
-@code{define-alien-routine} definitions, because they expand to such
-forms when the lisp program is loaded. @xref{Foreign Function Calls}.
+@itemize
+@item @code{:function}: alien function definitions are similar to
+  @code{define-alien-routine} definitions, because they expand to such
+  forms when the lisp program is loaded. See
+  @ref{foreign function calls}.
 
-@lisp
-(:function lisp-function-name ("alien_function_name" alien-return-type
-                                                     (argument alien-type)
-                                                     (argument2 alien-type)))
-@end lisp
+@example
+(:function lisp-function-name
+           ("alien_function_name" alien-return-type
+                                  (argument alien-type)
+                                  (argument2 alien-type)))
+@end example
 @end itemize
 
-
+@node sb grovel structures
 @subsection Programming with sb-grovel's structure types
 
 Let us assume that you have a grovelled structure definition:
-@lisp
- (:structure mystruct ("struct my_structure"
-                       (integer myint "int" "st_int")
-                       (c-string mystring "char[]" "st_str")))
-@end lisp
+
+@example
+(:structure mystruct ("struct my_structure"
+                      (integer myint "int" "st_int")
+                      (c-string mystring "char[]" "st_str")))
+@end example
 
 What can you do with it? Here's a short interface document:
 
 @itemize
-@item
-Creating and destroying objects:
+@item Creating and destroying objects:
+
 @itemize
-@item
-Function @code{(allocate-mystruct)} - allocates an object of type @code{mystruct}and
-returns a system area pointer to it.
-@item
-Macro @code{(with-mystruct var ((member init) [...]) &body body)} -
-allocates an object of type @code{mystruct} that is valid in
-@var{body}. If @var{body} terminates or control unwinds out of
-@var{body}, the object pointed to by @var{var} will be deallocated.
+@item Function @code{(allocate-mystruct)} allocates an object of type
+      @code{mystruct} and returns a system area pointer to it.
+
+@item Macro @code{(with-mystruct var ((member init) [...]) &body body)}
+      allocates an object of type @code{mystruct} that is valid in
+      @code{body}. If @code{body} terminates or performs an non-local exit,
+      the object pointed to by @code{var} will be deallocated.
 @end itemize
 
-@item
-Accessing structure members:
+@item Accessing structure members:
+
 @itemize
-@item
-@code{(mystruct-myint var)} and @code{(mystruct-mystring var)} return
-the value of the respective fields in @code{mystruct}.
-@item
-@code{(setf (mystruct-myint var) new-val)} and
-@code{(setf (mystruct-mystring var) new-val)} sets the value of the respective
-structure member to the value of @var{new-val}. Notice that in
-@code{(setf (mystruct-mystring var) new-val)}'s case, new-val is a lisp
-string.
+@item @code{(mystruct-myint var)} and @code{(mystruct-mystring var)} return
+      the value of the respective fields in @code{mystruct}.
+
+@item @code{(setf (mystruct-myint var) new-val)} and
+      @code{(setf (mystruct-mystring var) new-val)} sets the value of the
+      respective structure member to the value of @code{new-val}. Notice
+      that in @code{(setf (mystruct-mystring var) new-val)}'s case,
+      @code{new-val} is a lisp string.
 @end itemize
 @end itemize
 
-@subsubsection Traps and Pitfalls
+@node sb grovel traps
+@subsection Traps and Pitfalls
+
 Basically, you can treat functions and data structure definitions that
 sb-grovel spits out as if they were alien routines and types. This has
 a few implications that might not be immediately obvious (especially
@@ -239,13 +247,12 @@ if you have programmed in a previous version of sb-grovel that didn't
 use alien types):
 
 @itemize
-@item
-You must take care of grovel-allocated structures yourself. They are
-alien types, so the garbage collector will not collect them when you
-drop the last reference.
+@item You must take care of grovel-allocated structures yourself. They
+  are alien types, so the garbage collector will not collect them
+  when you drop the last reference.
 
-@item
-If you use the @code{with-mystruct} macro, be sure that no references
-to the variable thus allocated leaks out. It will be deallocated when
-the block exits.
+@item If you use the @code{with-mystruct} macro, be sure that no references
+  to the variable thus allocated leaks out. It will be deallocated
+  when the block exits.
 @end itemize
+

contrib/sb-introspect/introspect.lisp

@@ -17,11 +17,9 @@
 ;;; own project, there will be much wailing and gnashing of teeth.
 ;;; Your teeth.  If need be, we'll kick them for you.  This is a
 ;;; contrib, we're allowed to look in internals.  You're an
-;;; application programmer, and are not.
+;;; application programmer and are not.
 
 ;;; TODO
-;;; 1) structs don't have within-file location info.  problem for the
-;;;   structure itself, accessors, the copier and the predicate
 ;;; 3) error handling.  Signal random errors, or handle and resignal 'our'
 ;;;   error, or return NIL?
 ;;; 4) FIXMEs
@@ -69,7 +67,7 @@
 ;;;
 (deftype debug-info ()
   "Structure containing all the debug information related to a function.
-Function objects reference debug-infos which in turn reference
+Function objects reference debug-infos, which in turn reference
 debug-sources and so on."
   'sb-c::compiled-debug-info)
 
@@ -94,8 +92,10 @@ include the pathname of the file and the position of the definition."
   (sb-c::debug-info-source debug-info))
 
 (defun valid-function-name-p (name)
-  "True if NAME denotes a valid function name, ie. one that can be passed to
-FBOUNDP."
+  "See if NAME is a valid function name. In addition to the ANSI
+definition of function name, which is symbols plus lists like (SETF
+SYMBOL), SBCL allows (SB-EXT:CAS SYMBOL) and various internal
+constructs."
   (and (sb-int:valid-function-name-p name) t))
 
 ;;;; Utilities for code
@@ -110,9 +110,9 @@ FBOUNDP."
 (declaim (inline map-allocated-code-components))
 (defun map-allocated-code-components (spaces fn)
   "Call FN for each allocated code component in one of SPACES.  FN
-receives the object and its size as arguments.  SPACES should be a
-list of the symbols :dynamic, :static, :read-only, or :immobile on
-#+immobile-space"
+receives the object and its size as arguments. SPACES should be a list
+of the symbols :DYNAMIC, :STATIC, :READ-ONLY, or :IMMOBILE on
+#+IMMOBILE-SPACE. The shorthand (:ALL) is also accepted."
   (apply #'sb-vm:map-allocated-objects
      (lambda (obj header size)
        (when (= sb-vm:code-header-widetag header)
@@ -121,7 +121,7 @@ list of the symbols :dynamic, :static, :read-only, or :immobile on
 
 (declaim (inline map-caller-code-components))
 (defun map-caller-code-components (function spaces fn)
-  "Call FN for each code component with a fdefn for FUNCTION in its
+  "Call FN for each code component with a FDEFN for FUNCTION in its
 constant pool."
   (let ((function (coerce function 'function)))
     (map-allocated-code-components
@@ -148,32 +148,112 @@ constant pool."
 
 ;;;; Finding definitions
 
-(defstruct definition-source
-  ;; Pathname of the source file that the definition was compiled from.
-  ;; This is null if the definition was not compiled from a file.
-  (pathname nil :type (or null pathname))
-  ;; Source-path of the definition within the file.
-  ;; This may be incomplete depending on the debug level at which the
-  ;; source was compiled.
-  (form-path '() :type list)
-  ;; Depth first number of the form.
-  ;; FORM-PATH above usually contains just the top-level form number,
-  ;; ideally the proper form path could be dervied from the
-  ;; form-number and the tlf-number, but it's a bit complicated and
-  ;; Slime already knows how to deal with form numbers, so delegate
-  ;; that job to Slime.
-  (form-number nil :type (or null unsigned-byte))
-  ;; Character offset of the top-level-form containing the definition.
-  ;; This corresponds to the first element of form-path.
-  (character-offset nil :type (or null unsigned-byte))
-  ;; File-write-date of the source file when compiled.
-  ;; Null if not compiled from a file.
-  (file-write-date nil :type (or null unsigned-byte))
-  ;; plist from WITH-COMPILATION-UNIT
-  (plist nil)
+(defmacro defstruct* (name-and-options &rest slot-descriptions)
+  "Like DEFSTRUCT, but support :DOCUMENTATION among slot options.
+The documentation is attached to the slot's STRUCTURE-ACCESSOR.
+Example:
+
+    (defstruct* my-struct
+      (my-slot nil :documentation \"docstring\"))
+
+In addition to the normal DEFSTRUCT processing, the above also does
+the moral equivalent of
+
+    (setf (documentation 'my-struct-my-slot 'function) \"docstring\")"
+  (destructuring-bind (name &rest options)
+      (sb-c::ensure-list name-and-options)
+    (let* ((conc-name-option (find :conc-name options :key (lambda (x)
+                                                             (if (consp x)
+                                                                 (car x)
+                                                                 x))))
+           (conc-name (cond ((not conc-name-option)
+                             (format nil "~A-" name))
+                            ((or (atom conc-name-option)
+                                 (null (cdr conc-name-option))
+                                 (null (second conc-name-option)))
+                             "")
+                            (t
+                             (string (second conc-name-option)))))
+           (not-found (gensym))
+           (set-doc-forms ())
+           (new-sds
+             (loop
+               for sd in slot-descriptions
+               collect (let ((documentation
+                               (and (listp sd)
+                                    (getf (cddr sd) :documentation not-found))))
+                         (cond
+                           ((or (not (listp sd))
+                                (eq documentation not-found))
+                            sd)
+                           (t
+                            (let ((accessor-name (intern
+                                                  (format nil "~A~A" conc-name
+                                                          (first sd)))))
+                              (push `(setf (documentation ',accessor-name
+                                                          'function)
+                                           ,documentation)
+                                    set-doc-forms))
+                            (let ((sd (copy-seq sd)))
+                              (remf (cddr sd) :documentation)
+                              sd)))))))
+      `(progn
+         (defstruct ,name-and-options
+           ,@new-sds)
+         ,@set-doc-forms))))
+
+;;; FIXME: Rename this
+(defstruct* definition-source
+  "This structure identifies a sexp in a compiled file.
+Despite the name, the source location may not correspond to a
+definition but to e.g. a function call (see WHO-CALLS)."
+  (pathname
+   nil :type (or null pathname)
+   :documentation "Pathname of the source file.
+This is NIL if the source location is not in a compiled file.")
+  (form-path
+   '() :type list
+   :documentation "List of indices that identify the sexp in the
+file given by DEFINITION-SOURCE-PATHNAME. The first element in the
+list is the index of the top-level form that contains the sexp. If the
+file was compiled at a high enough debug level, then the rest of the
+elements recursively index into the list structure of the top-level
+form.
+
+Thus, the form path is somewhat stable regarding edits in the file,
+but it gets invalidated by, for example, inserting a new top-level
+form before the sexp in question.")
+  (form-number
+   nil :type (or null unsigned-byte)
+   :documentation "Depth-first index of the sexp within the top-level
+form identified by the first element of DEFINITION-SOURCE-FORM-PATH.
+That is, this is the index of the sexp in the list of subexpressions
+of the top-level form ordered according to depth-first traversal. 0
+corresponds to the top-level form itself.
+
+When combined with the index of the top-level form (given by the first
+element of DEFINITION-SOURCE-FORM-PATH), the form number allows
+reconstruction of the rest of the form path, which may be missing.
+This requires parsing the source file. Currently, this job is
+delegated to e.g. SLIME.")
+  (character-offset
+   nil :type (or null unsigned-byte)
+   :documentation "Character offset of the top-level form containing
+the sexp.")
+  (file-write-date
+   nil :type (or null unsigned-byte)
+   :documentation "FILE-WRITE-DATE of DEFINITION-SOURCE-PATHNAME at
+the time of compilation. NIL if not compiled from a file.")
+  (plist
+   nil
+   :documentation "The `SOURCE-PLIST` from WITH-COMPILATION-UNIT in effect
+when the file was compiled.")
   ;; Any extra metadata that the caller might be interested in. For
-  ;; example the specializers of the method whose definition-source this
-  ;; is.
+  ;; example, DEFINITION-SOURCE of a method contains the specializers
+  ;; of the method to help disambiguate it.
+  ;;
+  ;; FIXME: This is currently unexported, but it is also necessary to
+  ;; disambiguate methods. See e.g. WHO-SPECIALIZES-DIRECTLY.
   (description nil :type list))
 
 (defun vops-translating-fun (name)
@@ -205,48 +285,63 @@ constant pool."
                     source))))
 
 (defun find-definition-sources-by-name (name type)
-  "Returns a list of DEFINITION-SOURCEs for the objects of type TYPE
-defined with name NAME. NAME may be a symbol or a extended function
-name. Type can currently be one of the following:
+  "Returns a list of DEFINITION-SOURCEs for definitions of NAME with
+the given definition TYPE. A DEFINITION-SOURCE object is always
+returned for definitions that exist, but the source location (e.g.
+DEFINITION-SOURCE-PATHNAME) may be missing. TYPE can currently be one
+of the following.
 
-   (Public)
-   :CLASS
-   :COMPILER-MACRO
-   :CONDITION
-   :CONSTANT
-   :FUNCTION
-   :GENERIC-FUNCTION
-   :MACRO
-   :METHOD
-   :METHOD-COMBINATION
-   :PACKAGE
-   :SETF-EXPANDER
-   :STRUCTURE
-   :SYMBOL-MACRO
-   :TYPE
-   :ALIEN-TYPE
-   :VARIABLE
-   :DECLARATION
+- Public definition types:
 
-   (Internal)
-   :OPTIMIZER
-   :SOURCE-TRANSFORM
-   :TRANSFORM
-   :VOP
-   :IR1-CONVERT
+    :CLASS
+    :COMPILER-MACRO
+    :CONDITION
+    :CONSTANT
+    :FUNCTION
+    :GENERIC-FUNCTION
+    :MACRO
+    :METHOD
+    :METHOD-COMBINATION
+    :PACKAGE
+    :SETF-EXPANDER
+    :STRUCTURE
+    :SYMBOL-MACRO
+    :TYPE
+    :ALIEN-TYPE
+    :ALIEN-CALLBACK
+    :VARIABLE
+    :DECLARATION
 
-If an unsupported TYPE is requested, the function will return NIL.
-"
+- Internal definition types:
+
+    :OPTIMIZER
+    :SOURCE-TRANSFORM
+    :TRANSFORM
+    :VOP
+    :IR1-CONVERT
+
+Definition types are disjoint. For example, :TYPE refers to DEFTYPEs
+but not CLASSes or SB-ALIEN:DEFINE-ALIEN-TYPE, as those are of
+definition type :CLASS and :ALIEN-TYPE, respectively. :FUNCTION does
+not include :GENERIC-FUNCTION, :CLASS does not include :STRUCTURE,
+etc. :VARIABLE refers to non-constant dynamic variables (e.g. those
+defined with DEFVAR, DEFPARAMETER, SB-EXT:DEFGLOBAL or
+SB-ALIEN:DEFINE-ALIEN-VARIABLE but not with DEFCONSTANT).
+
+Valid NAMEs are generally SYMBOLs with the following exceptions:
+
+- For :COMPILER-MACRO, :FUNCTION, :GENERIC-FUNCTION and :METHOD,
+  anything that's VALID-FUNCTION-NAME-P is valid.
+
+- For :PACKAGE, string designators are valid.
+
+If an unsupported TYPE is requested or NAME is invalid, this function
+returns NIL."
   (flet ((get-class (name)
            (and (symbolp name)
                 (find-class name nil)))
          (real-fdefinition (name)
-           ;; for getting the real function object, even if the
-           ;; function is being profiled
-           (let ((profile-info (gethash name sb-profile::*profiled-fun-name->info*)))
-             (if profile-info
-                 (sb-profile::profile-info-encapsulated-fun profile-info)
-                 (fdefinition name)))))
+           (sb-ext:unencapsulated-function name)))
     (ensure-list
      (case type
        ((:variable)
@@ -267,25 +362,28 @@ If an unsupported TYPE is requested, the function will return NIL.
                    (macro-function name))
           (find-definition-source (macro-function name))))
        ((:compiler-macro)
-        (when (compiler-macro-function name)
+        (when (and (valid-function-name-p name)
+                   (compiler-macro-function name))
           (find-definition-source (compiler-macro-function name))))
        (:ir1-convert
         (let ((converter (info :function :ir1-convert name)))
           (and converter
-           (find-definition-source converter))))
+               (find-definition-source converter))))
        ((:function :generic-function)
-        (if (fboundp name)
-            (when (and (or (consp name)
-                           (and
-                            (not (macro-function name))
-                            (not (special-operator-p name)))))
-              (let ((fun (real-fdefinition name)))
-                (when (eq (not (typep fun 'generic-function))
-                          (not (eq type :generic-function)))
-                  (find-definition-source fun))))
-            (let ((dd (info :function :source-transform name)))
-              (when (typep dd '(cons defstruct-description))
-                (find-definition-sources-by-name (dd-name (car dd)) :structure)))))
+        (when (valid-function-name-p name)
+          (if (fboundp name)
+              (when (and (or (consp name)
+                             (and
+                              (not (macro-function name))
+                              (not (special-operator-p name)))))
+                (let ((fun (real-fdefinition name)))
+                  (when (eq (not (typep fun 'generic-function))
+                            (not (eq type :generic-function)))
+                    (find-definition-source fun))))
+              (let ((dd (info :function :source-transform name)))
+                (when (typep dd '(cons defstruct-description))
+                  (find-definition-sources-by-name (dd-name (car dd))
+                                                   :structure))))))
        ((:type)
         ;; Source locations for types are saved separately when the expander
         ;; is a closure without a good source-location.
@@ -296,7 +394,7 @@ If an unsupported TYPE is requested, the function will return NIL.
                 (when (functionp expander-fun)
                   (find-definition-source expander-fun))))))
        ((:method)
-        (when (fboundp name)
+        (when (and (valid-function-name-p name) (fboundp name))
           (let ((fun (real-fdefinition name)))
             (when (typep fun 'generic-function)
               (loop for method in (sb-mop::generic-function-methods
@@ -334,10 +432,9 @@ If an unsupported TYPE is requested, the function will return NIL.
             (translate-source-location
              (sb-pcl::method-combination-info-source-location info)))))
        ((:package)
-        (when (symbolp name)
-          (let ((package (find-package name)))
-            (when package
-              (find-definition-source package)))))
+        (let ((package (ignore-errors (find-package name))))
+          (when package
+            (find-definition-source package))))
        ;; TRANSFORM and OPTIMIZER handling from swank-sbcl
        ((:transform)
         (let ((fun-info (info :function :info name)))
@@ -379,23 +476,30 @@ If an unsupported TYPE is requested, the function will return NIL.
                             (sb-c::fun-info-flushable . sb-c::flushable))))
               (loop for (reader . name) in otypes
                     for fn = (funcall reader fun-info)
-                    when (functionp fn) collect
-                    (let ((source (find-definition-source fn)))
-                      (setf (definition-source-description source)
-                            (list name))
-                      source))))))
+                    when (functionp fn)
+                      collect (let ((source (find-definition-source fn)))
+                                (setf (definition-source-description source)
+                                      (list name))
+                                source))))))
        (:vop
         (find-vop-source name))
        (:alien-type
         (let ((loc (info :source-location type name)))
           (and loc
                (translate-source-location loc))))
+       (:alien-callback
+        (let ((cb (gethash name sb-alien::*alien-callables*)))
+          (when cb
+            (let ((index (sb-alien::alien-callback-index cb)))
+              (when (and index
+                         (array-in-bounds-p sb-alien::*alien-callback-functions* index))
+                (find-definition-source (aref sb-alien::*alien-callback-functions* index)))))))
        ((:source-transform)
         (let* ((transform-fun
-                (or (info :function :source-transform name)
-                    (and (typep name '(cons (eql setf) (cons symbol null)))
-                         (info :function :source-transform
-                                      (second name)))))
+                 (or (info :function :source-transform name)
+                     (and (typep name '(cons (eql setf) (cons symbol null)))
+                          (info :function :source-transform
+                                (second name)))))
                ;; A cons for the :source-transform is essentially the same
                ;; info that was formerly in :structure-accessor.
                (accessor (and (consp transform-fun) (cdr transform-fun))))
@@ -409,16 +513,28 @@ If an unsupported TYPE is requested, the function will return NIL.
         (let ((locations (info :source-location :declaration name)))
           (loop for (kind loc) on locations by #'cddr
                 when loc
-                collect (let ((loc (translate-source-location loc)))
-                          (setf (definition-source-description loc)
-                                ;; Copy list to ensure that user code
-                                ;; cannot mutate the original.
-                                (copy-list (ensure-list kind)))
-                          loc))))
+                  collect (let ((loc (translate-source-location loc)))
+                            (setf (definition-source-description loc)
+                                  ;; Copy list to ensure that user code
+                                  ;; cannot mutate the original.
+                                  (copy-list (ensure-list kind)))
+                            loc))))
        (t
         nil)))))
 
 (defun find-definition-source (object)
+  "Return the DEFINITION-SOURCE corresponding to the definition of OBJECT
+or NIL if there is no corresponding definition. OBJECT must be a
+PACKAGE, FUNCTION, METHOD, METHOD-COMBINATION, SB-MOP:SLOT-DEFINITION,
+STANDARD-OBJECT, STRUCTURE-OBJECT, CONDITION, CLASS, STRUCTURE-CLASS,
+or a subclass of CONDITION. An error is signalled for other types.
+
+A DEFINITION-SOURCE object is always returned for definitions that
+exist, but the source location (e.g. DEFINITION-SOURCE-PATHNAME) may
+be missing.
+
+For definitions that do not define an object (e.g. DEFVAR), use
+FIND-DEFINITION-SOURCES-BY-NAME."
   (typecase object
     ((or sb-pcl::condition-class sb-pcl::structure-class)
      (let ((classoid (sb-pcl::class-classoid object)))
@@ -481,8 +597,9 @@ If an unsupported TYPE is requested, the function will return NIL.
      (when (stringp (sb-c::debug-source-namestring debug-source))
        (parse-namestring (sb-c::debug-source-namestring debug-source)))
      :character-offset
-     (if tlf
-         (elt (sb-c::debug-source-start-positions debug-source) tlf))
+     (let ((start-positions (sb-c::debug-source-start-positions debug-source)))
+       (when (and tlf (< tlf (length start-positions)))
+         (elt start-positions tlf)))
      :form-path (if tlf (list tlf))
      :form-number (handler-case (sb-di::code-location-form-number
                                  (sb-di::debug-fun-start-location debug-fun))
@@ -514,13 +631,13 @@ If an unsupported TYPE is requested, the function will return NIL.
   (function-lambda-list function))
 
 (defun function-lambda-list (function)
-  "Return the lambda list for the extended function designator FUNCTION.
-Works for special-operators, macros, simple functions, interpreted functions,
-and generic functions. Signals an error if FUNCTION is not a valid extended
-function designator.
+  "Return the lambda list of FUNCTION.
+FUNCTION must be a function object or a function name in the sense of
+VALID-FUNCTION-NAME-P. Works for special operators, macros, simple
+functions, interpreted functions, and generic functions.
 
-If the function does not have a lambda list (compiled with debug 0),
-then two values are returned: (values nil t)"
+The second return value indicates whether the lambda list could not be
+determined (e.g. because the function was compiled with DEBUG 0)."
   (cond ((and (symbolp function) (special-operator-p function))
          (function-lambda-list (info :function :ir1-convert function)))
         ((valid-function-name-p function)
@@ -530,26 +647,34 @@ then two values are returned: (values nil t)"
         ((typep function 'generic-function)
          (sb-pcl::generic-function-pretty-arglist function))
         (t
-         (let ((raw-result (%fun-lambda-list function)))
+         (let ((raw-result (%fun-lambda-list (sb-ext:unencapsulated-function function))))
            (if (eq raw-result :unknown)
                (values nil t)
                (values raw-result nil))))))
 
-(defun deftype-lambda-list (typespec-operator)
-  "Returns the lambda list of TYPESPEC-OPERATOR as first return
-value, and a flag whether the arglist could be found as second
-value."
-  (check-type typespec-operator symbol)
-  ;; Don't return a lambda-list for combinators AND,OR,NOT.
-  (let* ((f (and (info :type :kind typespec-operator)
-                 (info :type :expander typespec-operator)))
+(defun deftype-lambda-list (type-specifier-name)
+  "Returns the lambda list of TYPE-SPECIFIER-NAME as the first return
+value, and a flag whether the arglist could be found as the second
+value.
+
+TYPE-SPECIFIER-NAME must be a symbol. This function can find the
+lambda list of derived type specifiers (e.g. those defined with
+DEFTYPE) and classes with compound type specifier syntaxes (e.g. the
+class FLOAT). It returns NIL, NIL for other type specifiers (e.g. AND,
+OR, NOT) and types (e.g. LIST)."
+  (check-type type-specifier-name symbol)
+  (let* ((f (and (info :type :kind type-specifier-name)
+                 (info :type :expander type-specifier-name)))
          (f (if (listp f) (car f) f)))
     (if (functionp f)
-        (values (%fun-lambda-list f) t)
+        (let ((lambda-list (%fun-lambda-list f)))
+          (if (eq lambda-list :unknown)
+              (values nil nil)
+              (values lambda-list t)))
         (values nil nil))))
 
 (defun method-combination-lambda-list (method-combination)
-  "Return the lambda-list of METHOD-COMBINATION designator.
+  "Return the lambda list of the METHOD-COMBINATION designator.
 METHOD-COMBINATION can be a method combination object,
 or a method combination name."
   (let* ((name (etypecase method-combination
@@ -561,7 +686,7 @@ or a method combination name."
     (sb-pcl::method-combination-info-lambda-list info)))
 
 (defun function-type (function-designator)
-  "Returns the ftype of FUNCTION-DESIGNATOR, or NIL."
+  "Returns the ftype of FUNCTION-DESIGNATOR or NIL."
   (etypecase function-designator
     ((or symbol cons)
      ;; XXX: why require FBOUNDP? Would it be wrong to always report the proclaimed type?
@@ -629,7 +754,13 @@ or a method combination name."
         callees)))
 
 (defun find-function-callers (function &optional (spaces '(:all)))
-  "Return functions which call FUNCTION, by searching SPACES for code objects"
+  ;; FIXME: :IMMOBILE-SPACE is an internal feature
+  "List functions that call FUNCTION by searching SPACES for code objects.
+This can make previously garbage objects live.
+
+SPACES should be a list of the symbols :DYNAMIC, :STATIC, :READ-ONLY,
+or :IMMOBILE on `#+IMMOBILE-SPACE`. The shorthand (:ALL) is also
+accepted."
   (let ((referrers '()))
     (map-caller-code-components
      function
@@ -798,47 +929,45 @@ or a method combination name."
       result)))))
 
 (defun who-calls (function-name)
-  "Use the xref facility to search for source locations where the
-global function named FUNCTION-NAME is called. Returns a list of
-function name, definition-source pairs."
+  "Find the source locations where the global function FUNCTION-NAME is
+called, and return them as an alist of function or macro name,
+DEFINITION-SOURCE pairs."
   (collect-xref :calls function-name))
 
 (defun who-binds (symbol)
-  "Use the xref facility to search for source locations where the
-special variable SYMBOL is rebound. Returns a list of function name,
-definition-source pairs."
+  "Find the source locations where the special variable SYMBOL is bound,
+and return them as an alist of function or macro name,
+DEFINITION-SOURCE pairs."
   (collect-xref :binds symbol))
 
 (defun who-references (symbol)
-  "Use the xref facility to search for source locations where the
-special variable or constant SYMBOL is read. Returns a list of function
-name, definition-source pairs."
+  "Find the source locations where the special variable SYMBOL is read,
+and return them as an alist of function or macro name,
+DEFINITION-SOURCE pairs."
   (collect-xref :references symbol))
 
 (defun who-sets (symbol)
-  "Use the xref facility to search for source locations where the
-special variable SYMBOL is written to. Returns a list of function name,
-definition-source pairs."
+  "Find the source locations where the special variable SYMBOL is set,
+and return them as an alist of function or macro name,
+DEFINITION-SOURCE pairs."
   (collect-xref :sets symbol))
 
 (defun who-macroexpands (macro-name)
-  "Use the xref facility to search for source locations where the
-macro MACRO-NAME is expanded. Returns a list of function name,
-definition-source pairs."
+  "Find the source locations where the macro MACRO-NAME is expanded, and
+return them as an alist of function or macro name, DEFINITION-SOURCE
+pairs."
   (collect-xref :macroexpands macro-name))
 
 (defun who-specializes-directly (class-designator)
-  "Search for source locations of methods directly specializing on
-CLASS-DESIGNATOR. Returns an alist of method name, definition-source
-pairs.
+  "Find the source locations of methods directly specializing on
+CLASS-DESIGNATOR, and return them as an alist of generic function
+name, DEFINITION-SOURCE pairs.
 
 A method matches the criterion either if it specializes on the same
-class as CLASS-DESIGNATOR designates (this includes CLASS-EQ
-specializers), or if it eql-specializes on an instance of the
-designated class.
+class as CLASS-DESIGNATOR designates, or if it eql-specializes on an
+instance of the designated class.
 
-Experimental.
-"
+Experimental."
   (let ((class (canonicalize-class-designator class-designator)))
     (unless class
       (return-from who-specializes-directly nil))
@@ -861,17 +990,17 @@ Experimental.
                 result))))
 
 (defun who-specializes-generally (class-designator)
-  "Search for source locations of methods specializing on
-CLASS-DESIGNATOR, or a subclass of it. Returns an alist of method
-name, definition-source pairs.
+  "Find the source locations of methods specializing on
+CLASS-DESIGNATOR or a subclass of it, and return them as an alist of
+generic function name, DEFINITION-SOURCE pairs.
+DEFINITION-SOURCE-DESCRIPTION identifies the method.
 
 A method matches the criterion either if it specializes on the
 designated class itself or a subclass of it (this includes CLASS-EQ
 specializers), or if it eql-specializes on an instance of the
 designated class or a subclass of it.
 
-Experimental.
-"
+Experimental."
   (let ((class (canonicalize-class-designator class-designator)))
     (unless class
       (return-from who-specializes-generally nil))
@@ -923,12 +1052,12 @@ Experimental.
           1)))
 
 (defun allocation-information (object)
-  "Returns information about the allocation of OBJECT. Primary return value
-indicates the general type of allocation: :IMMEDIATE, :HEAP, :STACK,
-or :FOREIGN.
+  "Returns information about the allocation of OBJECT. The primary return
+value indicates the general type of allocation: :IMMEDIATE, :HEAP,
+:STACK, or :FOREIGN.
 
-Possible secondary return value provides additional information about the
-allocation.
+Non-NIL secondary return values provide additional information about
+the allocation.
 
 For :HEAP objects the secondary value is a plist:
 
@@ -936,7 +1065,7 @@ For :HEAP objects the secondary value is a plist:
     Indicates the heap segment the object is allocated in.
 
   :GENERATION
-    Is the current generation of the object: 0 for nursery, 6 for pseudo-static
+    The current generation of the object: 0 for nursery, 6 for pseudo-static
     generation loaded from core. (GENCGC and :SPACE :DYNAMIC only.)
 
   :LARGE
@@ -945,13 +1074,13 @@ For :HEAP objects the secondary value is a plist:
 
   :BOXED
     Indicates that the object is allocated in a boxed region. Unboxed
-    allocation is used for eg. specialized arrays after they have survived one
+    allocation is used for e.g. specialized arrays after they have survived one
     collection. (GENCGC and :SPACE :DYNAMIC only.)
 
   :PINNED
     Indicates that the page(s) on which the object resides are kept live due
     to conservative references. Note that object may reside on a pinned page
-    even if :PINNED in NIL if the GC has not had the need to mark the the page
+    even if :PINNED is NIL if the GC has not had the need to mark the page
     as pinned. (GENCGC and :SPACE :DYNAMIC only.)
 
   :WRITE-PROTECTED
@@ -960,11 +1089,11 @@ For :HEAP objects the secondary value is a plist:
     the last GC of its generation. (GENCGC and :SPACE :DYNAMIC only.)
 
   :PAGE
-    The index of the page the object resides on. (GENGC and :SPACE :DYNAMIC
+    The index of the page the object resides on. (GENCGC and :SPACE :DYNAMIC
     only.)
 
-For :STACK objects secondary value is the thread on whose stack the object is
-allocated.
+For :STACK objects, the secondary value is the thread on whose stack
+the object is allocated.
 
 Expected use-cases include introspection to gain insight into allocation and
 GC behaviour and restricting memoization to heap-allocated arguments.
@@ -1022,17 +1151,17 @@ Experimental: interface subject to change."
 Returns OBJECT.
 
 If SIMPLE is true (default is NIL), elides those pointers that are not
-notionally part of certain built-in objects, but backpointers to a
-conceptual parent: eg. elides the pointer from a SYMBOL to the
+notionally part of certain built-in objects but backpointers to a
+conceptual parent: e.g. elides the pointer from a SYMBOL to the
 corresponding PACKAGE.
 
 If EXT is true (default is T), includes some pointers that are not
-actually contained in the object, but found in certain well-known
+actually contained in the object but found in certain well-known
 indirect containers: FDEFINITIONs, EQL specializers, classes, and
 thread-local symbol values in other threads fall into this category.
 
-NOTE: calling MAP-ROOT with a THREAD does not currently map over
-conservative roots from the thread registers and interrupt contexts.
+> _Note_: calling MAP-ROOT with a THREAD does not currently map over
+> conservative roots from the thread registers and interrupt contexts.
 
 Experimental: interface subject to change."
   (when (typep object '(or bignum float sb-sys:system-area-pointer

contrib/sb-introspect/manual.lisp

@@ -0,0 +1,45 @@
+(in-package :sb-manual)
+
+(defsection @sb-introspect (:title "sb-introspect")
+  "The `SB-INTROSPECT` module is about finding definitions, as well
+  as querying their properties and relationships in the running image."
+  (@finding-definitions section)
+  (@sb-introspect-variables section)
+  (@sb-introspect-functions section)
+  (@sb-introspect-types section)
+  (@sb-introspect-allocation section))
+
+(defsection @finding-definitions (:title "Finding Definitions")
+  (sb-introspect:definition-source structure)
+  (sb-introspect:definition-source-pathname function)
+  (sb-introspect:definition-source-form-path function)
+  (sb-introspect:definition-source-form-number function)
+  (sb-introspect:definition-source-character-offset function)
+  (sb-introspect:definition-source-file-write-date function)
+  (sb-introspect:definition-source-plist function)
+  (sb-introspect:find-definition-source function)
+  (sb-introspect:find-definition-sources-by-name function))
+
+(defsection @sb-introspect-variables (:title "Special Variables")
+  (sb-introspect:who-binds function)
+  (sb-introspect:who-references function)
+  (sb-introspect:who-sets function))
+
+(defsection @sb-introspect-functions (:title "Functions")
+  (sb-introspect:function-lambda-list function)
+  (sb-introspect:function-type function)
+  (sb-introspect:method-combination-lambda-list function)
+  (sb-introspect:valid-function-name-p function)
+  (sb-introspect:find-function-callers function)
+  (sb-introspect:find-function-callees function)
+  (sb-introspect:who-calls function)
+  (sb-introspect:who-macroexpands function))
+
+(defsection @sb-introspect-types (:title "Types and Classes")
+  (sb-introspect:deftype-lambda-list function)
+  (sb-introspect:who-specializes-directly function)
+  (sb-introspect:who-specializes-generally function))
+
+(defsection @sb-introspect-allocation (:title "Allocation")
+  (sb-introspect:allocation-information function)
+  (sb-introspect:map-root function))

contrib/sb-introspect/sb-introspect.texinfo

@@ -0,0 +1,363 @@
+@c Generated by the sb-manual contrib. Do not edit.
+
+@node sb introspect
+@section sb-introspect
+
+@menu
+* Finding Definitions: finding definitions.
+* Special Variables: sb introspect variables.
+* Functions: sb introspect functions.
+* Types and Classes: sb introspect types.
+* Allocation: sb introspect allocation.
+@end menu
+
+The @code{sb-introspect} module is about finding definitions, as well
+as querying their properties and relationships in the running image.
+
+@node finding definitions
+@subsection Finding Definitions
+
+@anchor{Structure sb-introspect definition-source}
+@ttindex @sortas{definition-source sb-introspect} definition-source [sb-introspect]
+@deffn{Structure} sb-introspect:definition-source
+This structure identifies a sexp in a compiled file.
+Despite the name, the source location may not correspond to a
+definition but to e.g. a function call (see @code{who-calls}).
+@end deffn
+@anchor{Function sb-introspect definition-source-pathname}
+@ffindex @sortas{definition-source-pathname sb-introspect} definition-source-pathname [sb-introspect]
+@deffn{Function} sb-introspect:definition-source-pathname instance
+Pathname of the source file.
+This is @code{nil} if the source location is not in a compiled file.
+@end deffn
+@anchor{Function sb-introspect definition-source-form-path}
+@ffindex @sortas{definition-source-form-path sb-introspect} definition-source-form-path [sb-introspect]
+@deffn{Function} sb-introspect:definition-source-form-path instance
+List of indices that identify the sexp in the
+file given by @code{definition-source-pathname}. The first element in the
+list is the index of the top-level form that contains the sexp. If the
+file was compiled at a high enough debug level, then the rest of the
+elements recursively index into the list structure of the top-level
+form.
+
+Thus, the form path is somewhat stable regarding edits in the file,
+but it gets invalidated by, for example, inserting a new top-level
+form before the sexp in question.
+@end deffn
+@anchor{Function sb-introspect definition-source-form-number}
+@ffindex @sortas{definition-source-form-number sb-introspect} definition-source-form-number [sb-introspect]
+@deffn{Function} sb-introspect:definition-source-form-number instance
+Depth-first index of the sexp within the top-level
+form identified by the first element of @code{definition-source-form-path}.
+That is, this is the index of the sexp in the list of subexpressions
+of the top-level form ordered according to depth-first traversal. 0
+corresponds to the top-level form itself.
+
+When combined with the index of the top-level form (given by the first
+element of @code{definition-source-form-path}), the form number allows
+reconstruction of the rest of the form path, which may be missing.
+This requires parsing the source file. Currently, this job is
+delegated to e.g. SLIME.
+@end deffn
+@anchor{Function sb-introspect definition-source-character-offset}
+@ffindex @sortas{definition-source-character-offset sb-introspect} definition-source-character-offset [sb-introspect]
+@deffn{Function} sb-introspect:definition-source-character-offset instance
+Character offset of the top-level form containing
+the sexp.
+@end deffn
+@anchor{Function sb-introspect definition-source-file-write-date}
+@ffindex @sortas{definition-source-file-write-date sb-introspect} definition-source-file-write-date [sb-introspect]
+@deffn{Function} sb-introspect:definition-source-file-write-date instance
+@code{file-write-date} of @code{definition-source-pathname} at
+the time of compilation. @code{nil} if not compiled from a file.
+@end deffn
+@anchor{Function sb-introspect definition-source-plist}
+@ffindex @sortas{definition-source-plist sb-introspect} definition-source-plist [sb-introspect]
+@deffn{Function} sb-introspect:definition-source-plist instance
+The @code{source-plist} from @code{with-compilation-unit} in effect
+when the file was compiled.
+@end deffn
+@anchor{Function sb-introspect find-definition-source}
+@ffindex @sortas{find-definition-source sb-introspect} find-definition-source [sb-introspect]
+@deffn{Function} sb-introspect:find-definition-source object
+Return the @code{definition-source} corresponding to the definition of @code{object}
+or @code{nil} if there is no corresponding definition. @code{object} must be a
+@code{package}, @code{function}, @code{method}, @code{method-combination}, @code{sb-mop:slot-definition},
+@code{standard-object}, @code{structure-object}, @code{condition}, @code{class}, @code{structure-class},
+or a subclass of @code{condition}. An error is signalled for other types.
+
+A @code{definition-source} object is always returned for definitions that
+exist, but the source location (e.g. @code{definition-source-pathname}) may
+be missing.
+
+For definitions that do not define an object (e.g. @code{defvar}), use
+@code{find-definition-sources-by-name}.
+@end deffn
+@anchor{Function sb-introspect find-definition-sources-by-name}
+@ffindex @sortas{find-definition-sources-by-name sb-introspect} find-definition-sources-by-name [sb-introspect]
+@deffn{Function} sb-introspect:find-definition-sources-by-name name type
+Returns a list of @code{definition-source}s for definitions of @code{name} with
+the given definition @code{type}. A @code{definition-source} object is always
+returned for definitions that exist, but the source location (e.g.
+@code{definition-source-pathname}) may be missing. @code{type} can currently be one
+of the following.
+
+@itemize
+@item Public definition types:
+
+@code{:class}
+@code{:compiler-macro}
+@code{:condition}
+@code{:constant}
+@code{:function}
+@code{:generic-function}
+@code{:macro}
+@code{:method}
+@code{:method-combination}
+@code{:package}
+@code{:setf-expander}
+@code{:structure}
+@code{:symbol-macro}
+@code{:type}
+@code{:alien-type}
+@code{:alien-callback}
+@code{:variable}
+@code{:declaration}
+
+@item Internal definition types:
+
+@code{:optimizer}
+@code{:source-transform}
+@code{:transform}
+@code{:vop}
+@code{:ir1-convert}
+@end itemize
+
+Definition types are disjoint. For example, @code{:type} refers to @code{deftype}s
+but not @code{class}es or @code{sb-alien:define-alien-type}, as those are of
+definition type @code{:class} and @code{:alien-type}, respectively. @code{:function} does
+not include @code{:generic-function}, @code{:class} does not include @code{:structure},
+etc. @code{:variable} refers to non-constant dynamic variables (e.g. those
+defined with @code{defvar}, @code{defparameter}, @code{sb-ext:defglobal} or
+@code{sb-alien:define-alien-variable} but not with @code{defconstant}).
+
+Valid @code{name}s are generally @code{symbol}s with the following exceptions:
+
+@itemize
+@item For @code{:compiler-macro}, @code{:function}, @code{:generic-function} and @code{:method},
+  anything that's @code{valid-function-name-p} is valid.
+
+@item For @code{:package}, string designators are valid.
+@end itemize
+
+If an unsupported @code{type} is requested or @code{name} is invalid, this function
+returns @code{nil}.
+@end deffn
+@node sb introspect variables
+@subsection Special Variables
+
+@anchor{Function sb-introspect who-binds}
+@ffindex @sortas{who-binds sb-introspect} who-binds [sb-introspect]
+@deffn{Function} sb-introspect:who-binds symbol
+Find the source locations where the special variable @code{symbol} is bound,
+and return them as an alist of function or macro name,
+@code{definition-source} pairs.
+@end deffn
+@anchor{Function sb-introspect who-references}
+@ffindex @sortas{who-references sb-introspect} who-references [sb-introspect]
+@deffn{Function} sb-introspect:who-references symbol
+Find the source locations where the special variable @code{symbol} is read,
+and return them as an alist of function or macro name,
+@code{definition-source} pairs.
+@end deffn
+@anchor{Function sb-introspect who-sets}
+@ffindex @sortas{who-sets sb-introspect} who-sets [sb-introspect]
+@deffn{Function} sb-introspect:who-sets symbol
+Find the source locations where the special variable @code{symbol} is set,
+and return them as an alist of function or macro name,
+@code{definition-source} pairs.
+@end deffn
+@node sb introspect functions
+@subsection Functions
+
+@anchor{Function sb-introspect function-lambda-list}
+@ffindex @sortas{function-lambda-list sb-introspect} function-lambda-list [sb-introspect]
+@deffn{Function} sb-introspect:function-lambda-list function
+Return the lambda list of @code{function}.
+@code{function} must be a function object or a function name in the sense of
+@code{valid-function-name-p}. Works for special operators, macros, simple
+functions, interpreted functions, and generic functions.
+
+The second return value indicates whether the lambda list could not be
+determined (e.g. because the function was compiled with @code{debug} 0).
+@end deffn
+@anchor{Function sb-introspect function-type}
+@ffindex @sortas{function-type sb-introspect} function-type [sb-introspect]
+@deffn{Function} sb-introspect:function-type function-designator
+Returns the ftype of @code{function-designator} or @code{nil}.
+@end deffn
+@anchor{Function sb-introspect method-combination-lambda-list}
+@ffindex @sortas{method-combination-lambda-list sb-introspect} method-combination-lambda-list [sb-introspect]
+@deffn{Function} sb-introspect:method-combination-lambda-list method-combination
+Return the lambda list of the @code{method-combination} designator.
+@code{method-combination} can be a method combination object,
+or a method combination name.
+@end deffn
+@anchor{Function sb-introspect valid-function-name-p}
+@ffindex @sortas{valid-function-name-p sb-introspect} valid-function-name-p [sb-introspect]
+@deffn{Function} sb-introspect:valid-function-name-p name
+See if @code{name} is a valid function name. In addition to the ANSI
+definition of function name, which is symbols plus lists like (@code{setf}
+@code{symbol}), SBCL allows (@code{sb-ext:cas} @code{symbol}) and various internal
+constructs.
+@end deffn
+@anchor{Function sb-introspect find-function-callers}
+@ffindex @sortas{find-function-callers sb-introspect} find-function-callers [sb-introspect]
+@deffn{Function} sb-introspect:find-function-callers function &optional spaces
+List functions that call @code{function} by searching @code{spaces} for code objects.
+This can make previously garbage objects live.
+
+@code{spaces} should be a list of the symbols @code{:dynamic}, @code{:static}, @code{:read-only},
+or @code{:immobile} on @code{#+immobile-space}. The shorthand (@code{:all}) is also
+accepted.
+@end deffn
+@anchor{Function sb-introspect find-function-callees}
+@ffindex @sortas{find-function-callees sb-introspect} find-function-callees [sb-introspect]
+@deffn{Function} sb-introspect:find-function-callees function
+Return functions called by @code{function}.
+@end deffn
+@anchor{Function sb-introspect who-calls}
+@ffindex @sortas{who-calls sb-introspect} who-calls [sb-introspect]
+@deffn{Function} sb-introspect:who-calls function-name
+Find the source locations where the global function @code{function-name} is
+called, and return them as an alist of function or macro name,
+@code{definition-source} pairs.
+@end deffn
+@anchor{Function sb-introspect who-macroexpands}
+@ffindex @sortas{who-macroexpands sb-introspect} who-macroexpands [sb-introspect]
+@deffn{Function} sb-introspect:who-macroexpands macro-name
+Find the source locations where the macro @code{macro-name} is expanded, and
+return them as an alist of function or macro name, @code{definition-source}
+pairs.
+@end deffn
+@node sb introspect types
+@subsection Types and Classes
+
+@anchor{Function sb-introspect deftype-lambda-list}
+@ffindex @sortas{deftype-lambda-list sb-introspect} deftype-lambda-list [sb-introspect]
+@deffn{Function} sb-introspect:deftype-lambda-list type-specifier-name
+Returns the lambda list of @code{type-specifier-name} as the first return
+value, and a flag whether the arglist could be found as the second
+value.
+
+@code{type-specifier-name} must be a symbol. This function can find the
+lambda list of derived type specifiers (e.g. those defined with
+@code{deftype}) and classes with compound type specifier syntaxes (e.g. the
+class @code{float}). It returns @code{nil}, @code{nil} for other type specifiers (e.g. @code{and},
+@code{or}, @code{not}) and types (e.g. @code{list}).
+@end deffn
+@anchor{Function sb-introspect who-specializes-directly}
+@ffindex @sortas{who-specializes-directly sb-introspect} who-specializes-directly [sb-introspect]
+@deffn{Function} sb-introspect:who-specializes-directly class-designator
+Find the source locations of methods directly specializing on
+@code{class-designator}, and return them as an alist of generic function
+name, @code{definition-source} pairs.
+
+A method matches the criterion either if it specializes on the same
+class as @code{class-designator} designates, or if it eql-specializes on an
+instance of the designated class.
+
+Experimental.
+@end deffn
+@anchor{Function sb-introspect who-specializes-generally}
+@ffindex @sortas{who-specializes-generally sb-introspect} who-specializes-generally [sb-introspect]
+@deffn{Function} sb-introspect:who-specializes-generally class-designator
+Find the source locations of methods specializing on
+@code{class-designator} or a subclass of it, and return them as an alist of
+generic function name, @code{definition-source} pairs.
+@code{definition-source-description} identifies the method.
+
+A method matches the criterion either if it specializes on the
+designated class itself or a subclass of it (this includes CLASS-EQ
+specializers), or if it eql-specializes on an instance of the
+designated class or a subclass of it.
+
+Experimental.
+@end deffn
+@node sb introspect allocation
+@subsection Allocation
+
+@anchor{Function sb-introspect allocation-information}
+@ffindex @sortas{allocation-information sb-introspect} allocation-information [sb-introspect]
+@deffn{Function} sb-introspect:allocation-information object
+Returns information about the allocation of @code{object}. The primary return
+value indicates the general type of allocation: @code{:immediate}, @code{:heap},
+@code{:stack}, or @code{:foreign}.
+
+Non-NIL secondary return values provide additional information about
+the allocation.
+
+For @code{:heap} objects the secondary value is a plist:
+
+  @code{:space}
+    Indicates the heap segment the object is allocated in.
+
+  @code{:generation}
+    The current generation of the object: 0 for nursery, 6 for pseudo-static
+    generation loaded from core. (GENCGC and @code{:space} @code{:dynamic} only.)
+
+  @code{:large}
+    Indicates a "large" object subject to non-copying
+    promotion. (GENCGC and @code{:space} @code{:dynamic} only.)
+
+  @code{:boxed}
+    Indicates that the object is allocated in a boxed region. Unboxed
+    allocation is used for e.g. specialized arrays after they have survived one
+    collection. (GENCGC and @code{:space} @code{:dynamic} only.)
+
+  @code{:pinned}
+    Indicates that the page(s) on which the object resides are kept live due
+    to conservative references. Note that object may reside on a pinned page
+    even if @code{:pinned} is @code{nil} if the GC has not had the need to mark the page
+    as pinned. (GENCGC and @code{:space} @code{:dynamic} only.)
+
+  @code{:write-protected}
+    Indicates that the page on which the object starts is write-protected,
+    which indicates for @code{:boxed} objects that it hasn't been written to since
+    the last GC of its generation. (GENCGC and @code{:space} @code{:dynamic} only.)
+
+  @code{:page}
+    The index of the page the object resides on. (GENCGC and @code{:space} @code{:dynamic}
+    only.)
+
+For @code{:stack} objects, the secondary value is the thread on whose stack
+the object is allocated.
+
+Expected use-cases include introspection to gain insight into allocation and
+GC behaviour and restricting memoization to heap-allocated arguments.
+
+Experimental: interface subject to change.
+@end deffn
+@anchor{Function sb-introspect map-root}
+@ffindex @sortas{map-root sb-introspect} map-root [sb-introspect]
+@deffn{Function} sb-introspect:map-root function object &key simple ext
+Call @code{function} with all non-immediate objects pointed to by @code{object}.
+Returns @code{object}.
+
+If @code{simple} is true (default is @code{nil}), elides those pointers that are not
+notionally part of certain built-in objects but backpointers to a
+conceptual parent: e.g. elides the pointer from a @code{symbol} to the
+corresponding @code{package}.
+
+If @code{ext} is true (default is @code{t}), includes some pointers that are not
+actually contained in the object but found in certain well-known
+indirect containers: @code{fdefinition}s, @code{eql} specializers, classes, and
+thread-local symbol values in other threads fall into this category.
+
+@quotation
+@emph{Note}: calling @code{map-root} with a THREAD does not currently map over
+conservative roots from the thread registers and interrupt contexts.
+@end quotation
+
+Experimental: interface subject to change.
+@end deffn

contrib/sb-introspect/test-driver.lisp

@@ -8,7 +8,7 @@
 ;;;; files for more information.
 
 (defpackage :sb-introspect-test
-  (:import-from #:test-util #:deftest)
+  (:import-from #:test-util #:deftest #:with-test)
   (:use "SB-INTROSPECT" "CL"))
 
 (in-package :sb-introspect-test)
@@ -47,6 +47,11 @@
            '(x b))
   t)
 
+(deftest deftype-lambda-list.1
+    (deftype-lambda-list 'cl-user::0-debug-type)
+  nil
+  nil)
+
 (defmacro interpret (form)
   `(let ((sb-ext:*evaluator-mode* :interpret))
      (eval ',form)))
@@ -70,7 +75,7 @@
            '(x &key (b "abc")))
   t)
 
-(test-util:with-test (:name definition-source.1 :skipped-on :no-source-locs)
+(with-test (:name definition-source.1 :skipped-on :no-source-locs)
   #+sb-devel
   (assert (consp (find-definition-sources-by-name 'vectorp :vop)))
   (assert (consp (find-definition-sources-by-name 'check-type :macro))))
@@ -92,7 +97,7 @@
 
 ;;; The behavior of :SOURCE-PLIST on nested WITH-COMPILATION-UNIT
 ;;; is to append. This is documented in source/compiler/main
-(test-util:with-test (:name :definition-source-plist.2)
+(with-test (:name :definition-source-plist.2)
     (let ((plist (definition-source-plist
                      (find-definition-source #'cl-user::four))))
       (assert (equal (getf plist :test-outer) "OUT"))
@@ -122,25 +127,25 @@
     (matchp #'cl-user::one 2)
   t)
 
-(test-util:with-test (:name find-source-stuff.3 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.3 :skipped-on :no-source-locs)
   (assert (matchp-name :generic-function 'cl-user::two 3)))
 
-(test-util:with-test (:name find-source-stuff.4 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.4 :skipped-on :no-source-locs)
   (assert (matchp (car (sb-mop:generic-function-methods #'cl-user::two)) 4)))
 
-(test-util:with-test (:name find-source-stuff.5 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.5 :skipped-on :no-source-locs)
   (assert (matchp-name :variable 'cl-user::*a* 8)))
 
-(test-util:with-test (:name find-source-stuff.6 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.6 :skipped-on :no-source-locs)
   (assert (matchp-name :variable 'cl-user::*b* 9)))
 
-(test-util:with-test (:name find-source-stuff.7 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.7 :skipped-on :no-source-locs)
   (assert (matchp-name :class 'cl-user::a 10)))
 
-(test-util:with-test (:name find-source-stuff.8 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.8 :skipped-on :no-source-locs)
   (assert (matchp-name :condition 'cl-user::b 11)))
 
-(test-util:with-test (:name find-source-stuff.9 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.9 :skipped-on :no-source-locs)
   (assert (matchp-name :structure 'cl-user::c 12)))
 
 (deftest find-source-stuff.10
@@ -151,7 +156,7 @@
     (matchp-name :function 'cl-user::c-e 12)
   t)
 
-(test-util:with-test (:name find-source-stuff.12 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.12 :skipped-on :no-source-locs)
   (assert (matchp-name :structure 'cl-user::d 13)))
 
 (deftest find-source-stuff.13
@@ -162,16 +167,16 @@
     (matchp-name :function 'cl-user::d-e 13)
   t)
 
-(test-util:with-test (:name find-source-stuff.15 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.15 :skipped-on :no-source-locs)
   (assert (matchp-name :package 'cl-user::e 14)))
 
-(test-util:with-test (:name find-source-stuff.16 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.16 :skipped-on :no-source-locs)
   (assert (matchp-name :symbol-macro 'cl-user::f 15)))
 
-(test-util:with-test (:name find-source-stuff.17 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.17 :skipped-on :no-source-locs)
   (assert (matchp-name :type 'cl-user::g 16)))
 
-(test-util:with-test (:name find-source-stuff.18 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.18 :skipped-on :no-source-locs)
   (assert (matchp-name :constant 'cl-user::+h+ 17)))
 
 (deftest find-source-stuff.19
@@ -194,7 +199,7 @@
     (matchp-name :function '(setf cl-user::o) 23)
   t)
 
-(test-util:with-test (:name find-source-stuff.24 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.24 :skipped-on :no-source-locs)
   (assert (matchp-name :method '(setf cl-user::p) 24)))
 
 (deftest find-source-stuff.25
@@ -202,7 +207,7 @@
   t)
 
 
-(test-util:with-test (:name find-source-stuff.26 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.26 :skipped-on :no-source-locs)
   (assert (matchp-name :method-combination 'cl-user::r 26)))
 
 
@@ -238,7 +243,7 @@
     (matchp-name :function 'cl-user::loaded-as-source-fun 3)
   t)
 
-(test-util:with-test (:name find-source-stuff.33 :skipped-on :no-source-locs)
+(with-test (:name find-source-stuff.33 :skipped-on :no-source-locs)
   (assert (matchp-name :variable 'cl-user::**global** 29)))
 
 ;;; Check wrt. interplay of generic functions and their methods.
@@ -366,7 +371,7 @@
     (tai 42s0 :immediate nil)
   t)
 
-(test-util:with-test (:name :allocation-information.4
+(with-test (:name :allocation-information.4
            ;; Ignored as per the comment above, even though it seems
            ;; unlikely that this is the right condition.
            :fails-on (or :ppc64 (and :sparc :gencgc)))
@@ -383,7 +388,7 @@
   (setq *large-obj* (make-array (* sb-vm:gencgc-page-bytes 4)
                                 :element-type '(unsigned-byte 8)))
   (sb-ext:gc :gen 1) ; Array won't move to a large unboxed page until GC'd
-  (test-util:with-test (:name allocation-information.5
+  (with-test (:name allocation-information.5
                         :skipped-on :mark-region-gc) ; doesn't move to an unboxed page
           (tai *large-obj* :heap
                `(:space :dynamic :generation 1 :boxed nil :pinned nil :large t)
@@ -731,20 +736,23 @@
   t
   t)
 
-(test-util:with-test (:name alien-type.1 :skipped-on :no-source-locs)
+(with-test (:name alien-type.1 :skipped-on :no-source-locs)
   (assert (matchp-name :alien-type 'cl-user::test-alien-type 30)))
 
-(test-util:with-test (:name alien-type.2 :skipped-on :no-source-locs)
+(with-test (:name alien-type.2 :skipped-on :no-source-locs)
   (assert (matchp-name :alien-type 'cl-user::test-alien-struct 31)))
 
-(test-util:with-test (:name alien-variable :skipped-on :no-source-locs)
+(with-test (:name alien-variable :skipped-on :no-source-locs)
   (assert (matchp-name :variable 'cl-user::test-alien-var 32)))
 
-(test-util:with-test (:name condition-slot-reader :skipped-on :no-source-locs)
-  (matchp-name :method 'cl-user::condition-slot-reader 33))
+(with-test (:name alien-callable :skipped-on :no-source-locs)
+  (assert (matchp-name :alien-callback 'cl-user::alien-comparator 33)))
 
-(test-util:with-test (:name condition-slot-writer :skipped-on :no-source-locs)
-  (matchp-name :method 'cl-user::condition-slot-writer 33))
+(with-test (:name condition-slot-reader :skipped-on :no-source-locs)
+  (matchp-name :method 'cl-user::condition-slot-reader 34))
+
+(with-test (:name condition-slot-writer :skipped-on :no-source-locs)
+  (matchp-name :method 'cl-user::condition-slot-writer 34))
 
 (deftest function-with-a-local-function
     (sb-introspect:definition-source-form-number
@@ -796,10 +804,50 @@
 (defun sets-equalp (a b) (and (subsetp a b) (subsetp b a)))
 (deftest find-callees
     (sets-equalp (sb-introspect:find-function-callees #'sb-c::find-dominators)
-                 (list #'sb-c::dfo-as-needed))
+                 (list #'sb-c::dfo-as-needed #'sb-c::lowest-common-dominator))
   t)
 (deftest find-callers
     (let ((callers (sb-introspect:find-function-callers #'sb-c::dfo-as-needed)))
       (and (>= (length callers) 5)
            (not (null (member #'sb-c::find-dominators callers)))))
   t)
+
+(defparameter *definition-types*
+  '(:class
+    :compiler-macro
+    :condition
+    :constant
+    :function
+    :generic-function
+    :macro
+    :method
+    :method-combination
+    :package
+    :setf-expander
+    :structure
+    :symbol-macro
+    :type
+    :alien-type
+    :alien-callback
+    :variable
+    :declaration
+    :optimizer
+    :source-transform
+    :transform
+    :vop
+    :ir1-convert))
+
+(with-test (:name :find-definition-sources-by-name-error-handling)
+  (dolist (definition-type *definition-types*)
+    (assert (null (sb-introspect:find-definition-sources-by-name
+                   7 definition-type)))
+    (assert (null (sb-introspect:find-definition-sources-by-name
+                   "xxx" definition-type)))
+    (assert (null (sb-introspect:find-definition-sources-by-name
+                   '(setf (setf xxx)) definition-type))))
+  (assert (null (sb-introspect:find-definition-sources-by-name 'print :junk))))
+
+(with-test (:name (:find-definition-sources-by-name :package))
+  (assert (sb-introspect:find-definition-sources-by-name "CL" :package))
+  (assert (sb-introspect:find-definition-sources-by-name :cl :package))
+  (assert (null (sb-introspect:find-definition-sources-by-name #\a :package))))

contrib/sb-introspect/test.lisp

@@ -85,6 +85,11 @@
 
 (sb-alien:define-alien-variable ("errno" test-alien-var) sb-alien:int)
 
+(sb-alien:define-alien-callable alien-comparator sb-alien:int
+  ((p1 (* t)) (p2 (* t)))
+  (- (sb-alien:deref (sb-alien:cast p1 (* sb-alien:int)))
+     (sb-alien:deref (sb-alien:cast p2 (* sb-alien:int)))))
+
 (define-condition test-condition (error)
   ((a :reader condition-slot-reader
       :writer condition-slot-writer)))
@@ -95,3 +100,11 @@
     (x)))
 
 (defun 0-debug (a b c) (declare (optimize (debug 0))) (+ a b c))
+
+(defvar *x* 1)
+
+(eval-when (:compile-toplevel)
+  (declaim (optimize (debug 0))))
+
+(deftype 0-debug-type ()
+  `(member ,*x*))

contrib/sb-manual/.gitignore

@@ -0,0 +1 @@
+output/

contrib/sb-manual/Makefile

@@ -0,0 +1,2 @@
+SYSTEM=sb-manual
+include ../asdf-module.mk

contrib/sb-manual/README.md

@@ -0,0 +1,264 @@
+Docstring Style Guide
+---------------------
+
+The Markdown-to-Texinfo converter is documented in `markdown.lisp`.
+Here, we provide a quick howto and style guide. If something is
+unclear (there should be lots), then you can test things with e.g.
+
+    (sb-manual::markdown-to-texinfo "PRINT")
+
+## Writing Names Inline
+
+- Upcase symbols naming definitions (or arguments of the function
+  being documented):
+
+        SB-EXT:EXIT
+        PRINT
+        X
+
+    Unqualified symbols must be accessible in the package where the
+    docstring is parsed. A good practice is to rely on Slime's TAB
+    completion on the name; then it's clear that the symbol exists.
+
+- Upcase and mark strings naming definitions as code:
+
+        `SB-EXT`
+
+    This is necessary because we don't want the creation of package
+    `ANSI` to automatically codify all occurrences of `ANSI` in the
+    documentation.
+
+- Upcase and mark non-existent symbols as code:
+
+        `WIN64::WHATEVER`
+
+    For when the code that defines the package or the symbol cannot be
+    loaded.
+
+- Mark C symbols as code:
+
+        `send`
+        `send(2)`
+
+    The latter is preferable to `send`(2), as it can be parsed by
+    `manual-entry` in Emacs.
+
+    If there is a Lisp symbol `SEND` accessible in the current package
+    and it names a definition, then you need to prevent it from being
+    autolinked (only with PAX, currently):
+
+        `\send(2)`
+
+    > _Note_: Here and in general, the backslashes need to be doubled
+    > when writing docstrings.
+
+- Mark C constants as code and escape downcasing:
+
+       `\\AF_LOCAL`
+
+    All capital code is downcased, which is great in the common case
+    but not here.
+
+## Writing Sexps Inline
+
+- One option is to mark the whole expression as code:
+
+        `(PRINT 'HELLO)`
+
+    Here, everything is rendered in monospace, downcased, no autolinks.
+
+- Alternatively, one could simply write
+
+        (PRINT '`HELLO`)
+
+    and rely on automatic codification for `PRINT`, codify `HELLO`
+    manually, and accept the fact that the parens will be in the
+    proportional font. In return, `PRINT` is autolinked.
+
+- Even more alternatively:
+
+        `(`PRINT` 'HELLO)`
+
+    Here, everything is monospace (except maybe the space), and
+    `PRINT` is autolinked.
+
+## Writing Signatures Inline
+
+Follow the somewhat usual `<TERMINAL-NAME>` convention:
+
+    One option is `(:USE <PACKAGE-NAME>*)`, where `<PACKAGE-NAME>`
+    is a package designator.
+
+If you wrote the mixed-case `(:USE <package-name>*)`, then the
+expression would be downcased, which is bad here.
+
+## Code Blocks
+
+In code blocks, always write as you would write in a source file,
+which means downcase the symbols. Prefer indented code blocks
+
+```
+    this is indented
+```
+
+to fenced code blocks:
+
+    ```
+    this is fenced
+    ```
+
+Use fenced code blocks only if there multiple successive code blocks
+that you definitely want to render as distinct "boxes".
+
+One gotcha to look out for is indenting code blocks is in list items:
+
+```
+- this is a list
+
+    A separate child paragraph indented to be "within" the list item.
+
+        (some code)
+```
+
+Note how the code block is indented 8 spaces from the `-` character.
+The required indentation remains the same if the child paragraph above
+is not present.
+
+## Links
+
+### Plain Links
+
+For short links:
+
+    <http://x.y>
+
+### Explicit Links
+
+For overly long links:
+
+    [label](http://x.y/a?b=c)
+
+When generating Texinfo, this is translated to `@url`, which is
+rendered like a HTML link in HTML and PDF, but with `label (see
+<uri>)` in Info.
+
+PAX will generate a normal link except in plain text, where it simply
+drops the URI.
+
+### Reflinks
+
+In PAX-generated output only, `FUNCTION` is autolinked to its
+definitions. If there is more than one, you can dismambiguate:
+
+    [FUNCTION][type]
+
+PAX will produce a single link in this case. If `(FUNCTION CLASS)` is
+documented in some section, then the link will point to that
+documentation. Else, the link will go to the CLHS. The latter can be
+forced:
+
+    [FUNCTION][(clhs type)]
+
+Since we don't link to definitions in Texinfo (except to sections),
+the above examples are translated to `FUNCTION` there.
+
+## Notes
+
+- In many cases, just write a sentence:
+
+        Note that this may not be always so.
+
+- To add a note, use blockquotes:
+
+        > _Note_: This is not terribly important,
+        > but it can span multiple lines.
+
+    Blockquotes render indented in most formats, sometimes with a
+    vertical line to their left (e.g. Markdown on GitHub).
+
+- To add a warning, use blockquotes:
+
+        > __Warning__: Beware of the dog!
+
+Blockquotes without `_Note_` and `__Warning__` are fine for their
+other purposes: citing stuff, add copyright notices, etc. Markdown is
+not semantic. Use it freely and visually.
+
+Footnotes are not supported.
+
+## Typographic Miscellanea
+
+The Markdown-to-Texinfo converter does not convert e.g. `--` to
+en-dash. This is to prevent messing up the output when something like
+`git <option>* -- <path>` is written without proper markup. The issue
+is similar but maybe more pronounced with literal strings and curly
+quotes: you don't want to copy the rendered version of
+
+    (print "Hello, world")
+
+to the REPL, only to find the double quotes have been replaced by some
+fancy characters.
+
+## Inline Quotes and Emphasis
+
+Use strong emphasis (e.g. `__not__`) very sparingly. It usually gets
+rendered bold, which draws too much attention. However, emphasis
+(usually italic) is fine.
+
+You may use emphasis instead of single or double quotes:
+
+          CLHS `14.1` says that _depending on context, a group of
+          connected conses can be viewed in a variety of different
+          ways_, but that's trivial.
+
+You may also use double quotes, but the straight double quotes are
+kind of ugly in proportional fonts.
+
+## Sections
+
+### Titles
+
+Use title case:
+
+    Big and Bigger Words
+
+### Names
+
+Name the section `@BIG-AND-BIGGER-WORDS` or something shorter like
+`@BIG-WORDS`. In a docstring, you may read `"@BIG-WORDS are
+necessary"`, and `"Big and Bigger Words are necessary"` when it's
+rendered, so make sure that they are similar enough.
+
+Note that section names live in a flat namespace: they are all
+exported from SB-MANUAL, and they also show up the Texinf indices
+without the context of their parents. So, their names should be
+globally meaningful. Thus, it is better to name the section describing
+`SB-ACLREPL`'s usage `@SB-ACLREPL-USAGE` than `@USAGE`.
+
+## Docstring Formatting
+
+The Emacs style:
+
+    (defun foo (x)
+      "Return X.
+    It signals no errors."
+      x)
+
+You may also indent all but the first line as long as you do it
+consistently within the docstring. Maybe one day we could even agree
+upon a universally accepted style ... just joking.
+
+## Locale Stuff
+
+Currently the manual is a mix of American and British English.
+
+In the vast majority of the cases, the existing docstrings use `e.g.
+X` and `i.e. X` (the British version). For consistency, do not use
+other forms such as `eg. X` or `i.e., X`.
+
+## Paragraph Formatting
+
+Note that docstrings are also inconsistent about whether one or two
+spaces follow a full stop (controlled by `sentence-end-double-space`
+in Emacs) and their `fill-column`. It would be nice to have them all
+`fill-paragraph`ed with the same settings ...

contrib/sb-manual/TODO.md

@@ -0,0 +1,26 @@
+- How/when to load/include docs of contribs?
+
+    Currently, `sb-manual` loads *all* contribs to be able to query
+    the definition docstrings. Each contrib directory has a
+    `manual.lisp` file, which is part of the `sb-manual` contrib.
+
+    On the positive side, this does not load extra stuff until the
+    user `require`s `sb-manual`. However, then it loads all contribs.
+
+    It shouldn't be difficult to show documtation only for actually
+    loaded contribs.
+
+- How to deal with repetitive package names?
+
+    For example, `sb-alien` is `:use`d by `sb-manual` so that the
+    section docstrings need not fully qualify with `sb-alien:` a
+    thousand times. In the generated Texinfo, this can be a tad
+    confusing. In output formats with links (e.g. HTML from PAX), this
+    is clearly preferable.
+
+    Nicknames, maybe?
+
+- Improve section names
+
+    They are a soft interface: exported from `sb-manual` and visible
+    to the user via HTML anchors.

contrib/sb-manual/doc/compiler.lisp

@@ -0,0 +1,903 @@
+(in-package :sb-manual)
+
+(defsection @compiler (:title "Compiler")
+  "This chapter will discuss most compiler issues other than efficiency,
+  including compiler error messages, the SBCL compiler's unusual
+  approach to type safety in the presence of type declarations, the
+  effects of various compiler optimization policies, and the way that
+  inlining and open coding may cause optimized code to differ from a
+  naive translation. Efficiency issues are sufficiently varied and
+  separate that they have their own chapter, @EFFICIENCY."
+  (@diagnostic-messages section)
+  (@handling-of-types section)
+  (@compiler-policy section)
+  (@compiler-errors section)
+  (@open-coding-and-inline-expansion section)
+  (@interpreter section)
+  (@advanced-compiler-use-and-efficiency-hints section))
+
+(defsection @diagnostic-messages (:title "Diagnostic Messages"
+                                  :concepts (("compiler" "messsage")
+                                             ("messsage," "compiler")))
+  (@controlling-verbosity section)
+  (@diagnostic-severity section)
+  (@understanding-compiler-diagnostics section))
+
+(defsection @controlling-verbosity
+    (:title "Controlling Verbosity"
+     :concepts (("compiler" "messsage" "verbosity")
+                ("verbosity" "of compiler messsages")))
+  "The compiler can be quite verbose in its diagnostic reporting, rather
+  more then some users would prefer -- the amount of noise emitted can
+  be controlled, however.
+
+  To control emission of compiler diagnostics (of any severity other
+  than ERROR: @DIAGNOSTIC-SEVERITY) use the SB-EXT:MUFFLE-CONDITIONS
+  and SB-EXT:UNMUFFLE-CONDITIONS declarations, specifying the type of
+  condition that is to be muffled (the muffling is done using an
+  associated MUFFLE-WARNING restart).
+
+  Global control:
+
+      ;;; Muffle compiler-notes globally
+      (declaim (sb-ext:muffle-conditions sb-ext:compiler-note))
+
+  Local control:
+
+      ;;; Muffle compiler-notes based on lexical scope
+      (defun foo (x)
+        (declare (optimize speed) (fixnum x)
+                 (sb-ext:muffle-conditions sb-ext:compiler-note))
+        (values (* x 5) ; no compiler note from this
+          (locally
+            (declare (sb-ext:unmuffle-conditions sb-ext:compiler-note))
+            ;; this one gives a compiler note
+            (* x -5))))"
+  (sb-ext:muffle-conditions declaration)
+  (sb-ext:unmuffle-conditions declaration)
+  "Various details of _how_ the compiler messages are printed can be
+  controlled via the alist SB-EXT:*COMPILER-PRINT-VARIABLE-ALIST*."
+  (sb-ext:*compiler-print-variable-alist* variable)
+  "For information about muffling warnings signaled outside of the
+  compiler, see @CUSTOMIZATION-HOOKS-FOR-USERS.")
+
+;; FIXME: How much control over error messages is in SBCL? How much
+;; should be? How much of this documentation should we save or adapt?
+;;
+;; %%\node Error Message Parameterization,  , Read Errors, Interpreting Error Messages
+;; \subsection{Error Message Parameterization}
+;; \cpsubindex{error messages}{verbosity}
+;; \cpsubindex{verbosity}{of error messages}
+;;
+;; There is some control over the verbosity of error messages.  See also
+;; \varref{undefined-warning-limit}, \code{*efficiency-note-limit*} and
+;; \varref{efficiency-note-cost-threshold}.
+;;
+;; \begin{defvar}{}{enclosing-source-cutoff}
+;;
+;;   This variable specifies the number of enclosing actual source forms
+;;   that are printed in full, rather than in the abbreviated processing
+;;   path format.  Increasing the value from its default of \code{1}
+;;   allows you to see more of the guts of the macroexpanded source,
+;;   which is useful when debugging macros.
+;; \end{defvar}
+;;
+;; \begin{defmac}{extensions:}{define-source-context}{%
+;;     \args{\var{name} \var{lambda-list} \mstar{form}}}
+;;
+;;   This macro defines how to extract an abbreviated source context from
+;;   the \var{name}d form when it appears in the compiler input.
+;;   \var{lambda-list} is a \code{defmacro} style lambda-list used to
+;;   parse the arguments.  The \var{body} should return a list of
+;;   subforms that can be printed on about one line.  There are
+;;   predefined methods for \code{defstruct}, \code{defmethod}, etc.  If
+;;   no method is defined, then the first two subforms are returned.
+;;   Note that this facility implicitly determines the string name
+;;   associated with anonymous functions.
+;; \end{defmac}
+
+(defsection @diagnostic-severity
+    (:title "Diagnostic Severity"
+     :concepts (("compiler" "message" "severity")
+                ("severity" "of compiler message")))
+  "There are four levels of compiler diagnostic severity:
+
+  - error
+  - warning
+  - style warning
+  - note
+
+  The first three levels correspond to condition classes which are
+  defined in the ANSI standard for Common Lisp and which have special
+  significance to the COMPILE and COMPILE-FILE functions. These levels
+  of compiler error severity occur when the compiler handles
+  conditions of these classes.
+
+  The fourth level of compiler error severity, _note_, corresponds to
+  the SB-EXT:COMPILER-NOTE, and is used for problems which are too
+  mild for the standard condition classes, typically hints about how
+  efficiency might be improved. The SB-EXT:CODE-DELETION-NOTE, a
+  subtype of SB-EXT:COMPILER-NOTE, is signalled when the compiler
+  deletes user-supplied code after proving that the code in question
+  is unreachable.
+
+  Future work for SBCL includes expanding this hierarchy of types to
+  allow more fine-grained control over emission of diagnostic
+  messages."
+  (sb-ext:compiler-note condition)
+  (sb-ext:code-deletion-note condition))
+
+(defsection @understanding-compiler-diagnostics
+    (:title "Understanding Compiler Diagnostics")
+  "The messages emitted by the compiler contain a lot of detail in a
+  terse format, so they may be confusing at first. The messages will be
+  illustrated using this example program:
+
+      (defmacro zoq (x)
+        `(roq (ploq (+ ,x 3))))
+
+      (defun foo (y)
+        (declare (symbol y))
+        (zoq y))
+
+  The main problem with this program is that it is trying to add `3`
+  to a symbol. Note also that the functions `ROQ` and `PLOQ` aren't
+  defined anywhere."
+  (@parts-of-a-compiler-diagnostic section)
+  (@original-and-actual-source section)
+  (@processing-paths section))
+
+(defsection @parts-of-a-compiler-diagnostic
+    (:title "Parts of a Compiler Diagnostic")
+  "When processing this program, the compiler will produce this warning:
+
+      ; file: /tmp/foo.lisp
+      ; in: DEFUN FOO
+      ;     (ZOQ Y)
+      ; --> ROQ PLOQ
+      ; ==>
+      ;   (+ Y 3)
+      ;
+      ; caught WARNING:
+      ;   Asserted type NUMBER conflicts with derived type (VALUES SYMBOL &OPTIONAL).
+
+  In this example we see each of the six possible parts of a compiler
+  diagnostic:
+
+  - `file: /tmp/foo.lisp` is the name of the file that the compiler
+    read the relevant code from. The file name is displayed because it
+    may not be immediately obvious when there is an error during
+    compilation of a large system, especially when
+    WITH-COMPILATION-UNIT is used to delay undefined warnings.
+
+  - `in: DEFUN FOO` is the definition top level form responsible for
+    the diagnostic. It is obtained by taking the first two elements of
+    the enclosing form whose first element is a symbol beginning with
+    `DEF`. If there is no such enclosing `DEF` form, then the
+    outermost form is used. If there are multiple `DEF` forms, then
+    they are all printed from the outside in, separated by `=>`s. In
+    this example, the problem was in the DEFUN for `FOO`.
+
+  - `(ZOQ Y)` is the _@ORIGINAL-SOURCE_ form responsible for the
+    diagnostic. Original source means that the form directly appeared
+    in the original input to the compiler, i.e. in the lambda passed
+    to COMPILE or in the top level form read from the source file. In
+    this example, the expansion of the `ZOQ` macro was responsible for
+    the message.
+
+  - `--> ROQ PLOQ` This is the _@PROCESSING-PATH_ that the compiler
+    used to produce the code that caused the message to be emitted.
+    The processing path is a representation of the evaluated forms
+    enclosing the @ACTUAL-SOURCE that the compiler encountered when
+    processing the original source. The path is the first element of
+    each form, or the form itself if the form is not a list. These
+    forms result from the expansion of macros or source-to-source
+    transformation done by the compiler. In this example, the
+    enclosing evaluated forms are the calls to `ROQ` and `PLOQ`. These
+    calls resulted from the expansion of the `ZOQ` macro.
+
+  - `==> (+ Y 3)` is the _actual source_ responsible for the
+    diagnostic. If the actual source appears in the explanation, then
+    we print the next enclosing evaluated form, instead of printing
+    the actual source twice. (This is the form that would otherwise
+    have been the last form of the processing path.) In this example,
+    the problem is with the evaluation of the reference to the
+    variable `Y`.
+
+  - `caught WARNING: Asserted type NUMBER conflicts with derived type
+    (VALUES SYMBOL &OPTIONAL).` is the _explanation_ of the problem.
+    In this example, the problem is that, while the call to `+`
+    requires that its arguments are all of type NUMBER, the compiler
+    has derived that Y will evaluate to a SYMBOL. Note that
+    `(VALUES SYMBOL &OPTIONAL)` expresses that `Y` evaluates to
+    precisely one value.
+
+  Note that each part of the message is distinctively marked:
+
+  - `file:` and `in:` mark the file and definition, respectively.
+
+  - The original source is an indented form with no prefix.
+
+  - Each line of the processing path is prefixed with `-->`.
+
+  - The actual source form is indented like the original source, but
+    is marked by a preceding `==>` line. (FIXME: no it isn't.)
+
+  - The explanation is prefixed with the diagnostic severity, which
+    can be `caught ERROR:`, `caught WARNING:`, `caught
+    STYLE-WARNING:`, or `note:`.
+
+  Each part of the message is more specific than the preceding one. If
+  consecutive messages are for nearby locations, then the front part
+  of the messages would be the same. In this case, the compiler omits
+  as much of the second message as in common with the first. For
+  example:
+
+      ; file: /tmp/foo.lisp
+      ; in: DEFUN FOO
+      ;     (ZOQ Y)
+      ; --> ROQ
+      ; ==>
+      ;   (PLOQ (+ Y 3))
+      ;
+      ; caught STYLE-WARNING:
+      ;   undefined function: PLOQ
+
+      ; ==>
+      ;   (ROQ (PLOQ (+ Y 3)))
+      ;
+      ; caught STYLE-WARNING:
+      ;   undefined function: ROQ
+
+  In this example, the file, definition and original source are
+  identical for the two messages, so the compiler omits them in the
+  second message. If consecutive messages are entirely identical, then
+  the compiler prints only the first message, followed by: `[Last
+  message occurs <repeats> times]` where `<repeats>` is the number of
+  times the message was given.
+
+  If the source was not from a file, then no file line is printed. If
+  the actual source is the same as the original source, then the
+  processing path and actual source will be omitted. If no forms
+  intervene between the original source and the actual source, then
+  the processing path will also be omitted.")
+
+(defsection @original-and-actual-source (:title "Original and Actual Source"
+                                         :concepts (@original-source
+                                                    @actual-source))
+  "The _original source_ displayed will almost always be a list. If
+  the actual source for an message is a symbol, the original source will
+  be the immediately enclosing evaluated list form. So even if the
+  offending symbol does appear in the original source, the compiler will
+  print the enclosing list and then print the symbol as the actual
+  source (as though the symbol were introduced by a macro.)
+
+  When the _actual source_ is displayed (and is not a symbol), it will
+  always be code that resulted from the expansion of a macro or a
+  source-to-source compiler optimization. This is code that did not
+  appear in the original source program; it was introduced by the
+  compiler.
+
+  Keep in mind that when the compiler displays a source form in an
+  diagnostic message, it always displays the most specific (innermost)
+  responsible form. For example, compiling this function
+
+      (defun bar (x)
+        (let (a)
+          (declare (fixnum a))
+          (setq a (foo x))
+          a))
+
+  gives this error message
+
+      ; file: /tmp/foo.lisp
+      ; in: DEFUN BAR
+      ;     (LET (A)
+      ;     (DECLARE (FIXNUM A))
+      ;     (SETQ A (FOO X))
+      ;     A)
+      ;
+      ; caught WARNING:
+      ;   Asserted type FIXNUM conflicts with derived type (VALUES NULL &OPTIONAL).
+
+  This message is not saying that there is a problem somewhere in this
+  LET -- it is saying that there is a problem with the LET itself. In
+  this example, the problem is that `A`'s NIL initial value is not a
+  FIXNUM.")
+
+(defsection @processing-paths (:title "Processing Paths"
+                               :concepts (@processing-path))
+  "The processing path is mainly useful for debugging macros, so if you
+  don't write macros, you can probably ignore it. Consider this example:
+
+      (defun foo (n)
+        (dotimes (i n *undefined*)))
+
+  Compiling results in this error message:
+
+      ; in: DEFUN FOO
+      ;     (DOTIMES (I N *UNDEFINED*))
+      ; --> DO BLOCK LET TAGBODY RETURN-FROM
+      ; ==>
+      ;   (PROGN *UNDEFINED*)
+      ;
+      ; caught WARNING:
+      ;   undefined variable: *UNDEFINED*
+
+  Note that DO appears in the processing path. This is because
+  DOTIMES expands into:
+
+      (do ((i 0 (1+ i)) (#:g1 n))
+          ((>= i #:g1) *undefined*)
+        (declare (type unsigned-byte i)))
+
+  The rest of the processing path results from the @MACROEXPANSION of
+  DO: ~SOURCE-TRANSFORM
+
+      (block nil
+        (let ((i 0) (#:g1 n))
+          (declare (type unsigned-byte i))
+          (tagbody (go #:g3)
+            #:g2    (psetq i (1+ i))
+            #:g3    (unless (>= i #:g1) (go #:g2))
+            (return-from nil (progn *undefined*)))))
+
+  In this example, the compiler descended into the BLOCK, LET, TAGBODY
+  and RETURN-FROM to reach the PROGN printed as the actual source.
+  This is a place where the \"actual source appears in explanation\"
+  rule was applied. The innermost actual source form was the symbol
+  _undefined_ itself, but that also appeared in the explanation, so
+  the compiler backed out one level.")
+
+(defsection @handling-of-types (:title "Handling of Types")
+  "One of the most important features of the SBCL compiler (similar to
+  the original CMUCL compiler, also known as _Python_) is its fairly
+  sophisticated understanding of the Common Lisp type system and its
+  conservative approach to the implementation of type declarations.
+
+  These two features reward the use of type declarations throughout
+  development, even when high performance is not a concern. Also, as
+  discussed in the chapter on performance (see @EFFICIENCY), the use
+  of appropriate type declarations can be very important for
+  performance as well.
+
+  The SBCL compiler also has a greater knowledge of the Common Lisp
+  type system than other compilers. Support is incomplete only for
+  types involving the SATISFIES type specifier."
+  (@declarations-as-assertions section)
+  (@precise-type-checking section)
+  (@getting-existing-programs-to-run section)
+  (@implementation-limitations section))
+
+;; FIXME: See also sections \ref{advanced-type-stuff} and
+;; \ref{type-inference}, once we snarf them from the CMU CL manual.
+;;
+;; Also see my paper on improving Baker, when I get round to it.
+;;
+;; Whose paper?
+
+(defsection @declarations-as-assertions (:title "Declarations as Assertions")
+  "The SBCL compiler treats type declarations differently from most other
+  Lisp compilers. Under default compilation policy the compiler doesn't
+  blindly believe type declarations, but considers them assertions about
+  the program that should be checked: all type declarations that have
+  not been proven to always hold are asserted at runtime.
+
+  _Remaining bugs in the compiler's handling of types unfortunately
+  provide some exceptions to this rule, see
+  @IMPLEMENTATION-LIMITATIONS._
+
+  CLOS slot types form a notable exception. Types declared using the
+  :TYPE slot option in DEFCLASS are asserted if and only if the class
+  was defined in _safe code_ ~SAFETY and the slot access location is
+  in _safe code_ as well. This laxness does not pose any internal
+  consistency issues, as the CLOS slot types are not available for the
+  type inferencer, nor do CLOS slot types provide any efficiency
+  benefits.
+
+  There are three type checking policies available in SBCL, selectable
+  via OPTIMIZE declarations."
+  ;; FIXME: This should be properly integrated with general policy
+  ;; stuff, once that gets cleaned up.
+  "- __Full Type Checks__
+
+      All declarations are considered assertions to be checked at
+      runtime, and all type checks are precise. The default
+      compilation policy provides full type checks.
+
+      Used when `(OR (>= SAFETY 2) (>= SAFETY SPEED 1))`.
+
+  - __Weak Type Checks__
+
+      Declared types may be simplified into faster to check
+      supertypes: for example, `(OR (INTEGER -17 -7) (INTEGER 7 17))`
+      is simplified into `(INTEGER -17 17)`.
+
+      > __Warning__: It is relatively easy to corrupt the heap when
+      > weak type checks are used if the program contains type-errors.
+
+      Used when `(AND (< SAFETY 2) (< SAFETY SPEED))`.
+
+  - __No Type Checks__
+
+      All declarations are believed without assertions. Also disables
+      argument count and array bounds checking.
+
+      > __Warning__: Any type errors in code where type checks are not
+      > performed are liable to corrupt the heap.
+
+      Used when `(= SAFETY 0)`.")
+
+(defsection @precise-type-checking (:title "Precise Type Checking"
+                                    :concepts (("type checking," "precise")
+                                               ("precise" "type checking")))
+  "Precise checking means that the check is done as though TYPEP
+  had been called with the exact type specifier that appeared in the
+  declaration.
+
+  If a variable is declared to be `(INTEGER 3 17)`, then its value
+  must always be an integer between `3` and `17`. If multiple type
+  declarations apply to a single variable, then all the declarations
+  must be correct; it is as though all the types were intersected
+  producing a single AND type specifier.
+
+  To gain maximum benefit from the compiler's type checking, you
+  should always declare the types of function arguments and structure
+  slots as precisely as possible. This often involves the use of OR,
+  MEMBER, and other list-style type specifiers.")
+
+(defsection @getting-existing-programs-to-run
+    (:title "Getting Existing Programs to Run"
+     :concepts (("existing programs," "getting them to run")
+                ("types," "portability")
+                ("compatibility" "with other Lisps")))
+  "Since SBCL's compiler does much more comprehensive type checking than
+  most Lisp compilers, SBCL may detect type errors in programs that have
+  been debugged using other compilers. These errors are mostly incorrect
+  declarations, although compile-time type errors can find actual bugs
+  if parts of the program have never been tested.
+
+  Some incorrect declarations can only be detected by run-time type
+  checking. It is very important to initially compile a program with
+  full type checks (high @SAFETY optimization) and then test this safe
+  version. After the checking version has been tested, then you can
+  consider weakening or eliminating type checks. _This applies even to
+  previously debugged programs_ because the SBCL compiler does much
+  more type inference than other Common Lisp compilers, so an
+  incorrect declaration can do more damage.
+
+  The most common problem is with variables whose constant initial
+  value doesn't match the type declaration. Incorrect constant initial
+  values will always be flagged by a compile-time type error, and they
+  are simple to fix once located. Consider this code fragment:
+
+      (prog (foo)
+        (declare (fixnum foo))
+        (setq foo ...)
+        ...)
+
+  Here `FOO` is given an initial value of NIL but is declared to be a
+  FIXNUM. Even if it is never read, the initial value of a variable
+  must match the declared type. There are two ways to fix this
+  problem. Change the declaration
+
+      (prog (foo)
+        (declare (type (or fixnum null) foo))
+        (setq foo ...)
+        ...)
+
+  or change the initial value
+
+      (prog ((foo 0))
+        (declare (fixnum foo))
+        (setq foo ...)
+        ...)
+
+  It is generally preferable to change to a legal initial value rather
+  than to weaken the declaration, but sometimes it is simpler to
+  weaken the declaration than to try to make an initial value of the
+  appropriate type.
+
+  Another declaration problem occasionally encountered is incorrect
+  declarations on DEFMACRO arguments. This can happen when a function
+  is converted into a macro. Consider this macro:
+
+      (defmacro my-1+ (x)
+        (declare (fixnum x))
+        `(the fixnum (1+ ,x)))
+
+  Although legal and well-defined Common Lisp code, this meaning of
+  this definition is almost certainly not what the writer intended.
+  For example, this call is illegal:
+
+      (my-1+ (+ 4 5))
+
+  This call is illegal because the argument to the macro is `(+ 4 5)`,
+  which is a LIST, not a FIXNUM. Because of macro semantics, it is
+  hardly ever useful to declare the types of macro arguments. If you
+  really want to assert something about the type of the result of
+  evaluating a macro argument, then put a THE in the expansion:
+
+      (defmacro my-1+ (x)
+        `(the fixnum (1+ (the fixnum ,x))))
+
+
+  In this case, it would be stylistically preferable to change this
+  macro back to a function and declare it inline."
+  ;; FIXME: <xref>inline-expansion, once we crib the relevant text
+  ;; from the CMU CL manual.
+  "Some more subtle problems are caused by incorrect declarations that
+  can't be detected at compile time. Consider this code:
+
+      (do ((pos 0 (position #\a string :start (1+ pos))))
+        ((null pos))
+        (declare (fixnum pos))
+        ...)
+
+  Although `POS` is almost always a FIXNUM, it is NIL at the end of
+  the loop. If this example is compiled with full type checks (the
+  default), then running it will signal a type error at the end of the
+  loop. If compiled without type checks, the program will go into an
+  infinite loop (or perhaps POSITION will complain because `(1+ NIL)`
+  isn't a sensible start.) Why? Because if you compile without type
+  checks, the compiler just quietly believes the type declaration.
+  Since the compiler believes that `POS` is always a FIXNUM, it
+  believes that `POS` is never NIL, so `(NULL POS)` is never true, and
+  the loop exit test is optimized away. Such errors are sometimes
+  flagged by unreachable code notes, but it is still important to
+  initially compile and test any system with full type checks, even if
+  the system works fine when compiled using other compilers.
+
+  In this case, the fix is to weaken the type declaration to `(OR
+  FIXNUM NULL)`. (Actually, this declaration is unnecessary in SBCL,
+  since it already knows that POSITION returns a non-negative FIXNUM
+  or NIL.)
+
+  Note that there is usually little performance penalty for weakening
+  a declaration in this way. Any numeric operations in the body can
+  still assume that the variable is a FIXNUM, since NIL is not a legal
+  numeric argument. Another possible fix would be to say:
+
+      (do ((pos 0 (position #\a string :start (1+ pos))))
+          ((null pos))
+        (let ((pos pos))
+          (declare (fixnum pos))
+          ...))
+
+  This would be preferable in some circumstances, since it would allow
+  a non-standard representation to be used for the local `POS`
+  variable in the loop body."
+  ;; FIXME: <xref>ND-variables, once we crib the text from the CMU CL
+  ;; manual.
+  )
+
+(defsection @implementation-limitations (:title "Implementation Limitations")
+  "If an FTYPE is placed after the function definition the function won't
+  perform any type checks, and the calls to the function will blindly
+  trust the declared types.
+  (OPTIMIZE (DEBUG 3)) will not trust any FTYPE declarations.")
+
+(defsection @compiler-policy (:title "Compiler Policy")
+  "Compiler policy is controlled by the OPTIMIZE declaration,
+  supporting all ANSI optimization qualities (DEBUG, safety, space,
+  and speed). (A deprecated extension SB-EXT:INHIBIT-WARNINGS is still
+  supported but liable to go away at any time.)
+
+  For effects of various optimization qualities on type-safety and
+  debuggability see @DECLARATIONS-AS-ASSERTIONS and
+  @DEBUGGER-POLICY-CONTROL.
+
+  Ordinarily, when the speed quality is high, the compiler emits notes
+  to notify the programmer about its inability to apply various
+  optimizations. For selective muffling of these notes, see
+  @CONTROLLING-VERBOSITY.
+
+  The value of space mostly influences the compiler's decision whether
+  to inline operations, which tend to increase the size of programs.
+  Use the value `0` with caution, since it can cause the compiler to
+  inline operations so indiscriminately that the net effect is to slow
+  the program by causing cache misses or even swapping."
+  (sb-ext:describe-compiler-policy function)
+  (sb-ext:restrict-compiler-policy function)
+  (with-compilation-unit macro))
+
+;; FIXME: old CMU CL compiler policy, should perhaps be adapted for
+;; SBCL. (Unfortunately, the CMU CL docs are out of sync with the CMU
+;; CL code, so adapting this requires not only reformatting the
+;; documentation, but rooting out code rot.)
+;;
+;; <sect2 id=\")compiler-policy\"><title>Compiler Policy</1000
+;;   INDEX {policy}{compiler}
+;;   INDEX compiler policy
+;;
+;; <para>The policy is what tells the compiler <emphasis>how</emphasis> to
+;; compile a program. This is logically (and often textually) distinct
+;; from the program itself. Broad control of policy is provided by the
+;; <parameter>optimize</parameter> declaration; other declarations and variables
+;; control more specific aspects of compilation.
+;;
+;; \begin{comment}
+;; * The Optimize Declaration::
+;; * The Optimize-Interface Declaration::
+;; \end{comment}
+;;
+;; %%\node The Optimize Declaration, The Optimize-Interface Declaration, Compiler Policy, Compiler Policy
+;; \subsection{The Optimize Declaration}
+;; \label{optimize-declaration}
+;; \cindex{optimize declaration}
+;; \cpsubindex{declarations}{\code{optimize}}
+;;
+;; The \code{optimize} declaration recognizes six different
+;; \var{qualities}.  The qualities are conceptually independent aspects
+;; of program performance.  In reality, increasing one quality tends to
+;; have adverse effects on other qualities.  The compiler compares the
+;; relative values of qualities when it needs to make a trade-off; i.e.,
+;; if \code{speed} is greater than \code{safety}, then improve speed at
+;; the cost of safety.
+;;
+;; The default for all qualities (except \code{debug}) is \code{1}.
+;; Whenever qualities are equal, ties are broken according to a broad
+;; idea of what a good default environment is supposed to be.  Generally
+;; this downplays \code{speed}, \code{compile-speed} and \code{space} in
+;; favor of \code{safety} and \code{debug}.  Novice and casual users
+;; should stick to the default policy.  Advanced users often want to
+;; improve speed and memory usage at the cost of safety and
+;; debuggability.
+;;
+;; If the value for a quality is \code{0} or \code{3}, then it may have a
+;; special interpretation.  A value of \code{0} means ``totally
+;; unimportant'', and a \code{3} means ``ultimately important.''  These
+;; extreme optimization values enable ``heroic'' compilation strategies
+;; that are not always desirable and sometimes self-defeating.
+;; Specifying more than one quality as \code{3} is not desirable, since
+;; it doesn't tell the compiler which quality is most important.
+;;
+;;
+;; These are the optimization qualities:
+;; \begin{Lentry}
+;;
+;; \item[\code{speed}] \cindex{speed optimization quality}How fast the
+;;   program should is run.  \code{speed 3} enables some optimizations
+;;   that hurt debuggability.
+;;
+;; \item[\code{compilation-speed}] \cindex{compilation-speed optimization
+;;     quality}How fast the compiler should run.  Note that increasing
+;;   this above \code{safety} weakens type checking.
+;;
+;; \item[\code{space}] \cindex{space optimization quality}How much space
+;;   the compiled code should take up.  Inline expansion is mostly
+;;   inhibited when \code{space} is greater than \code{speed}.  A value
+;;   of \code{0} enables indiscriminate inline expansion.  Wide use of a
+;;   \code{0} value is not recommended, as it may waste so much space
+;;   that run time is slowed.  \xlref{inline-expansion} for a discussion
+;;   of inline expansion.
+;;
+;; \item[\code{debug}] \cindex{debug optimization quality}How debuggable
+;;   the program should be.  The quality is treated differently from the
+;;   other qualities: each value indicates a particular level of debugger
+;;   information; it is not compared with the other qualities.
+;;   \xlref{debugger-policy} for more details.
+;;
+;; \item[\code{safety}] \cindex{safety optimization quality}How much
+;;   error checking should be done.  If \code{speed}, \code{space} or
+;;   \code{compilation-speed} is more important than \code{safety}, then
+;;   type checking is weakened (\pxlref{weakened-type-checks}).  If
+;;   \code{safety} if \code{0}, then no run time error checking is done.
+;;   In addition to suppressing type checks, \code{0} also suppresses
+;;   argument count checking, unbound-symbol checking and array bounds
+;;   checks.
+;;   ... and checking of tag existence in RETURN-FROM and GO.
+;;
+;; \item[\code{extensions:inhibit-warnings}] \cindex{inhibit-warnings
+;;     optimization quality}This is a CMU extension that determines how
+;;   little (or how much) diagnostic output should be printed during
+;;   compilation.  This quality is compared to other qualities to
+;;   determine whether to print style notes and warnings concerning those
+;;   qualities.  If \code{speed} is greater than \code{inhibit-warnings},
+;;   then notes about how to improve speed will be printed, etc.  The
+;;   default value is \code{1}, so raising the value for any standard
+;;   quality above its default enables notes for that quality.  If
+;;   \code{inhibit-warnings} is \code{3}, then all notes and most
+;;   non-serious warnings are inhibited.  This is useful with
+;;   \code{declare} to suppress warnings about unavoidable problems.
+;; \end{Lentry}
+;;
+;; %%\node The Optimize-Interface Declaration,  , The Optimize Declaration, Compiler Policy
+;; \subsection{The Optimize-Interface Declaration}
+;; \label{optimize-interface-declaration}
+;; \cindex{optimize-interface declaration}
+;; \cpsubindex{declarations}{\code{optimize-interface}}
+;;
+;; The \code{extensions:optimize-interface} declaration is identical in
+;; syntax to the \code{optimize} declaration, but it specifies the policy
+;; used during compilation of code the compiler automatically generates
+;; to check the number and type of arguments supplied to a function.  It
+;; is useful to specify this policy separately, since even thoroughly
+;; debugged functions are vulnerable to being passed the wrong arguments.
+;; The \code{optimize-interface} declaration can specify that arguments
+;; should be checked even when the general \code{optimize} policy is
+;; unsafe.
+;;
+;; Note that this argument checking is the checking of user-supplied
+;; arguments to any functions defined within the scope of the
+;; declaration, \code{not} the checking of arguments to \llisp{}
+;; primitives that appear in those definitions.
+;;
+;; The idea behind this declaration is that it allows the definition of
+;; functions that appear fully safe to other callers, but that do no
+;; internal error checking.  Of course, it is possible that arguments may
+;; be invalid in ways other than having incorrect type.  Functions
+;; compiled unsafely must still protect themselves against things like
+;; user-supplied array indices that are out of bounds and improper lists.
+;; See also the \kwd{context-declarations} option to
+;; \macref{with-compilation-unit}.
+;;
+;; (end of section on compiler policy)
+
+(defsection @compiler-errors (:title "Compiler Errors")
+  (@type-errors-at-compile-time section)
+  (@errors-during-macroexpansion section)
+  (@read-errors section))
+
+(defsection @type-errors-at-compile-time
+    (:title "Type Errors at Compile Time"
+     :concepts (("compile-time" "type error")
+                ("type error," "compile-time")))
+  "If the compiler can prove at compile time that some portion of the
+  program cannot be executed without a type error, then it will give a
+  warning at compile time.
+
+  It is possible that the offending code would never actually be
+  executed at run-time due to some higher level consistency constraint
+  unknown to the compiler, so a type warning doesn't always indicate an
+  incorrect program.
+
+  For example, consider this code fragment:
+
+      (defun raz (foo)
+        (let ((x (case foo
+                    (:this 13)
+                    (:that 9)
+                    (:the-other 42))))
+          (declare (fixnum x))
+          (foo x)))
+
+  Compilation produces this warning:
+
+      ; in: DEFUN RAZ
+      ;     (CASE FOO (:THIS 13) (:THAT 9) (:THE-OTHER 42))
+      ; --> LET COND IF COND IF COND IF
+      ; ==>
+      ;   (COND)
+      ;
+      ; caught WARNING:
+      ;   This is not a FIXNUM:
+      ;   NIL
+
+  In this case, the warning means that if `FOO` isn't any of `:THIS`,
+  `:THAT` or `:THE-OTHER`, then `x` will be initialized to NIL, which
+  the FIXNUM declaration makes illegal. The warning will go away if
+  ECASE is used instead of CASE, or if `:THE-OTHER` is changed to T.
+
+  This sort of spurious type warning happens moderately often in the
+  expansion of complex macros and in inline functions. In such cases,
+  there may be dead code that is impossible to correctly execute. The
+  compiler can't always prove this code is dead (could never be
+  executed), so it compiles the erroneous code (which will always signal
+  an error if it is executed) and gives a warning.")
+
+(defsection @errors-during-macroexpansion
+    (:title "Errors During Macroexpansion"
+     :concepts (("macroexpansion," "errors during")))
+  "The compiler handles errors that happen during macroexpansion, turning
+  them into compiler errors. If you want to debug the error (to debug
+  a macro), you can set *BREAK-ON-SIGNALS* to ERROR. For example, this
+  definition:
+
+      (defun foo (e l)
+        (do ((current l (cdr current))
+             ((atom current) nil))
+            (when (eq (car current) e) (return current))))
+
+  gives this error:
+
+      ; in: DEFUN FOO
+      ;     (DO ((CURRENT L (CDR CURRENT))
+      ;        ((ATOM CURRENT) NIL))
+      ;       (WHEN (EQ (CAR CURRENT) E) (RETURN CURRENT)))
+      ;
+      ; caught ERROR:
+      ;   (in macroexpansion of (DO # #))
+      ;   (hint: For more precise location, try *BREAK-ON-SIGNALS*.)
+      ;   DO step variable is not a symbol: (ATOM CURRENT)")
+
+(defsection @read-errors (:title "Read Errors"
+                          :concepts (("compiler" "read error")
+                                     ("read error," "compiler")))
+  "SBCL's compiler does not attempt to recover from read errors when
+  reading a source file, but instead just reports the offending
+  character position and gives up on the entire source file.")
+
+(defsection @open-coding-and-inline-expansion
+    (:title "Open Coding and Inline Expansion"
+     :concepts ("open-coding"
+                ("inline" "expansion")
+                ("static" "functions")))
+  "Since Common Lisp forbids the redefinition of standard functions, the
+  compiler can have special knowledge of these standard functions
+  embedded in it. This special knowledge is used in various ways (open
+  coding, inline expansion, source transformation), but the implications
+  to the user are basically the same:
+
+  - Attempts to redefine standard functions may be frustrated, since
+    the function may never be called. Although it is technically
+    illegal to redefine standard functions, users sometimes want to
+    implicitly redefine these functions when they are debugging using
+    the TRACE macro. Special-casing of standard functions can be
+    inhibited using the NOTINLINE declaration, but even then some
+    phases of analysis such as type inferencing are applied by the
+    compiler.
+
+  - The compiler can have multiple alternate implementations of
+    standard functions that implement different trade-offs of speed,
+    space and safety. This selection is based on the @COMPILER-POLICY.
+
+  When a function call is _open coded_, inline code whose effect is
+  equivalent to the function call is substituted for that function
+  call. When a function call is _closed coded_, it is usually left as
+  is, although it might be turned into a call to a different function
+  with different arguments. As an example, if NTHCDR were to be open
+  coded, then
+
+      (nthcdr 4 foobar)
+
+  might turn into
+
+      (cdr (cdr (cdr (cdr foobar))))
+
+  or even
+
+      (do ((i 0 (1+ i))
+        (list foobar (cdr foobar)))
+        ((= i 4) list))
+
+  If NTH is closed coded, then
+
+      (nth x l)
+
+  might stay the same, or turn into something like
+
+      (car (nthcdr x l))
+
+  In general, open coding sacrifices space for speed, but some functions
+  (such as CAR) are so simple that they are always open-coded. Even
+  when not open-coded, a call to a standard function may be
+  transformed into a different function call (as in the last example)
+  or compiled as _static call_. Static function call uses a more
+  efficient calling convention that forbids redefinition.")
+
+(defsection @interpreter (:title "Interpreter"
+                          :concepts ("interpreter"))
+  "By default SBCL implements EVAL by calling the native code
+  compiler.
+
+  SBCL also includes an interpreter for use in special cases where
+  using the compiler is undesirable, for example due to compilation
+  overhead. Unlike in some other Lisp implementations, in SBCL
+  interpreted code is not safer or more debuggable than compiled code."
+  (sb-ext:*evaluator-mode* variable))
+
+(defsection @advanced-compiler-use-and-efficiency-hints
+    (:title "Advanced Compiler Use and Efficiency Hints")
+  "For more advanced usages of the compiler, please see the chapter of the
+  same name in the CMUCL manual. Many aspects of the compiler have stayed
+  exactly the same, and there is a much more detailed explanation of the
+  compiler's behavior and how to maximally optimize code in their
+  manual. In particular, while SBCL no longer supports byte-code
+  compilation, it does support CMUCL's block compilation facility allowing
+  whole program optimization and increased use of the local call
+  convention.
+
+  Unlike CMUCL, SBCL is able to open-code forward-referenced type
+  tests while block compiling. This helps for mutually referential
+  DEFSTRUCTs in particular.")

contrib/sb-manual/doc/concepts.lisp

@@ -0,0 +1,108 @@
+(in-package :sb-manual)
+
+(define-concept @interning-symbols (:title "interning symbols"
+                                    :keys (("interning" "symbols")
+                                           ("symbols," "interning"))))
+
+(define-concept @package-lock (:title "package lock"
+                               :keys (("package" "lock")
+                                      ("lock," "package"))))
+
+(define-concept @unicode (:title "Unicode" :keys ("Unicode")))
+
+(define-concept @nfkc (:title "NFKC"
+                       :keys ("NFKC"
+                              "normalization form compatibility composition")))
+
+(define-concept ~unbound-slot (:keys (("unbound" "slot")
+                                      ("slot," "unbound"))))
+
+(define-concept ~character-name (:keys (("character" "name")
+                                        ("name" "of character"))))
+
+(define-concept @hash-table (:title "hash table"
+                             :keys (("hash" "table"))))
+
+(define-concept @actual-source (:title "actual source"
+                                :keys (("actual" "source")
+                                       ("source," "actual"))))
+
+(define-concept @original-source (:title "original source"
+                                  :keys (("original" "source")
+                                         ("source," "original"))))
+
+(define-concept @processing-path (:title "processing path"
+                                  :keys (("processing" "path"))))
+
+(define-concept @macroexpansion (:title "macroexpansion"
+                                 :keys ("macroexpansion")))
+
+(define-concept ~source-transform (:keys (("source" "transform"))))
+
+(define-concept ~safety (:keys (("safety," "optimization quality")
+                                ("optimization quality" "safety"))))
+
+(define-concept @safety (:title "safety" :keys (~safety)))
+
+(define-concept ~debug (:keys (("debug," "optimization quality")
+                               ("optimization quality" "debug"))))
+
+(define-concept @debug (:title "debug" :keys (~debug)))
+
+(define-concept @tail-recursion (:title "tail recursion"
+                                 :keys (("tail" "recursion")
+                                        ("recursion," "tail"))))
+
+(define-concept @tail-recursive (:title "tail recursive"
+                                 :keys (@tail-recursion)))
+
+(define-concept @interrupt (:title "interrupt"
+                            :keys ("interrupt")))
+
+(define-concept ~run-time-error (:keys (("run-time" "error")
+                                        ("error," "run-time"))))
+
+(define-concept @basic-block (:title "basic block"
+                              :keys ("basic block"
+                                     ("block," "basic"))))
+
+(define-concept @block-start (:title "block start"
+                              :keys (("block," "start location"))))
+
+(define-concept @semi-inline (:title "semi inline"
+                              :keys (("inline," "semi")
+                                     ("semi-inline"))))
+
+(define-concept @external-format (:title "external format"
+                              :keys (("external" "format")
+                                     ("format," "external"))))
+
+(define-concept @generational-gc
+    (:title "generational GC"
+     :keys (("garbage collector," "generational")
+            ("generational" "garbage collector"))))
+
+(define-concept @conservative-gc
+    (:title "conservative GC"
+     :keys (("garbage collector," "conservative")
+            ("conservative" "garbage collector"))))
+
+(define-concept @declaration (:title "declaration"
+                              :keys ("declaration")))
+
+(define-concept @logical-pathname (:title "logical pathname"
+                                   :keys (("logical" "pathname")
+                                          ("pathname," "logical"))))
+
+(define-concept @ldb (:title "LDB" :keys ("LDB")))
+
+(define-concept ~disabling-ldb (:title "disabling LDB"
+                                :keys (("disabling" "LDB")
+                                       ("LDB," "disabling"))))
+
+(define-concept ~enabling-ldb (:title "enabling LDB"
+                                :keys (("enabling" "LDB")
+                                       ("LDB," "enabling"))))
+(define-concept ~repl (:keys ("Read-Eval-Print Loop" "REPL")))
+
+(define-concept @repl (:title "REPL" :keys (~repl)))

contrib/sb-manual/doc/contrib-modules.lisp

@@ -0,0 +1,29 @@
+(in-package :sb-manual)
+
+(defsection @contributed-modules (:title "Contributed Modules")
+  "SBCL comes with a number of modules that are not part of the core
+  system. These are loaded via `(REQUIRE :<MODULENAME>)`
+  (see @CUSTOMIZATION-HOOKS-FOR-USERS). This section contains
+  documentation (or pointers to documentation) for some of the
+  contributed modules."
+  #+#.(sb-manual::package-exists-p/reader '#:sb-aclrepl)
+  (@sb-aclrepl section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-concurrency)
+  (@sb-concurrency section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-cover)
+  (@sb-cover section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-grovel)
+  (@sb-grovel section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-introspect)
+  (@sb-introspect section)
+  (@sb-manual section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-md5)
+  (@sb-md5 section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-posix)
+  (@sb-posix section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-queue)
+  (@sb-queue section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-rotate-byte)
+  (@sb-rotate-byte section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-simd)
+  (@sb-simd section))

contrib/sb-manual/doc/debugger.lisp

@@ -0,0 +1,896 @@
+(in-package :sb-manual)
+
+(defsection @debugger (:title "Debugger"
+                       :concepts ("debugger"))
+  "This chapter documents the debugging facilities of SBCL, including
+  the debugger, single-stepper and TRACE, and the effect of `(OPTIMIZE
+  DEBUG)` declarations."
+  (@debugger-entry section)
+  (@debugger-command-loop section)
+  (@stack-frames section)
+  (@variable-access section)
+  (@source-location-printing section)
+  (@debugger-policy-control section)
+  (@exiting-commands section)
+  (@information-commands section)
+  (@breakpoint-commands section)
+  (@function-tracing section)
+  (@single-stepping section)
+  (@enabling-and-disabling-the-debugger section))
+
+(defsection @debugger-entry (:title "Debugger Entry")
+  (@debugger-banner section)
+  (@debugger-invocation section))
+
+(defsection @debugger-banner (:title "Debugger Banner")
+  "When you enter the debugger, it looks something like this:
+
+      debugger invoked on a TYPE-ERROR in thread 11184:
+        The value 3 is not of type LIST.
+
+      You can type HELP for debugger help, or (SB-EXT:QUIT) to exit from SBCL.
+
+      restarts (invokable by number or by possibly-abbreviated name):
+        0: [ABORT   ] Reduce debugger level (leaving debugger, returning to toplevel).
+        1: [TOPLEVEL] Restart at toplevel READ/EVAL/PRINT loop.
+      (CAR 1 3)
+      0]
+
+  The first group of lines describe what the error was that put us in
+  the debugger. In this case CAR was called on `3`, causing a
+  TYPE-ERROR.
+
+  This is followed by the \"beginner help line\", which appears only
+  if SB-DEBUG:*DEBUG-BEGINNER-HELP-P* is true (default).
+
+  Next comes a listing of the active restart names, along with their
+  descriptions -- the ways we can restart execution after this error.
+  In this case, both options return to top-level. Restarts can be
+  selected by entering the corresponding number or name.
+
+  The current frame appears right underneath the restarts, immediately
+  followed by the debugger prompt.")
+
+(defsection @debugger-invocation (:title "Debugger Invocation")
+  "The debugger is invoked when:
+
+  - ERROR is called, and the condition it signals is not handled.
+
+  - BREAK is called, or SIGNAL is called with a condition that matches
+    the current *BREAK-ON-SIGNALS*.
+
+  - The debugger is explicitly entered with the INVOKE-DEBUGGER
+    function.
+
+  When the debugger is invoked by a condition, ANSI mandates that the
+  value of *DEBUGGER-HOOK*, if any, be called with two arguments: the
+  condition that caused the debugger to be invoked and the previous
+  value of *DEBUGGER-HOOK*. When this happens, *DEBUGGER-HOOK* is
+  bound to NIL to prevent recursive errors. However, ANSI also
+  mandates that *DEBUGGER-HOOK* not be invoked when the debugger is to
+  be entered by the BREAK function. For users who wish to provide an
+  alternate debugger interface (and thus catch BREAK entries into the
+  debugger), SBCL provides SB-EXT:*INVOKE-DEBUGGER-HOOK*, which is
+  invoked during any entry into the debugger."
+  ;; When Swank is loaded, it sets this variable.
+  (sb-ext:*invoke-debugger-hook* (variable nil)))
+
+(defsection @debugger-command-loop (:title "Debugger Command Loop")
+  "The debugger is an interactive read-eval-print loop much like the
+  normal top level, but some symbols are interpreted as debugger
+  commands instead of being evaluated. A debugger command starts with
+  the symbol name of the command, possibly followed by some arguments
+  on the same line. Some commands prompt for additional input.
+  Debugger commands can be abbreviated by any unambiguous prefix:
+  `help` can be typed as `h`, `he`, etc.
+
+  The package is not significant in debugger commands; any symbol with
+  the name of a debugger command will work. If you want to show the
+  value of a variable that happens also to be the name of a debugger
+  command you can wrap the variable in a PROGN to hide it from
+  the command loop.
+
+  The debugger prompt is `<frame>]`, where `<frame>` is the number of
+  the current frame. Frames are numbered starting from zero at the
+  top (most recent call), increasing down to the bottom. The current
+  frame is the frame that commands refer to.
+
+  It is possible to override the normal printing behaviour in the
+  debugger by using the SB-EXT:*DEBUG-PRINT-VARIABLE-ALIST*."
+  (sb-ext:*debug-print-variable-alist* variable))
+
+(defsection @stack-frames (:title "Stack Frames"
+                           :concepts ("stack frame"))
+  "A _stack frame_ is the run-time representation of a call to a
+  function; the frame stores the state that a function needs to
+  remember what it is doing. Frames have:
+
+  - _Variables_ (see @VARIABLE-ACCESS), which are the values being
+    operated on.
+
+  - _Arguments_ to the call (which are really just particularly
+    interesting variables).
+
+  - A current source location (@SOURCE-LOCATION-PRINTING), which is
+    the place in the program where the function was running when it
+    stopped to call another function, or because of an interrupt or
+    error."
+  (@stack-motion section)
+  (@how-arguments-are-printed section)
+  (@function-names section)
+  (@debug-tail-recursion section)
+  (@unknown-locations-and-interrupts section))
+
+(defsection @stack-motion (:title "Stack Motion")
+  "These commands move to a new stack frame and print the name of the
+  function and the values of its arguments in the style of a Lisp
+  function call:
+
+  - `up`: Move up to the next higher frame. More recent function calls
+     are considered to be higher on the stack.
+
+  - `down`: Move down to the next lower frame.
+
+  - `top`: Move to the highest frame, that is, the frame where the
+    debugger was entered.
+
+  - `bottom`: Move to the lowest frame.
+
+  - `frame [<n>]`: Move to the frame with the specified number.
+    Prompts for the number if not supplied. The frame with number 0 is
+    the frame where the debugger was entered.")
+
+(defsection @how-arguments-are-printed (:title "How Arguments are Printed")
+  "A frame is printed to look like a function call, but with the actual
+  argument values in the argument positions.  So the frame for this call
+  in the source:
+
+      (myfun (+ 3 4) 'a)
+
+  would look like this:
+
+      (MYFUN 7 A)
+
+  All keyword and optional arguments are displayed with their actual
+  values; if the corresponding argument was not supplied, the value will
+  be the default.  So this call:
+
+      (subseq \"foo\" 1)
+
+  would look like this:
+
+      (SUBSEQ \"foo\" 1 3)
+
+  And this call:
+
+      (string-upcase \"test case\")
+
+  would look like this:
+
+      (STRING-UPCASE \"test case\" :START 0 :END NIL)
+
+  The arguments to a function call are displayed by accessing the
+  argument variables. Although those variables are initialized to the
+  actual argument values, they can be set inside the function; in this
+  case the new value will be displayed.
+
+  &REST arguments are handled somewhat differently. The value of the
+  rest argument variable is displayed as the spread-out arguments to
+  the call, so:
+
+      (format t \"~A is a ~A.\" \"This\" 'test)
+
+  would look like this:
+
+      (FORMAT T \"~A is a ~A.\" \"This\" 'TEST)
+
+  Rest arguments cause an exception to the normal display of keyword
+  arguments in functions that have both &REST and &KEY arguments. In
+  this case, the keyword argument variables are not displayed at all;
+  the rest arg is displayed instead. So for these functions, only the
+  keywords actually supplied will be shown, and the values displayed
+  will be the argument values, not values of the
+  (possibly modified) variables.
+
+  If the variable for an argument is never referenced by the function,
+  it will be deleted. The variable value is then unavailable, so the
+  debugger prints `#<unused-arg>` instead of the value. Similarly, if
+  for any of a number of reasons the value of the variable is
+  unavailable or not known to be available (@VARIABLE-ACCESS), then
+  `#<unavailable-arg>` will be printed instead of the argument value.
+
+  Note that inline expansion and open-coding affect what frames are
+  present in the debugger, see @DEBUGGER-POLICY-CONTROL."
+  ;; FIXME: Link here to section about open coding once it exists.
+  )
+
+(defsection @function-names (:title "Function Names")
+  "If a function is defined by DEFUN it will appear in backtrace
+  by that name. Functions defined by LABELS and FLET will appear as
+  `(FLET <NAME>)` and `(LABELS <NAME>)` respectively. Anonymous
+  lambdas will appear as `(LAMBDA <LAMBDA-LIST>)`."
+  (@entry-point-details section))
+
+(defsection @entry-point-details
+    (:title "Entry Point Details"
+     :concepts (("external" "entry point")
+                ("entry point," "external")
+                ("block compilation," "debugger implications")
+                ("external," "stack frame kind")
+                ("optional," "stack frame kind")
+                ("cleanup," "stack frame kind")))
+  "Sometimes the compiler introduces new functions that are used to
+  implement a user function, but are not directly specified in the
+  source. This is mostly done for argument type and count checking.
+
+  With recursive or block compiled functions, an additional `external`
+  frame may appear before the frame representing the first call to the
+  recursive function or entry to the compiled block. This is a
+  consequence of the way the compiler works: there is nothing odd with
+  your program. You may also see `cleanup` frames during the execution
+  of UNWIND-PROTECT cleanup code, and `optional` for variable argument
+  entry points.")
+
+(defsection @debug-tail-recursion (:title "Debug Tail Recursion")
+  "The compiler is _properly tail recursive_. If a function call is
+  in a @TAIL-RECURSIVE position, the stack frame will be deallocated
+  _at the time of the call_, rather than after the call returns.
+  Consider this backtrace:
+
+      (BAR ...)
+      (FOO ...)
+
+  Because of tail recursion, it is not necessarily the case that `FOO`
+  directly called `BAR`. It may be that `FOO` called some other
+  function `FOO2`, which then called `BAR` tail-recursively, as in
+  this example:
+
+      (defun foo ()
+        ...
+        (foo2 ...)
+        ...)
+
+      (defun foo2 (...)
+        ...
+        (bar ...))
+
+      (defun bar (...)
+        ...)
+
+  Usually the elimination of tail-recursive frames makes debugging
+  more pleasant, since these frames are mostly uninformative. If there
+  is any doubt about how one function called another, it can usually
+  be eliminated by finding the source location in the calling frame.
+  See @SOURCE-LOCATION-PRINTING.
+
+  The elimination of tail-recursive frames can be prevented by
+  disabling tail-recursion optimization, which happens when the DEBUG
+  optimization quality is greater than 2. See
+  @DEBUGGER-POLICY-CONTROL."
+  ;; FIXME: reinstate this link once the chapter is in the manual. For
+  ;; a more thorough discussion of tail recursion, see @TAIL-RECURSION.
+  )
+
+(defsection @unknown-locations-and-interrupts
+    (:title "Unknown Locations and Interrupts"
+     :concepts (("unknown code location")
+                ("code location" "unknown")))
+  "The debugger operates using special debugging information attached to
+  the compiled code. This debug information tells the debugger what it
+  needs to know about the locations in the code where the debugger can
+  be invoked. If the debugger somehow encounters a location not
+  described in the debug information, then it is said to be _unknown_.
+  If the code location for a frame is unknown, then some variables may
+  be inaccessible, and the source location cannot be precisely
+  displayed.
+
+  There are three reasons why a code location could be unknown:
+
+  - There is inadequate debug information due to the value of the
+    DEBUG optimization quality. See @DEBUGGER-POLICY-CONTROL.
+
+  - The debugger was entered because of an @INTERRUPT such as `C-c`.
+
+  - A hardware error ~RUN-TIME-ERROR such as a bus error occurred in
+    code that was compiled unsafely due to the value of the SAFETY
+    optimization quality."
+  ;; FIXME: reinstate link when section on optimize qualities exists.
+  ;; @OPTIMIZE-DECLARATION.
+  "In the last two cases, the values of argument variables are
+  accessible, but may be incorrect. For more details on when variable
+  values are accessible, see @VARIABLE-VALUE-AVAILABILITY.
+
+  It is possible for an interrupt to happen when a function call or
+  return is in progress. The debugger may then flame out with some
+  obscure error or insist that the bottom of the stack has been
+  reached, when the real problem is that the current stack frame can't
+  be located. If this happens, return from the interrupt and try
+  again.")
+
+(defsection @variable-access (:title "Variable Access"
+                              :concepts (("debug" "variables")
+                                         ("variable," "debugger access")))
+  "There are two ways to access the current frame's local variables in
+  the debugger: `list-locals` and SB-DEBUG:VAR.
+
+  The debugger doesn't really understand lexical scoping; it has just
+  one namespace for all the variables in the current stack frame. If a
+  symbol is the name of multiple variables in the same function, then
+  the reference appears ambiguous, even though lexical scoping
+  specifies which value is visible at any given source location. If
+  the scopes of the two variables are not nested, then the debugger
+  can resolve the ambiguity by observing that only one variable is
+  accessible.
+
+  When there are ambiguous variables, the evaluator assigns each one a
+  small integer identifier. The SB-DEBUG:VAR function uses this
+  identifier to distinguish between ambiguous variables. The
+  `list-locals` command prints the identifier. In the following
+  example, there are two variables named `X`. The first one has
+  identifier 0 (which is not printed), the second one has identifier
+  1.
+
+      X  =  1
+      X#1  =  2
+
+  - `list-locals [<prefix>]`: This command prints the name and value
+    of all variables in the current frame whose name has the specified
+    `<prefix>`, which may be a string or a symbol. If no `<prefix>` is
+    given, then all available variables are printed. If a variable has
+    a potentially ambiguous name, then the name is printed with a
+    `#<identifier>` suffix, where `<identifier>` is the small integer
+    used to make the name unique."
+  (sb-debug:var function)
+  (@variable-value-availability section)
+  (@note-on-lexical-variable-access section))
+
+(defsection @variable-value-availability
+    (:title "Variable Value Availability"
+     :concepts (("variable," "value availabilty in debugger")
+                ("debug variable," "value availabilty")))
+  "The value of a variable may be unavailable to the debugger in portions
+  of the program where Lisp says that the variable is defined. If a
+  variable value is not available, the debugger will not let you read
+  or write that variable. With one exception, the debugger will never
+  display an incorrect value for a variable. Rather than displaying
+  incorrect values, the debugger tells you the value is unavailable.
+
+  The one exception is this: if you interrupt (e.g. with `C-c`) or if
+  there is an unexpected hardware error such as a bus error (which
+  should only happen in unsafe code), then the values displayed for
+  arguments to the interrupted frame might be incorrect. This
+  exception applies only to the interrupted frame: any frame farther
+  down the stack will be fine.
+
+  > _Note_: Since the location of an interrupt or hardware error will
+  > always be an unknown location, non-argument variable values will
+  > never be available in the interrupted frame. See
+  > @UNKNOWN-LOCATIONS-AND-INTERRUPTS.)
+
+  The value of a variable may be unavailable for these reasons:
+
+  - The value of the @DEBUG optimization quality may have omitted
+    debug information needed to determine whether the variable is
+    available. Unless a variable is an argument, its value will only
+    be available when DEBUG is at least 2.
+
+  - The compiler did lifetime analysis and determined that the value
+    was no longer needed, even though its scope had not been exited.
+    Lifetime analysis is inhibited when the DEBUG optimization
+    quality is 3.
+
+  - The variable's name is an uninterned symbol (gensym). To save
+    space, the compiler only dumps debug information about uninterned
+    variables when the DEBUG optimization quality is 3.
+
+  - The frame's location is unknown (see
+    @UNKNOWN-LOCATIONS-AND-INTERRUPTS) because the debugger was
+    entered due to an interrupt or unexpected hardware error. Under
+    these conditions the values of arguments will be available, but
+    might be incorrect. This is the exception mentioned above.
+
+  - The variable (or the code referencing it) was optimized out of
+    existence. Variables with no reads are always optimized away. The
+    degree to which the compiler deletes variables will depend on the
+    value of the COMPILATION-SPEED optimization quality, but most
+    source-level optimizations are done under all compilation
+    policies.
+
+  - The variable is never set and its definition looks like
+
+          (LET ((var1 var2))
+             ...)
+
+      In this case, `VAR1` is substituted with `VAR2`.
+
+  - The variable is never set and is referenced exactly once. In this
+    case, the reference is substituted with the variable initial
+    value.
+
+  Since it is especially useful to be able to get the arguments to a
+  function, argument variables are treated specially when the SPEED
+  optimization quality is less than 3 and the DEBUG quality is at
+  least 1. With this compilation policy, the values of argument
+  variables are almost always available everywhere in the function,
+  even at unknown locations. For non-argument variables, DEBUG must be
+  at least 2 for values to be available, and even then, values are
+  only available at known locations.")
+
+(defsection @note-on-lexical-variable-access
+    (:title "Note On Lexical Variable Access")
+  "When the debugger command loop establishes variable bindings for
+  available variables, these variable bindings have lexical scope and
+  dynamic extent. You can close over them, but such closures can't be
+  used as upward function arguments.
+
+  > _Note_: The variable bindings are actually created using the Lisp
+  > SYMBOL-MACROLET special form.
+
+  You can also set local variables using SETQ, but if the variable was
+  closed over in the original source and never set, then setting the
+  variable in the debugger may not change the value in all the
+  functions the variable is defined in. Another risk of setting
+  variables is that you may assign a value of a type that the compiler
+  proved the variable could never take on. This may result in bad
+  things happening.")
+
+(defsection @source-location-printing
+    (:title "Source Location Printing"
+     :concepts (("source location" "in debugger")))
+  "One of the debugger's capabilities is source level debugging of
+  compiled code.  These commands display the source location for the
+  current frame:
+
+  - `source [<context>]`: This command displays the file that the
+    current frame's function was defined from (if it was defined from
+    a file), and then the source form responsible for generating the
+    code that the current frame was executing. If `<context>` is
+    specified, then it is an integer specifying the number of
+    enclosing levels of list structure to print.
+
+  The source form for a location in the code is the innermost list
+  present in the original source that encloses the form responsible
+  for generating that code. If the actual source form is not a list,
+  then some enclosing list will be printed. For example, if the source
+  form was a reference to the variable `*SOME-RANDOM-SPECIAL*`, then
+  the innermost enclosing evaluated form will be printed. Here are
+  some possible enclosing forms:
+
+      (let ((a *some-random-special*))
+        ...)
+
+      (+ *some-random-special* ...)
+
+  If the code at a location was generated from the expansion of a
+  macro or a source-level compiler optimization, then the form in the
+  original source that expanded into that code will be printed.
+  Suppose the file `/usr/me/mystuff.lisp` looked like this:
+
+      (defmacro mymac ()
+        '(myfun))
+
+      (defun foo ()
+        (mymac)
+        ...)
+
+  If `FOO` has called `MYFUN`, and is waiting for it to return, then
+  the `source` command would print:
+
+      ; File: /usr/me/mystuff.lisp
+
+      (MYMAC)
+
+  Note that the macro use was printed, not the actual function call form,
+  `(MYFUN)`.
+
+  If enclosing source is printed by giving an argument to `source` or
+  `vsource`, then the actual source form is marked by wrapping it in a
+  list whose first element is `#:***HERE***`. In the previous example,
+  `source 1` would print:
+
+      ; File: /usr/me/mystuff.lisp
+
+      (DEFUN FOO ()
+        (#:***HERE***
+         (MYMAC))
+        ...)"
+  (@how-the-source-is-found section)
+  (@source-location-availability section))
+
+(defsection @how-the-source-is-found (:title "How the Source is Found")
+  "If the code was defined from Lisp by COMPILE or EVAL, then the source
+  can always be reliably located. If the code was defined from a FASL
+  file created by COMPILE-FILE, then the debugger gets the source
+  forms it prints by reading them from the original source file. This
+  is a potential problem, since the source file might have moved or
+  changed since the time it was compiled.
+
+  The source file is opened using the TRUENAME of the source file
+  pathname originally given to the compiler. This is an absolute
+  pathname with all logical names and symbolic links expanded. If the
+  file can't be located using this name, then the debugger gives up
+  and signals an error.
+
+  If the source file can be found, but has been modified since the time it was
+  compiled, the debugger prints this warning:
+
+      ; File has been modified since compilation:
+      ;   <filename>
+      ; Using form offset instead of character position.
+
+  where `<filename>` is the name of the source file. It then proceeds
+  using a robust but not foolproof heuristic for locating the source.
+  This heuristic works if:
+
+  - No top-level forms before the top-level form containing the source
+    have been added or deleted, and
+
+  - the top-level form containing the source has not been modified
+    much. (More precisely, none of the list forms beginning before the
+    source form have been added or deleted.)
+
+  If the heuristic doesn't work, the displayed source will be wrong,
+  but will probably be near the actual source. If the \"shape\" of the
+  top-level form in the source file is too different from the original
+  form, then an error will be signaled. When the heuristic is used,
+  the source location commands are noticeably slowed.
+
+  Source location printing can also be confused if (after the source
+  was compiled) a read-macro you used in the code was redefined to
+  expand into something different, or if a read-macro ever returns the
+  same EQ list twice. If you don't define read macros and don't use
+  `##` in perverted ways, you don't need to worry about this.")
+
+(defsection @source-location-availability
+    (:title "Source Location Availability")
+  "Source location information is only available when the @DEBUG
+  optimization quality is at least 2. If source location information
+  is unavailable, the source commands will give an error message.
+
+  If source location information is available, but the source location
+  is unknown because of an interrupt or unexpected hardware error
+  (see @UNKNOWN-LOCATIONS-AND-INTERRUPTS), then the command will
+  print
+
+      Unknown location: using block start.
+
+  and then proceed to print the source location for the start of the
+  _@BASIC-BLOCK_ enclosing the code location. It's a bit complicated to
+  explain exactly what a basic block is, but here are some properties
+  of the block start location:
+
+  - The @BLOCK-START location may be the same as the true location.
+
+  - The block start location will never be later in the program's flow
+    of control than the true location.
+
+  - No conditional control structures (such as IF, COND, OR) will
+    intervene between the block start and the true location (but note
+    that some conditionals present in the original source could be
+    optimized away.) Function calls _do not_ end basic blocks.
+
+  - The head of a loop will be the start of a block.
+
+  - The programming language concept of block structure and the Lisp
+    BLOCK special form are totally unrelated to the compiler's basic
+    block.
+
+  In other words, the true location lies between the printed location
+  and the next conditional (but watch out because the compiler may
+  have changed the program on you.)")
+
+(defsection @debugger-policy-control (:title "Debugger Policy Control"
+                                      :concepts (("debugger" "policy")
+                                                 ("policy," "debugger")))
+  "The compilation policy specified by OPTIMIZE declarations
+  affects the behavior seen in the debugger. The @DEBUG quality
+  directly affects the debugger by controlling the amount of debugger
+  information dumped. Other optimization qualities have indirect but
+  observable effects due to changes in the way compilation is done.
+
+  Unlike the other optimization qualities (which are compared in
+  relative value to evaluate tradeoffs), the DEBUG optimization
+  quality is directly translated to a level of debug information. This
+  absolute interpretation allows the user to count on a particular
+  amount of debug information being available even when the values of
+  the other qualities are changed during compilation. These are the
+  levels of debug information that correspond to the values of the
+  DEBUG quality:
+
+  - `0`: Only the function name and enough information to allow the
+    stack to be parsed.
+
+  - `> 0`: Any level greater than 0 gives level 0 plus all argument
+    variables. Values will only be accessible if the argument variable
+    is never set and SPEED is not 3. SBCL allows any real value for
+    optimization qualities. It may be useful to specify 0.5 to get
+    backtrace argument display without argument documentation.
+
+  - `1`: Level 1 provides argument documentation (printed argument
+    lists) and derived argument/result type information. This makes
+    DESCRIBE more informative, and allows the compiler to do
+    compile-time argument count and type checking for any calls
+    compiled at run-time. This is the default.
+
+  - `2`: Level 1 plus all interned local variables, source location
+    information, and lifetime information that tells the debugger when
+    arguments are available (even when SPEED is 3 or the argument is
+    set).
+
+  - `> 2`: Any level greater than 2 gives level 2 and in addition
+    disables tail-call optimization, so that the backtrace will
+    contain frames for all invoked functions, even those in tail
+    positions.
+
+  - `3`: Level 2 plus all uninterned variables. In addition, lifetime
+    analysis is disabled (even when SPEED is 3), ensuring that all
+    variable values are available at any known location within the
+    scope of the binding. This has a speed penalty in addition to the
+    obvious space penalty.
+
+  Inlining of local functions is inhibited so that they may be TRACEd.
+
+  - `> (MAX SPEED SPACE)`: If DEBUG is greater than both SPEED and
+    SPACE, the command `return` can be used to continue execution by
+    returning a value from the current stack frame.
+
+  - `> (MAX SPEED SPACE COMPILATION-SPEED)`: If DEBUG is greater than
+    all of SPEED, SPACE and COMPILATION-SPEED the code will be
+    steppable (see @SINGLE-STEPPING).
+
+  As you can see, if the SPEED quality is 3, debugger performance is
+  degraded. This effect comes from the elimination of argument
+  variable special-casing (see @VARIABLE-VALUE-AVAILABILITY). Some
+  degree of speed/debuggability tradeoff is unavoidable, but the
+  effect is not too drastic when DEBUG is at least 2.
+
+  In addition to INLINE and NOTINLINE declarations, the relative
+  values of the SPEED and SPACE qualities also change whether
+  functions are inline expanded. If a function is inline expanded,
+  then there will be no frame to represent the call, and the arguments
+  will be treated like any other local variable. Functions may also be
+  _@SEMI-INLINE_, in which case there is a frame to represent the
+  call, but the call is to an optimized local version of the function,
+  not to the original function."
+  ;; FIXME: link to section about inline expansion when it exists
+  ;; (@INLINE-EXPANSION).
+  )
+
+(defsection @exiting-commands (:title "Exiting Commands")
+  "These commands get you out of the debugger.
+
+  - `toplevel`: Throw to top level.
+
+  - `restart [<n>]`: Invoke the `<n>`th restart case as displayed by
+    the `error` command. If `<n>` is not specified, the available
+    restart cases are reported.
+
+  - `\\continue`: Call CONTINUE on the condition given to DEBUG. If
+    there is no restart case named CONTINUE, then an error is
+    signaled.
+
+  - `\\abort`: Call ABORT on the condition given to DEBUG. This is
+    useful for popping debug command loop levels or aborting to top
+    level, as the case may be.
+
+  - `return <value>`: Return `VALUE` from the current stack frame.
+    This command is available when the DEBUG optimization quality is
+    greater than both SPEED and SPACE. Care must be taken that the
+    value is of the same type as SBCL expects the stack frame to
+    return.
+
+  - `restart-frame`: Restart execution of the current stack frame.
+    This command is available when the DEBUG optimization quality is
+    greater than both SPEED and SPACE and when the frame is for a
+    global function. If the function is redefined in the debugger
+    before the frame is restarted, the new function will be used.")
+
+(defsection @information-commands (:title "Information Commands")
+  "Most of these commands print information about the current frame or
+  function, but a few show general information.
+
+  - `help` or `?`: Display a synopsis of debugger commands.
+
+  - `\\describe`: Call DESCRIBE on the current function and displays the
+    number of local variables.
+
+  - `\\print`: Display the current function call as it would be
+    displayed by moving to this frame.
+
+  - `\\error`: Print the condition given to INVOKE-DEBUGGER and the
+    active proceed cases.
+
+  - `backtrace [<n>]`: Display all the frames from the current to the
+    bottom. Only shows `<n>` frames if specified. The printing is
+    controlled by SB-DEBUG:*DEBUG-PRINT-VARIABLE-ALIST*.")
+
+(defsection @breakpoint-commands (:title "Breakpoint Commands"
+                                  :concepts ("breakpoint"))
+  "SBCL supports setting of breakpoints inside compiled functions and
+  stepping of compiled code. Breakpoints can only be set at known
+  locations (see @UNKNOWN-LOCATIONS-AND-INTERRUPTS), so these commands
+  are largely useless unless the DEBUG optimize quality is at least
+  2 (see @DEBUGGER-POLICY-CONTROL). These commands manipulate
+  breakpoints:
+
+  - `breakpoint <location> [<option> <value>]*`: Set a breakpoint in
+    some function. `<location>` may be an integer code location
+    number (as displayed by `list-locations`) or a keyword. The
+    keyword can be used to indicate setting a breakpoint at the
+    function start (:START, `:S`) or function end (:END, `:E`). The
+    `breakpoint` command has :CONDITION, :BREAK, :PRINT and :FUNCTION
+    options which work similarly to the TRACE options.
+
+  - `list-locations [<function>]` or `ll [<function>]`: List all the
+    code locations in the current frame's function, or in `<function>`
+    if it is supplied. The display format is the code location number,
+    a colon and then the source form for that location:
+
+          3: (1- N)
+
+      If consecutive locations have the same source, then a numeric
+      range like `3-5:` will be printed. For example, a default
+      function call has a known location both immediately before and
+      after the call, which would result in two code locations with
+      the same source. The listed function becomes the new default
+      function for breakpoint setting (via the `breakpoint`) command.
+
+  - `list-breakpoints` or `lb`: List all currently active breakpoints
+    with their breakpoint number.
+
+  - `delete-breakpoint [<number>]` or `db [<number>]`: Delete a
+    breakpoint specified by its breakpoint number. If no number is
+    specified, delete all breakpoints.
+
+  - `step*`: Step to the next possible breakpoint location in the
+    current function. This always steps over function calls, instead
+    of stepping into them."
+  (@breakpoint-example section))
+
+(defsection @breakpoint-example (:title "Breakpoint Example")
+  "Consider this definition of the factorial function:
+
+      (defun ! (n)
+        (if (zerop n)
+            1
+            (* n (! (1- n)))))
+
+  This debugger session demonstrates the use of breakpoints:
+
+      * (break)  ; invoke debugger
+
+      debugger invoked on a SIMPLE-CONDITION in thread 11184: break
+
+      restarts (invokable by number or by possibly-abbreviated name):
+        0: [CONTINUE] Return from BREAK.
+        1: [ABORT   ] Reduce debugger level (leaving debugger, returning to toplevel).
+        2: [TOPLEVEL] Restart at toplevel READ/EVAL/PRINT loop.
+      (\"varargs entry for top level local call BREAK\" \"break\")
+      0] ll #'!
+
+      0-1: (SB-INT:NAMED-LAMBDA ! (N) (BLOCK ! (IF (ZEROP N) 1 (* N (! #)))))
+      2: (BLOCK ! (IF (ZEROP N) 1 (* N (! (1- N)))))
+      3: (ZEROP N)
+      4: (* N (! (1- N)))
+      5: (1- N)
+      6: (! (1- N))
+      7-8: (* N (! (1- N)))
+      9-10: (IF (ZEROP N) 1 (* N (! (1- N))))
+      0] br 4
+
+      (* N (! (1- N)))
+      1: 4 in !
+      added
+      0] toplevel
+
+      > (! 10) ; Call the function
+
+      *Breakpoint hit*
+
+      Restarts:
+        0: [CONTINUE] Return from BREAK.
+        1: [ABORT   ] Return to Top-Level.
+
+      Debug  (type H for help)
+
+      (! 10) ; We are now in first call (arg 10) before the multiply
+      Source: (* N (! (1- N)))
+      3] step*
+
+      *Step*
+
+      (! 10) ; We have finished evaluation of (1- n)
+      Source: (1- N)
+      3] step*
+
+      *Breakpoint hit*
+
+      Restarts:
+        0: [CONTINUE] Return from BREAK.
+        1: [ABORT   ] Return to Top-Level.
+
+      Debug  (type H for help)
+
+      (! 9) ; We hit the breakpoint in the recursive call
+      Source: (* N (! (1- N)))
+      3]
+
+  > _Note_: The `step*` command differs from the single stepping
+  > commands in that it also functions in compiled code which has not
+  > been compiled with stepping instrumentation. It simply steps to
+  > the next compiled code location. In the future, this form of
+  > stepping may be improved enough to subsume the instrumentation
+  > based stepping commands, which have much higher overhead.")
+
+(defsection @function-tracing (:title "Function Tracing"
+                               :concepts ("tracing"
+                                          ("function," "tracing")))
+  "The tracer causes selected functions to print their arguments and
+  their results whenever they are called.  Options allow conditional
+  printing of the trace information and conditional breakpoints on
+  function entry or exit.
+
+  In SBCL, tracing can be done either by temporarily redefining the
+  function name (encapsulation), or using breakpoints. When
+  breakpoints are used, the function object itself is destructively
+  modified to cause the tracing action. The advantage of using
+  breakpoints is that tracing works even when the function is
+  anonymously called via FUNCALL, that function object identity is
+  preserved, and that anonymous and local functions can also be
+  traced."
+  (trace macro)
+  "In the case of functions where the known return convention is used
+  to optimize, encapsulation may be necessary in order to make tracing
+  work at all. The symptom of this occurring is an error stating
+
+      Error in function FOO: :FUNCTION-END breakpoints are
+      currently unsupported for the known return convention.
+
+  in such cases we recommend using `(TRACE FOO :ENCAPSULATE t)`."
+  (untrace macro)
+  (sb-debug:*trace-indentation-step* variable)
+  (sb-debug:*max-trace-indentation* variable)
+  (sb-debug:*trace-encapsulate-default* variable)
+  (sb-debug:*trace-report-default* variable))
+
+(defsection @single-stepping (:title "Single Stepping"
+                              :concepts ("stepper"
+                                         "single-stepping"))
+  "SBCL includes an instrumentation based single-stepper for compiled
+  code, that can be invoked via the STEP macro, or from within the
+  debugger. See @DEBUGGER-POLICY-CONTROL, for details on enabling
+  stepping for compiled code.
+
+  The following debugger commands are used for controlling single stepping.
+
+  - `start`: Select the CONTINUE restart if one exists and starts
+    single stepping. None of the other single stepping commands can be
+    used before stepping has been started either by using `start` or
+    by using the standard STEP macro.
+
+  - `step`: Step into the current form. Stepping will be resumed when
+    the next form that has been compiled with stepper instrumentation
+    is evaluated.
+
+  - `next`: Step over the current form. Stepping will be disabled
+    until evaluation of the form is complete.
+
+  - `out`: Step out of the current frame. Stepping will be disabled
+    until the topmost stack frame that had been stepped into returns.
+
+  - `stop`: Stop the single stepper and resumes normal execution."
+  (step macro))
+
+(defsection @enabling-and-disabling-the-debugger
+    (:title "Enabling and Disabling the Debugger"
+     :concepts (("debugger," "enabling")
+                ("debugger," "disabling")
+                ("enabling" "debugger")
+                ("disabling" "debugger")
+                ~enabling-ldb
+                ~disabling-ldb))
+  "In certain contexts (e.g. non-interactive applications), it may be
+  desirable to turn off the SBCL debugger (and possibly re-enable it).
+  The functions here control the debugger."
+  (sb-ext:disable-debugger function)
+  (sb-ext:enable-debugger function))

contrib/sb-manual/doc/deprecation.lisp

@@ -0,0 +1,396 @@
+(in-package :sb-manual)
+
+(defsection @deprecation (:title "Deprecation"
+                          :concepts ("deprecation"))
+  "In order to support evolution of interfaces in SBCL as well as in user
+  code, SBCL allows declaring functions, variables and types as
+  deprecated. Users of deprecated things are notified by means of
+  warnings while the deprecated thing in question is still available.
+
+  This chapter documents the interfaces for being notified when using
+  deprecated thing and declaring things as deprecated, the deprecation
+  process used for SBCL interfaces, and lists legacy interfaces in
+  various stages of deprecation.
+
+  _Deprecation_ in this context should not be confused with those
+  things the ANSI Common Lisp standard calls _deprecated_: the
+  entirety of ANSI CL is supported by SBCL, and none of those
+  interfaces are subject to censure."
+  (@why-deprecate? section)
+  (@the-deprecation-pipeline section)
+  (@deprecation-conditions section)
+  (@introspecting-deprecation-information section)
+  (@deprecation-declaration section)
+  (@deprecation-examples section)
+  (@deprecated-interfaces-in-sbcl section))
+
+(defsection @why-deprecate? (:title "Why Deprecate?")
+  "While generally speaking we try to keep SBCL changes as backwards
+  compatible as feasible, there are situations when existing interfaces
+  are deprecated:
+
+  - __Broken Interfaces__
+
+      Sometimes it turns out that an interface is sufficiently
+      misdesigned that fixing it would be worse than deprecating it
+      and replacing it with another.
+
+      This is typically the case when fixing the interface would
+      change its semantics in ways that could break user code subtly:
+      in such cases we may end up considering the obvious breakage
+      caused by deprecation to be preferable.
+
+      Another example are functions or macros whose current signature
+      makes them hard or impossible to extend in the future: backwards
+      compatible extensions would either make the interface
+      intolerably hairy, or are sometimes outright impossible.
+
+  - __Internal Interfaces__
+
+      SBCL has several internal interfaces that were never meant to be
+      used in user code -- or at least never meant to be used in user
+      code unwilling to track changes to SBCL internals.
+
+      Ideally, we'd like to be free to refactor our own internals as
+      we please, without even going through the hassle of deprecating
+      things. Sometimes, however, it turns out that our internal
+      interfaces have several external users who aren't using them
+      advisedly, but due to misunderstandings regarding their status
+      or stability.
+
+      Consider a deprecated internal interface a reminder for SBCL
+      maintainers not to delete the thing just yet, even though it is
+      seems unused -- because it has external users.
+
+      When internal interfaces are deprecated we try our best to
+      provide supported alternatives.
+
+  - __Aesthetics & Ease of Maintenance__
+
+      Sometimes an interface isn't broken or internal but just
+      inconsistent somehow.
+
+      This mostly happens only with historical interfaces inherited
+      from CMUCL which often haven't been officially supported in SBCL
+      before, or with new extensions to SBCL that haven't been around
+      for very long in the first place.
+
+      The alternative would be to keep the suboptimal version around
+      forever, possibly alongside an improved version. Sometimes we
+      may do just that, but because every line of code comes with a
+      maintenance cost, sometimes we opt to deprecate the suboptimal
+      version instead: SBCL doesn't have infinite developer resources.
+
+      We also believe that sometimes cleaning out legacy interfaces
+      helps keep the whole system more comprehensible to users, and
+      makes introspective tools such as APROPOS more useful.")
+
+(defsection @the-deprecation-pipeline (:title "The Deprecation Pipeline")
+  "SBCL uses a _deprecation pipeline_ with multiplestages: as
+  time time goes by, deprecated things move from earlier stages of
+  deprecation to later stages before finally being removed. The
+  intention is making users aware of necessary changes early but
+  allowing a migration to new interfaces at a reasonable pace.
+
+  Deprecation proceeds in three stages, each lasting approximately a
+  year. In some cases it might move slower or faster, but one year per
+  stage is what we aim at in general. During each stage warnings (and
+  errors) of increasing severity are signaled, which note that the
+  interface is deprecated, and point users towards any replacements
+  when applicable.
+
+  - __Early Deprecation__
+
+      During early deprecation the interface is kept in working
+      condition. However, when a thing in this deprecation stage is
+      used, an SB-EXT:EARLY-DEPRECATION-WARNING, which is a
+      STYLE-WARNING, is signaled at compile-time.
+
+      The internals may change at this stage: typically because the
+      interface is re-implemented on top of its successor. While we
+      try to keep things as backwards-compatible as feasible (taking
+      maintenance costs into account), sometimes semantics change
+      slightly.
+
+      For example, when the spinlock API was deprecated, spinlock
+      objects ceased to exist, and the whole spinlock API became a
+      synonym for the mutex API -- so code using the spinlock API
+      continued working but silently switched to mutexes instead.
+      However, if someone relied on
+
+          (typep lock 'spinlock)
+
+      returning NIL for a mutexes, trouble could ensue.
+
+  - __Late Deprecation__
+
+      During late deprecation the interface remains as it was during
+      early deprecation, but the compile-time warning is upgraded:
+      when a thing in this deprecation stage is used, a
+      SB-EXT:LATE-DEPRECATION-WARNING, which is a full WARNING, is
+      signaled at compile-time.
+
+  - __Final Deprecation__
+
+      During final deprecation the symbols still exist. However, when
+      a thing in this deprecation stage is used, a
+      SB-EXT:FINAL-DEPRECATION-WARNING, which is a full WARNING, is
+      signaled at compile-time and an ERROR is signaled at run-time.
+
+  - __After Final Deprecation__
+
+      The interface is deleted entirely.")
+
+(defsection @deprecation-conditions (:title "Deprecation Conditions")
+  "SB-EXT:DEPRECATION-CONDITION is the superclass of all
+  deprecation-related warning and error conditions. All common slots and
+  readers are defined in this condition class."
+  (sb-ext:deprecation-condition condition)
+  (sb-ext:early-deprecation-warning condition)
+  (sb-ext:late-deprecation-warning condition)
+  (sb-ext:final-deprecation-warning condition)
+  (sb-ext:deprecation-error condition))
+
+(defsection @introspecting-deprecation-information
+    (:title "Introspecting Deprecation Information")
+  "The deprecation status of functions and variables can be inspected
+  using the SB-CLTL2:FUNCTION-INFORMATION and
+  SB-CLTL2:VARIABLE-INFORMATION functions provided by the `SB-CLTL2`
+  contributed module.")
+
+(defsection @deprecation-declaration (:title "Deprecation Declaration")
+  "The SB-EXT:DEPRECATED declaration can be used to declare objects
+  in various namespaces as deprecated.
+
+  > _Note_: See the `namespace` CLHS glossary entry in the glossary of
+  > the Common Lisp HyperSpec.)"
+  (sb-ext:deprecated declaration))
+
+(defsection @deprecation-examples (:title "Deprecation Examples")
+  "Marking functions as deprecated:
+
+      (defun foo ())
+      (defun bar ())
+      (declaim (deprecated :early (\"my-system\" \"1.2.3\")
+                           (function foo :replacement bar)))
+
+      ;; Remember: do not define the actual function or variable in case of
+      ;; :final deprecation:
+      (declaim (deprecated :final (\"my-system\" \"1.2.3\")
+                           (function fez :replacement whoop)))
+
+  Attempting to use the deprecated functions:
+
+      (defun baz ()
+        (foo))
+      | STYLE-WARNING: The function CL-USER::FOO has been deprecated...
+      => BAZ
+      (baz)
+      => NIL ; no error
+
+      (defun danger ()
+        (fez))
+      | WARNING: The function CL-USER::FEZ has been deprecated...
+      => DANGER
+      (danger)
+      |- ERROR: The function CL-USER::FEZ has been deprecated...")
+
+
+(defsection @deprecated-interfaces-in-sbcl
+    (:title "Deprecated Interfaces in SBCL")
+  "This sections lists legacy interfaces in various stages of deprecation."
+  (@list-of-deprecated-interfaces section)
+  (@historical-interfaces section))
+
+(defsection @list-of-deprecated-interfaces
+    (:title "List of Deprecated Interfaces")
+  (@early-deprecation section)
+  (@late-deprecation section)
+  (@final-deprecation section))
+
+(defsection @early-deprecation (:title "Early Deprecation")
+  "- `SOCKINT::WIN32-*`
+
+      Deprecated in favor of the corresponding prefix-less functions
+      (e.g. `SOCKINT::BIND` replaces `SOCKINT::WIN32-BIND`) as of
+      1.2.10 in March 2015. Expected to move into late deprecation in
+      August 2015.
+
+  - SB-UNIX:UNIX-EXIT
+
+      Deprecated as of 1.0.56.55 in May 2012. Expected to move into
+      late deprecation in May 2013.
+
+      When the SBCL process termination was refactored,
+      SB-UNIX:UNIX-EXIT ceased to be used internally. Since `SB-UNIX`
+      is an internal package not intended for user code to use, and
+      since we're slowly in the process of refactoring things to be
+      less Unix-oriented, SB-UNIX:UNIX-EXIT was initially deleted as
+      it was no longer used. Unfortunately it became apparent that it
+      was used by several external users, so it was re-instated in
+      deprecated form.
+
+      While the cost of keeping SB-UNIX:UNIX-EXIT indefinitely is
+      trivial, the ability to refactor our internals is important, so
+      its deprecation was taken as an opportunity to highlight that
+      `SB-UNIX` is an internal package and `SB-POSIX` should be used
+      by user-programs instead -- or alternatively calling the foreign
+      function directly if the desired interface doesn't for some
+      reason exist in `SB-POSIX`.
+
+      __Remedy__
+
+      For code needing to work with legacy SBCLs, use e.g.
+      `SYSTEM-EXIT`. In modern SBCLs, simply call either SB-POSIX:EXIT
+      or SB-EXT:EXIT with appropriate arguments.
+
+  - `SB-C::MERGE-TAIL-CALLS` compiler policy
+
+      Deprecated as of 1.0.53.74 in November 2011. Expected to move
+      into late deprecation in November 2012.
+
+      This compiler policy was never functional: SBCL has always
+      merged tail calls when it could, regardless of this policy
+      setting. (It was also never officially supported, but several
+      code-bases have historically used it.)
+
+      __Remedy__
+
+      Simply remove the policy declarations. They were never necessary: SBCL
+      always merged tail-calls when possible. To disable tail merging,
+      structure the code to avoid the tail position instead.
+
+  - The Spinlock API
+
+      Deprecated as of 1.0.53.11 in August 2011. Expected to move into
+      late deprecation in August 2012.
+
+      Spinlocks were an internal interface but had a number of
+      external users and were hence deprecated instead of being simply
+      deleted.
+
+      Affected symbols: SB-THREAD::SPINLOCK, SB-THREAD::MAKE-SPINLOCK,
+      SB-THREAD::WITH-SPINLOCK, SB-THREAD::WITH-RECURSIVE-SPINLOCK,
+      SB-THREAD::GET-SPINLOCK, SB-THREAD::RELEASE-SPINLOCK,
+      SB-THREAD::SPINLOCK-VALUE, and SB-THREAD::SPINLOCK-NAME.
+
+      __Remedy__
+
+      Use the mutex API instead, or implement spinlocks suiting your
+      needs on top of SB-EXT:COMPARE-AND-SWAP, SB-EXT:SPIN-LOOP-HINT,
+      etc.
+
+  - `SOCKINT::HANDLE->FD`, `SOCKINT::FD->HANDLE`
+
+      Internally deprecated in 2012. Declared deprecated as of 1.2.10
+      in March 2015. Expected to move into final deprecation in August
+      2015.")
+
+(defsection @late-deprecation (:title "Late Deprecation")
+  "- SB-THREAD:JOIN-THREAD-ERROR-THREAD and
+    SB-THREAD:INTERRUPT-THREAD-ERROR-THREAD
+
+      Deprecated in favor of SB-THREAD:THREAD-ERROR-THREAD as of
+      1.0.29.17 in June 2009. Expected to move into final deprecation
+      in June 2012.
+
+      __Remedy__
+
+      For code that needs to support legacy SBCLs, use e.g.:
+
+          (defun get-thread-error-thread (condition)
+            #+#.(cl:if (cl:find-symbol \"THREAD-ERROR-THREAD\" :sb-thread)
+                       '(and) '(or))
+            (sb-thread:thread-error-thread condition)
+            #-#.(cl:if (cl:find-symbol \"THREAD-ERROR-THREAD\" :sb-thread)
+                       '(and) '(or))
+            (etypecase condition
+             (sb-thread:join-thread-error
+              (sb-thread:join-thread-error-thread condition))
+             (sb-thread:interrupt-thread-error
+              (sb-thread:interrupt-thread-error-thread condition))))
+
+  - SB-INTROSPECT:FUNCTION-ARGLIST
+
+      Deprecated in favor of SB-INTROSPECT:FUNCTION-LAMBDA-LIST as of
+      1.0.24.5 in January 2009. Expected to move into final
+      deprecation in January 2012.
+
+      Renamed for consistency and aesthetics. Functions have
+      lambda-lists, not arglists.
+
+      __Remedy__
+
+          For code that needs to support legacy SBCLs, use e.g.:
+
+          (defun get-function-lambda-list (function)
+            #+#.(cl:if (cl:find-symbol \"FUNCTION-LAMBDA-LIST\" :sb-introspect)
+                       '(and) '(or))
+            (sb-introspect:function-lambda-list function)
+            #-#.(cl:if (cl:find-symbol \"FUNCTION-LAMBDA-LIST\" :sb-introspect)
+                       '(and) '(or))
+            (sb-introspect:function-arglist function))
+
+  - Stack Allocation Policies
+
+      Deprecated in favor of SB-EXT:*STACK-ALLOCATE-DYNAMIC-EXTENT* as
+      of 1.0.19.7 in August 2008, and are expected to be removed in
+      August 2012.
+
+      Affected symbols: `SB-C::STACK-ALLOCATE-DYNAMIC-EXTENT`,
+      `SB-C::STACK-ALLOCATE-VECTOR`, and
+      `SB-C::STACK-ALLOCATE-VALUE-CELLS`.
+
+      These compiler policies were never officially supported, and
+      turned out the be a flawed design.
+
+      __Remedy__
+
+      For code that needs stack-allocation in legacy SBCLs,
+      conditionalize using:
+
+          #-#.(cl:if (cl:find-symbol \"*STACK-ALLOCATE-DYNAMIC-EXTENT*\" :sb-ext)
+                     '(and) '(or))
+          (declare (optimize sb-c::stack-allocate-dynamic-extent))
+
+      However, unless stack allocation is essential, we recommend
+      simply removing these declarations. Refer to documentation on
+      `SB-EXT:*STACK-ALLOCATE-DYNAMIC*` for details on stack
+      allocation control in modern SBCLs.
+
+  - `SB-SYS:OUTPUT-RAW-BYTES`
+
+      Deprecated as of 1.0.8.16 in June 2007. Expected to move into final
+      deprecation in June 2012.
+
+      Internal interface with some external users. Never officially
+      supported, deemed unnecessary in presence of WRITE-SEQUENCE and
+      bivalent streams.
+
+      __Remedy__
+
+      Use streams with element-type (UNSIGNED-BYTE 8) or
+      :DEFAULT -- the latter allowing both binary and character IO --
+      in conjunction with WRITE-SEQUENCE.")
+
+(defsection @final-deprecation (:title "Final Deprecation")
+  "No interfaces are currently in final deprecation.")
+
+(defsection @historical-interfaces (:title "Historical Interfaces")
+  "The following is a partial list of interfaces present in historical
+  versions of SBCL, which have since then been deleted.
+
+  - `SB-KERNEL:INSTANCE-LAMBDA`
+
+      Historically needed for CLOS code. Deprecated as of 0.9.3.32 in
+      August 2005. Deleted as of 1.0.47.8 in April 2011. Plain LAMBDA
+      can be used where `SB-KERNEL:INSTANCE-LAMBDA` used to be needed.
+
+  - `SB-ALIEN:DEF-ALIEN-ROUTINE`, `SB-ALIEN:DEF-ALIEN-VARIABLE`,
+    `SB-ALIEN:DEF-ALIEN-TYPE`
+
+      Inherited from CMUCL, naming convention not consistent with
+      preferred SBCL style. Deprecated as of 0.pre7.90 in December
+      2001. Deleted as of 1.0.9.17 in September 2007. Replaced by
+      SB-ALIEN:DEFINE-ALIEN-ROUTINE, SB-ALIEN:DEFINE-ALIEN-VARIABLE,
+      and SB-ALIEN:DEFINE-ALIEN-TYPE.")

contrib/sb-manual/doc/efficiency.lisp

@@ -0,0 +1,387 @@
+(in-package :sb-manual)
+
+(defsection @efficiency (:title "Efficiency"
+                         :concepts ("efficicency"))
+  (@slot-access section)
+  (@stack-allocation section)
+  (@modular-arithmetic section)
+  (@recognized-idioms section)
+  (@global-and-always-bound-variables section)
+  (@miscellaneous-efficiency-issues section))
+
+(defsection @slot-access (:title "Slot Access"
+                          :concepts (("slot" "access")))
+  (@structure-object-slot-access section)
+  (@standard-object-slot-access section))
+
+(defsection @structure-object-slot-access
+    (:title "Structure Object Slot Access")
+  "Structure slot accessors are efficient only if the compiler is
+  able to open code them: compiling a call to a structure slot
+  accessor before the structure is defined, declaring one NOTINLINE,
+  or passing it as a functional argument to another function causes
+  severe performance degradation.")
+
+(defsection @standard-object-slot-access
+    (:title "Standard Object Slot Access")
+  "The most efficient way to access a slot of a STANDARD-OBJECT is
+  by using SLOT-VALUE with a constant slot name argument inside a
+  DEFMETHOD body, where the variable holding the instance is a
+  specializer parameter of the method and is never assigned to. The
+  cost is roughly 1.6 times that of an open coded structure slot
+  accessor.
+
+  Second most efficient way is to use a CLOS slot accessor, or
+  SLOT-VALUE with a constant slot name argument, but in circumstances
+  other than specified above. This may be up to 3 times as slow as the
+  method described above.
+
+  Example:
+
+      (defclass foo () ((bar)))
+
+      ;; Fast: specializer and never assigned to
+      (defmethod quux ((foo foo) new)
+        (let ((old (slot-value foo 'bar)))
+          (setf (slot-value foo 'bar) new)
+          old))
+
+      ;; Slow: not a specializer
+      (defmethod quux ((foo foo) new)
+        (let* ((temp foo)
+               (old (slot-value temp 'bar)))
+          (setf (slot-value temp 'bar) new)
+          old))
+
+      ;; Slow: assignment to FOO
+      (defmethod quux ((foo foo) new)
+        (let ((old (slot-value foo 'bar)))
+          (setf (slot-value foo 'bar) new)
+          (setf foo new)
+          old))
+
+  Note that when profiling code such as this, the first few calls to the
+  generic function are not representative, as the dispatch mechanism is
+  lazily set up during those calls.")
+
+(defsection @stack-allocation (:title "Stack Allocation")
+  "SBCL has fairly extensive support for performing allocations on the
+  stack when a variable or function is declared DYNAMIC-EXTENT. The
+  DYNAMIC-EXTENT declarations are not verified but are simply trusted
+  as long as SB-EXT:*STACK-ALLOCATE-DYNAMIC-EXTENT* is true."
+  (sb-ext:*stack-allocate-dynamic-extent* variable)
+  "SBCL recognizes any value which a variable declared DYNAMIC-EXTENT
+  can take on as having dynamic extent. This means that, in addition
+  to the value a variable is bound to initially, a value assigned to a
+  variable by SETQ is also recognized as having dynamic extent when
+  the variable is declared DYNAMIC-EXTENT. Users can thus build
+  complex structures on the stack using iteration and SETQ.
+
+  At present, SBCL implements stack allocation for the following kinds
+  of values when they are recognized as having dynamic extent:
+
+  - &REST lists;
+
+  - the results of CONS, LIST, LIST*, and VECTOR;
+
+  - the result of simple forms of MAKE-ARRAY: stack allocation is
+    possible only if the resulting array is known to be both simple
+    and one-dimensional, and has a constant :ELEMENT-TYPE;
+
+      > __Warning__: Stack space is limited, so allocation of a large
+      > vector may cause stack overflow. Stack overflow checks are
+      > done except in 0 @SAFETY policies.
+
+  - closures defined with FLET or LABELS with a bound DYNAMIC-EXTENT
+    declaration;
+
+  - anonymous closures defined with LAMBDA;
+
+  - user-defined structures when the structure constructor defined using
+    DEFSTRUCT has been declared INLINE;
+
+      > _Note_: Structures with _raw_ slots can currently be
+      > stack-allocated only on x86 and x86-64. A raw slot is one
+      > whose declared type is a subtype of exactly one of:
+      > DOUBLE-FLOAT, SINGLE-FLOAT, `(COMPLEX
+      > DOUBLE-FLOAT)`, `(COMPLEX SINGLE-FLOAT)`, or SB-EXT:WORD; but
+      > as an exception to the preceding, any subtype of FIXNUM is not
+      > stored as raw despite also being a subtype of SB-EXT:WORD.
+
+  - otherwise-inaccessible parts of objects recognized to be dynamic
+    extent. The support for detecting when this applies is very
+    sophisticated. The compiler can do this detection when any value
+    form for a variable contains conditional allocations, function
+    calls, inlined functions, anonymous closures, or even other
+    variables. This allows stack allocation of complex structures.
+
+  Examples:
+
+      ;;; Declaiming a structure constructor inline before definition makes
+      ;;; stack allocation possible.
+      (declaim (inline make-thing))
+      (defstruct thing obj next)
+
+      ;;; Stack allocation of various objects bound to DYNAMIC-EXTENT
+      ;;; variables.
+      (let* ((list (list 1 2 3))
+             (nested (cons (list 1 2) (list* 3 4 (list 5))))
+             (vector (make-array 3 :element-type 'single-float))
+             (thing (make-thing :obj list
+                                :next (make-thing :obj (make-array 3))))
+             (closure (let ((y ...)) (lambda () y))))
+        (declare (dynamic-extent list nested vector thing closure))
+        ...)
+
+      ;;; Stack allocation of objects assigned to DYNAMIC-EXTENT variables.
+      (let ((x nil))
+        (declare (dynamic-extent x))
+        (setq x (list 1 2 3))
+        (dotimes (i 10)
+          (setq x (cons i x)))
+        ...)
+
+      ;;; Stack allocation of arguments to a local function is equivalent
+      ;;; to stack allocation of local variable values.
+      (flet ((f (x)
+               (declare (dynamic-extent x))
+               ...))
+        ...
+        (f (list 1 2 3))
+        (f (cons (cons 1 2) (cons 3 4)))
+        ...)
+
+      ;;; Stack allocation of &REST lists
+      (defun foo (&rest args)
+        (declare (dynamic-extent args))
+        ...)
+
+  As a notable exception to recognizing otherwise inaccessible parts
+  of other recognized dynamic extent values, SBCL does not as of
+  1.0.48.21 propagate dynamic-extentness through &REST arguments --
+  but another conforming implementation might, so portable code should
+  not rely on this.
+
+      (declaim (inline foo))
+      (defun foo (fun &rest arguments)
+        (declare (dynamic-extent arguments))
+        (apply fun arguments))
+
+      (defun bar (a)
+        ;; SBCL will heap allocate the result of (LIST A), and stack
+        ;; allocate only the spine of the &rest list -- so this is
+        ;; safe but unportable.
+        ;;
+        ;; Another implementation, including earlier versions of SBCL
+        ;; might consider (LIST A) to be otherwise inaccessible and
+        ;; stack-allocate it as well!
+        (foo #'car (list a)))
+
+  If dynamic extent constraints specified in the Common Lisp standard
+  are violated, the best that can happen is for the program to have
+  garbage in variables and return values; more commonly, the system
+  will crash.
+
+  In particular, it is important to realize that this can interact in
+  suprising ways with the otherwise inaccessible parts criterion:
+
+      (let* ((a (list 1 2 3))
+             (b (cons a a)))
+         (declare (dynamic-extent b))
+         ;; Unless A is accessed elsewhere as well, SBCL will consider
+         ;; it to be otherwise inaccessible -- it can only be accessed
+         ;; through B, after all -- and stack allocate it as well.
+         ;;
+         ;; Hence returning (CAR B) here is unsafe.
+         ...)
+
+  SBCL also performs sophisticated escape analysis to enable automatic
+  stack allocation of local functions without any bound dynamic extent
+  declarations in many situations where the compiler can prove that no
+  uses escape (traditional Lisp terminology names this situation \"all
+  uses are downward funargs\"). For example, in the following
+  function, the local function `#'PREDICATEP` is stack allocated,
+  because the compiler understands that the built-in function
+  POSITION-IF only uses its first argument as a downward funarg:
+
+      (let ((acc 0))
+        (flet ((predicatep (num) (plusp (+ num off))))
+          (dotimes (i 10)
+            (incf acc (position-if #'predicatep array)))
+          (if (plusp off)
+              (incf acc (if (positivep acc) 10 3))
+              (incf acc (position-if #'predicatep array))))
+        acc)
+
+  Users can also declare that their own functions take downward
+  funargs by adding bound dynamic extent declarations on the function
+  arguments.
+
+      (defun trivial-hof (fun arg)
+        (declare (dynamic-extent fun))
+        (funcall fun 3 arg))
+
+  Currently, such dynamic extent declarations only cause stack
+  allocation of downward funargs at call sites on sufficiently unsafe
+  policy. This is partly because the compiler is currently not able to
+  detect incorrect usage of dynamic extent declarations.
+
+      (defun autodxclosure1 (&optional (x 4))
+        ;; Calling a higher-order function will only implicitly
+        ;; stack-allocate a funarg if the callee is trusted (a CL:
+        ;; function) or the caller is unsafe.
+        (declare (optimize speed (safety 0) (debug 0)))
+        (trivial-hof (lambda (a b) (+ a b x)) 92))")
+
+(defsection @modular-arithmetic (:title "Modular Arithmetic"
+                                 :concepts ("modular arithmetic"
+                                            ("arithmetic," "modular")
+                                            ("arithmetic," "hardware")))
+  "Some numeric functions have a property: n lower bits of the
+  result depend only on n lower bits of (all or some) arguments. If
+  the compiler sees an expression of form `(LOGAND <EXPR> <MASK>)`,
+  where `<EXPR>` is a tree of such _good_ functions and `<MASK>` is
+  known to be of type `(UNSIGNED-BYTE <W>)`, where `<W>` is a _good_
+  width, all intermediate results will be cut to `<W>` bits (but it is
+  not done for variables and constants!). This often results in an
+  ability to use simple machine instructions for the functions.
+
+  Consider this example:
+
+      (defun i (x y)
+        (declare (type (unsigned-byte 32) x y))
+        (ldb (byte 32 0) (logxor x (lognot y))))
+
+  The result of `(LOGNOT Y)` will be negative and of type
+  `(SIGNED-BYTE 33)`, so a naive implementation on a 32-bit platform
+  is unable to use 32-bit arithmetic here. But modular arithmetic
+  optimizer is able to do it: because the result is cut down to 32
+  bits, the compiler will replace LOGXOR and LOGNOT with versions
+  cutting results to 32 bits, and because terminals (here, expressions
+  `X` and `Y`) are also of type `(UNSIGNED-BYTE 32)`, 32-bit machine
+  arithmetic can be used.
+
+  As of SBCL 0.8.5 good functions are `+`, `-`, LOGAND, LOGIOR,
+  LOGXOR, LOGNOT and their combinations; and ASH with the positive
+  second argument. Good widths are 32 on 32-bit CPUs and 64 on 64-bit
+  CPUs. While it is possible to support smaller widths as well,
+  currently this is not implemented."
+  (@signed-modular-arithmetic section))
+
+(defsection @signed-modular-arithmetic (:title "Signed Modular Arithmetic")
+  "Sign-extending the result in the following way will be
+  translated into signed modular arithmetic:
+
+      (defun add (a b)
+        (declare (type (signed-byte 64) a b))
+        (let ((u (ldb (byte 64 0) (+ a b))))
+          (logior u (- (mask-field (byte 1 63) u)))))")
+
+(defsection @recognized-idioms (:title "Recognized Idioms"
+                                :concepts ("modular arithmetic"
+                                           ("arithmetic," "modular")
+                                           ("arithmetic," "hardware")))
+  "Common Lisp doesn't directly expose all features present in
+  modern hardware. Some code patterns are recognized and turned into
+  more efficient hardware instructions without requiring the use of
+  internal features."
+  (@count-trailing-zeros section))
+
+(defsection @count-trailing-zeros (:title "Count Trailing Zeros")
+  "    (defun ctz (n)
+        (declare (type (unsigned-byte 64) n))
+        (integer-length (ldb (byte 64 0) (lognor n (- n)))))
+
+  is turned into hardware instructions on arm64 and x86-64. It returns
+  64 when `N` is 0. `N` can also be `(SIGNED-BYTE 64)` or FIXNUM.")
+
+(defsection @global-and-always-bound-variables
+    (:title "Global and Always-bound Variables")
+  (sb-ext:defglobal macro)
+  (sb-ext:global declaration)
+  (sb-ext:always-bound declaration))
+
+(defsection @miscellaneous-efficiency-issues
+    (:title "Miscellaneous Efficiency Issues")
+  "FIXME: The material in the CMUCL manual about getting good
+  performance from the compiler should be reviewed, reformatted in
+  Texinfo, lightly edited for SBCL, and substituted into this
+  manual. In the meantime, the original CMUCL manual is still 95+%
+  correct for the SBCL version of the Python compiler. See the
+  sections
+
+  - Advanced Compiler Use and Efficiency Hints
+  - Advanced Compiler Introduction
+  - More About Types in Python
+  - Type Inference
+  - Source Optimization
+  - Tail Recursion
+  - Local Call
+  - Block Compilation
+  - Inline Expansion
+  - Object Representation
+  - Numbers
+  - General Efficiency Hints
+  - Efficiency Notes
+
+  Besides this information from the CMUCL manual, there are a few other
+  points to keep in mind.
+
+  - The CMUCL manual doesn't seem to state it explicitly, but Python
+    has a mental block about type inference when assignment is
+    involved. Python is very aggressive and clever about inferring the
+    types of values bound with LET, LET*, inline function call, and so
+    forth. However, it's much more passive and dumb about inferring
+    the types of values assigned with SETQ, SETF, and friends. It
+    would be nice to fix this, but in the meantime don't expect that
+    just because it's very smart about types in most respects it will
+    be smart about types involved in assignments. (This doesn't affect
+    its ability to benefit from explicit type declarations involving
+    the assigned variables, only its ability to get by without
+    explicit type declarations.)"
+  ;; FIXME: Python dislikes assignments but not in type inference. The
+  ;; real problems are loop induction, closed over variables and
+  ;; aliases.
+  "- Since the time the CMUCL manual was written, CMUCL (and thus SBCL)
+    has gotten a @GENERATIONAL-GC. This means that there are some
+    efficiency implications of various patterns of memory usage which
+    aren't discussed in the CMUCL manual. (Some new material should be
+    written about this.)
+
+  - SBCL has some important known efficiency problems. Perhaps the
+    most important are
+
+      - The garbage collector is not particularly efficient, at least
+        on platforms without the generational collector (as of SBCL
+        0.8.9, all except x86).
+
+      - Various aspects of the PCL implementation of CLOS are more
+        inefficient than necessary.
+
+  Finally, note that Common Lisp defines many constructs which, in the
+  infamous phrase, \"could be compiled efficiently by a sufficiently
+  smart compiler\". The phrase is infamous because making a compiler
+  which actually is sufficiently smart to find all these optimizations
+  systematically is well beyond the state of the art of current
+  compiler technology. Instead, they're optimized on a case-by-case
+  basis by hand-written code, or not optimized at all if the
+  appropriate case hasn't been hand-coded. Some cases where no such
+  hand-coding has been done as of SBCL version 0.6.3 include
+
+  - `(REDUCE #'F X)` where the type of `X` is known at compile time,
+
+  - various bit vector operations, e.g. `(POSITION 0 SOME-BIT-VECTOR)`,
+
+  - specialized sequence idioms, e.g. `(REMOVE ITEM LIST :COUNT 1)`,
+
+  - cases where local compilation policy does not require excessive
+    type checking, e.g. `(LOCALLY (DECLARE (SAFETY 1)) (ASSOC ITEM LIST))`
+    (which currently performs safe ENDP checking internal to ASSOC).
+
+  If your system's performance is suffering because of some construct
+  which could in principle be compiled efficiently, but which the SBCL
+  compiler can't in practice compile efficiently, consider writing a
+  patch to the compiler and submitting it for inclusion in the main
+  sources. Such code is often reasonably straightforward to write;
+  search the sources for the string `deftransform` to find many
+  examples (some straightforward, some less so).")

contrib/sb-manual/doc/external-formats.lisp

@@ -0,0 +1,137 @@
+(in-package :sb-manual)
+
+(defsection @external-formats (:title "External Formats"
+                               :concepts (@external-format))
+  "External formats determine the coding of characters from/to sequences
+  of octets when exchanging data with the outside world. Examples of
+  such exchanges are:
+
+  - Character streams associated with files, sockets and process
+    input/output (see @STREAM-EXTERNAL-FORMATS and
+    @RUNNING-EXTERNAL-PROGRAMS)
+
+  - Names of files
+
+  - Foreign strings (see @FOREIGN-TYPES-AND-LISP-TYPES)
+
+  - Posix interface (see @SB-POSIX)
+
+  - Hostname- and protocol-related functions of the BSD-socket interface
+    (see @NETWORKING)
+
+  Technically, external formats in SBCL are named objects describing
+  coding of characters as well as policies in case de- or encoding is
+  not possible. Each external format has a canonical name and zero or
+  more aliases. User code mostly interacts with external formats by
+  supplying external format designators to functions that use external
+  formats internally."
+  (@default-external-format section)
+  (@external-format-designators section)
+  (@character-coding-conditions section)
+  (@converting-between-strings-and-octet-vectors section)
+  (@supported-external-formats section))
+
+(defsection @default-external-format (:title "The Default External Format")
+  (sb-ext:*default-external-format* variable)
+  (sb-ext:*default-source-external-format* variable)
+  ;; FIXME: Move this to @FFI?
+  (sb-ext:*default-c-string-external-format* variable))
+
+(defsection @external-format-designators (:title "External Format Designators")
+  "In situations where an external format designator is required, such as
+  the :EXTERNAL-FORMAT argument in calls to OPEN or WITH-OPEN-FILE,
+  users may supply the name of an encoding to denote the external
+  format which is applying that encoding to Lisp characters.
+
+  In addition to the basic encoding for an external format, options
+  controlling various special cases may be passed, by using a list
+  (whose first element must be an encoding name and whose rest is a
+  plist) as an external file format designator.
+
+  More specifically, external format designators can take the
+  following forms:
+
+  - :DEFAULT: Designates the current default external format (see
+    @DEFAULT-EXTERNAL-FORMAT).
+
+  - `<keyword>`: Designates the supported external format that has
+    `<keyword>` as one of its names (see @SUPPORTED-EXTERNAL-FORMATS).
+
+  - `(<keyword> . <options-plist>)`: Designates an external format
+    that is like the one designated by `<keyword>` with options as
+    specified in `<options-plist>`.
+
+  Valid options for `<options-plist>` are:
+
+  - `:NEWLINE <newline>`
+
+      An external format with an explicit :NEWLINE option is like its
+      `<keyword>` parent but recognizes certain characters or
+      character sequences as newlines. For :LF (the default), the
+      `#\\Linefeed` character is treated as `#\\Newline` for both
+      input and output. For :CR, `#\\Return` is treated as
+      `#\\Newline`, while for :CRLF the two-character sequence
+      `#\\Return #\\Linefeed` is translated to and from
+      `#\\Newline`.
+
+  - `:REPLACEMENT <replacement>`
+
+      An external format with an explicit :REPLACEMENT option is like
+      its `<keyword>` parent but does not signal an error in case a
+      character or octet sequence cannot be en- or decoded. Instead,
+      it inserts `<replacement>` at the position in question.
+      `<replacement>` must be a string designator; that is, a
+      character or a string.
+
+  For example:
+
+      (with-open-file (stream pathname :external-format '(:utf-8 :replacement #\\?))
+        (read-line stream))
+
+  will read the first line of `\\PATHNAME`, replacing any octet
+  sequence that is not valid in the UTF-8 external format with a
+  question mark character.")
+
+(defsection @character-coding-conditions (:title "Character Coding Conditions")
+  "De- or encoding characters using a given external format is not always
+  possible:
+
+  - Decoding an octet vector using a given external format can fail if
+    it contains an octet or sequence of octets that does not have an
+    interpretation as a character according to the external format.
+
+  - Conversely, a string may contain characters that a given external
+    format cannot encode. For example, the ASCII external format
+    cannot encode the character `#\\ö`.
+
+  Unless the external format governing the coding uses the
+  :REPLACEMENT option, SBCL will signal (continuable) errors under the
+  above circumstances. The types of the condition signaled are not
+  currently exported or documented but will be in future SBCL
+  versions.")
+
+(defsection @converting-between-strings-and-octet-vectors
+    (:title "Converting between Strings and Octet Vectors")
+  "To encode Lisp strings as octet vectors and decode octet vectors as
+  Lisp strings, the following SBCL-specific functions can be used:"
+  (sb-ext:string-to-octets function)
+  (sb-ext:octets-to-string function))
+
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (defun list-external-formats-in-markdown ()
+    (flet ((table (items)
+             (with-output-to-string (s)
+               (loop for (canonical-name . names) in items
+                     do (format s "- `~S`~%~%    ~{`~S`~^, ~}~%~%"
+                                canonical-name names)))))
+      (let (result)
+        (loop for ef across sb-impl::*external-formats*
+              when (sb-impl::external-format-p ef)
+                do
+                   (pushnew (sb-impl::ef-names ef) result :test #'equal))
+        (table (sort result #'string< :key #'car))))))
+
+(defsection @supported-external-formats (:title "Supported External Formats")
+  "The following lists the external formats supported by SBCL in
+  the form of the respective canonical name followed by the list of aliases:"
+  #.(list-external-formats-in-markdown))

contrib/sb-manual/doc/ffi.lisp

@@ -0,0 +1,780 @@
+(in-package :sb-manual)
+
+(defsection @foreign-function-interface
+    (:title "Foreign Function Interface")
+  "This chapter describes SBCL's interface to C programs and
+  libraries (and, since C interfaces are a sort of _lingua franca_
+  of the Unix world, to other programs and libraries in general).
+
+  > _Note_: In the modern Lisp world, the usual term for this
+  > functionality is Foreign Function Interface, or FFI, where despite
+  > the mention of _function_ in this term, FFI also refers to direct
+  > manipulation of C data structures as well as functions. The
+  > traditional CMUCL terminology is Alien Interface, and while that
+  > older terminology is no longer used much in the system
+  > documentation, it still reflected in names in the implementation,
+  > notably in the name of the `SB-ALIEN` package."
+  (@introduction-to-the-foreign-function-interface section)
+  (@foreign-types section)
+  (@operations-on-foreign-values section)
+  (@foreign-variables section)
+  (@foreign-data-structure-examples section)
+  (@loading-shared-object-files section)
+  (@foreign-function-calls section)
+  (@calling-lisp-from-c section)
+  (@step-by-step-example-of-the-foreign-function-interface section))
+
+(defsection @introduction-to-the-foreign-function-interface
+    (:title "Introduction to the Foreign Function Interface")
+  ;; AKA Introduction to Aliens in the CMU CL manual
+  "Because of Lisp's emphasis on dynamic memory allocation and garbage
+  collection, Lisp implementations use non-C-like memory
+  representations for objects. This representation mismatch creates
+  friction when a Lisp program must share objects with programs which
+  expect C data. There are three common approaches to establishing
+  communication:
+
+  - The burden can be placed on the foreign program (and programmer)
+    by requiring the knowledge and use of the representations used
+    internally by the Lisp implementation. This can require a
+    considerable amount of \"glue\" code on the C side, and that code
+    tends to be sensitively dependent on the internal implementation
+    details of the Lisp system.
+
+  - The Lisp system can automatically convert objects back and forth
+    between the Lisp and foreign representations. This is convenient,
+    but translation becomes prohibitively slow when large or complex
+    data structures must be shared. This approach is supported by the
+    SBCL FFI, and used automatically when passing integers and
+    strings.
+
+  - The Lisp program can directly manipulate foreign objects through
+    the use of extensions to the Lisp language.
+
+  SBCL, like CMUCL before it, relies primarily on the automatic
+  conversion and direct manipulation approaches. The `SB-ALIEN`
+  package provides a facility wherein foreign values of simple scalar
+  types are automatically converted and complex types are directly
+  manipulated in their foreign representation. Additionally the
+  lower-level System Area Pointers (or SAPs) can be used where
+  necessary to provide untyped access to foreign memory.
+
+  Any foreign objects that can't automatically be converted into Lisp
+  values are represented by objects of type
+  SB-ALIEN-INTERNALS:ALIEN-VALUE Since Lisp is a dynamically typed
+  language, even foreign objects must have a run-time type; this type
+  information is provided by encapsulating the raw pointer to the
+  foreign data within an SB-ALIEN-INTERNALS:ALIEN-VALUE object.
+
+  The type language and operations on foreign types are intentionally
+  similar to those of the C language.")
+
+(defsection @foreign-types (:title "Foreign Types")
+  "Alien types have a description language based on nested list
+  structure. For example the C type
+
+      struct foo {
+          int a;
+          struct foo *b[100];
+      };
+
+  has the corresponding SBCL FFI type
+
+      (struct foo
+        (a int)
+        (b (array (* (struct foo)) 100)))"
+  (@defining-foreign-types section)
+  (@foreign-types-and-lisp-types section)
+  (@foreign-type-specifiers section))
+
+(defsection @defining-foreign-types (:title "Defining Foreign Types")
+  "Types may be either named or anonymous. With structure and union
+  types, the name is part of the type specifier, allowing recursively
+  defined types such as:
+
+      (struct foo (a (* (struct foo))))
+
+  An anonymous structure or union type is specified by using the name
+  NIL. The WITH-ALIEN macro defines a local scope which _captures_ any
+  named type definitions. Other types are not inherently named, but
+  can be given named abbreviations using the DEFINE-ALIEN-TYPE macro.")
+
+(defsection @foreign-types-and-lisp-types
+    (:title "Foreign Types and Lisp Types")
+  "The foreign types form a subsystem of the SBCL type system. An
+  ALIEN type specifier provides a way to use any foreign type as a
+  Lisp type specifier. For example,
+
+      (typep foo '(alien (* int)))
+
+  can be used to determine whether `FOO` is a pointer to a foreign
+  `int`. ALIEN type specifiers can be used in the same ways as
+  ordinary Lisp type specifiers (like STRING.) Alien type declarations
+  are subject to the same precise type checking as any other
+  declaration. See @PRECISE-TYPE-CHECKING.
+
+  Note that the type identifiers used in the foreign type system
+  overlap with native Lisp type specifiers in some cases. For example,
+  the type specifier `(ALIEN SINGLE-FLOAT)` is identical to
+  SINGLE-FLOAT, since foreign floats are automatically converted to
+  Lisp floats. When TYPE-OF is called on an alien value that is not
+  automatically converted to a Lisp value, then it will return an
+  ALIEN type specifier.")
+
+(defsection @foreign-type-specifiers (:title "Foreign Type Specifiers")
+  "> _Note_: All foreign type names are exported from the `SB-ALIEN`
+  > package. Some foreign type names are also symbols in the
+  > `COMMON-LISP` package, in which case they are reexported from the
+  > `SB-ALIEN` package, so that e.g. it is legal to refer to
+  > SINGLE-FLOAT.
+
+  These are the basic foreign type specifiers:
+
+  - The foreign type specifier `(* <FOO>)` describes a pointer to an
+    object of type `<FOO>`. A pointed-to type `<FOO>` of T indicates a
+    pointer to anything, similar to `void *` in ANSI C. A null alien
+    pointer can be detected with the NULL-ALIEN function.
+
+  - The foreign type specifier `(ARRAY <FOO> &REST <DIMENSIONS>)`
+    describes array of the specified `<DIMENSIONS>`, holding elements
+    of type `<FOO>`. Note that (unlike in C) `(* <FOO>)` and
+    `(ARRAY <FOO>)` are considered to be different types when
+    type checking is done. If equivalence of pointer and array types
+    is desired, it may be explicitly coerced using CAST.
+
+  Arrays are accessed using DEREF, passing the indices
+  as additional arguments.  Elements are stored in row-major order
+  (as in C), so the first dimension determines only the size of the
+  memory block, and not the layout of the higher dimensions. An array
+  whose first dimension is variable may be specified by using NIL as
+  the first dimension. Fixed-size arrays can be allocated as array
+  elements, structure slots or WITH-ALIEN variables. Dynamic arrays
+  can only be allocated using MAKE-ALIEN.
+
+  - The foreign type specifier `(STRUCT <NAME> &REST <FIELDS>)`
+    describes a structure type with the specified `<NAME>` and
+    `<FIELDS>`. Fields are allocated at the same offsets used by the
+    implementation's C compiler, as guessed by the SBCL internals.
+    An optional :ALIGNMENT keyword argument can be specified for each
+    field to explicitly control the alignment of a field. If `<NAME>`
+    is NIL then the structure is anonymous.
+
+      If a named foreign STRUCT specifier is passed to
+      DEFINE-ALIEN-TYPE or WITH-ALIEN, then this defines,
+      respectively, a new global or local foreign structure type. If
+      no `<FIELDS>` are specified, then the fields are taken from the
+      current (local or global) alien structure type definition of
+      `<NAME>`.
+
+  - The foreign type specifier `(UNION <NAME> &REST <FIELDS>)` is
+    similar to STRUCT but describes a union type. All fields are
+    allocated at the same offset, and the size of the union is the
+    size of the largest field. The programmer must determine which
+    field is active from context.
+
+  - The foreign type specifier `(ENUM <NAME> &REST <SPECS>)` describes
+    an enumeration type that maps between integer values and symbols.
+    If `<NAME>` is NIL, then the type is anonymous. Each element of
+    the `<SPECS>` list is either a Lisp symbol, or a list
+    `(<symbol> <value>)`. `<value>` is an integer. If `<value>` is not
+    supplied, then it defaults to one greater than the value for the
+    preceding spec (or to zero if it is the first spec).
+
+  - The foreign type specifier `(SIGNED &OPTIONAL <BITS>)` specifies a
+    signed integer with the specified number of `<BITS>` precision.
+    The upper limit on integer precision is determined by the
+    machine's word size. If `<BITS>` is not specified, the maximum
+    size will be used.
+
+  - The foreign type specifier `(INTEGER &OPTIONAL <BITS>)` is
+    equivalent to the corresponding type specifier using SIGNED
+    instead of INTEGER.
+
+  - The foreign type specifier `(UNSIGNED &OPTIONAL <BITS>)` is like
+    corresponding type specifier using SIGNED except that the variable
+    is treated as an unsigned integer.
+
+  - The foreign type specifier `(BOOLEAN &OPTIONAL <BITS>)` is similar
+    to an enumeration type but maps from Lisp NIL and T to C 0 and 1
+    respectively. `<BITS>` determines the amount of storage allocated
+    to hold the truth value.
+
+  - The foreign type specifier `\\SINGLE-FLOAT` describes a
+    floating-point number in IEEE single-precision format.
+
+  - The foreign type specifier `\\DOUBLE-FLOAT` describes a
+    floating-point number in IEEE double-precision format.
+
+  - The foreign type specifier `(FUNCTION <RESULT-TYPE> &REST
+    <ARG-TYPES>)` describes a foreign function that takes arguments of
+    the specified `<ARG-TYPES>` and returns a result of type
+    `<RESULT-TYPE>`. Note that the only context where a foreign
+    `\\FUNCTION` type is directly specified is in the argument to
+    ALIEN-FUNCALL. In all other contexts, foreign functions are
+    represented by foreign function pointer types: `(* (FUNCTION
+    ...))`.
+
+  - The foreign type specifier `\\SYSTEM-AREA-POINTER` describes a
+    pointer which is represented in Lisp as a SYSTEM-AREA-POINTER
+    object. SBCL exports this type from `SB-ALIEN` because CMUCL did,
+    but tentatively (as of the first draft of this section of the
+    manual, SBCL 0.7.6) it is deprecated, since it doesn't seem to be
+    required by user code.
+
+  - The foreign type specifier VOID is used in function types to
+    declare that no useful value is returned. Using ALIEN-FUNCALL to
+    call a VOID foreign function will return zero values.
+
+  - The foreign type specifier `(C-STRING &KEY <EXTERNAL-FORMAT>
+    <ELEMENT-TYPE> <NOT-NULL>)` is similar to `(* CHAR)` but is
+    interpreted as a null-terminated string, and is automatically
+    converted into a Lisp string when accessed; or if the pointer is C
+    `\\NULL` or 0, then accessing it gives Lisp NIL unless
+    `<NOT-NULL>` is true, in which case a TYPE-ERROR is signalled.
+
+      @EXTERNAL-FORMAT conversion is automatically done when Lisp
+      strings are passed to foreign code, or when foreign strings are
+      passed to Lisp code. If the type specifier has an explicit
+      `<EXTERNAL-FORMAT>`, that external format will be used.
+      Otherwise SB-EXT:*DEFAULT-C-STRING-EXTERNAL-FORMAT* will be
+      used. For example, when the following alien routine is called,
+      the Lisp string given as argument is converted to an \\EBCDIC
+      octet representation.
+
+          (define-alien-routine test int (str (c-string :external-format :ebcdic-us)))
+
+      Lisp strings of type BASE-STRING are stored with a trailing
+      `\\\\NUL` termination, so no copying (either by the user or the
+      implementation) is necessary when passing them to foreign code,
+      assuming that the `<EXTERNAL-FORMAT>` and `<ELEMENT-TYPE>` of
+      the C-STRING type are compatible with the internal
+      representation of the string. For an SBCL built with Unicode
+      support that means an `<EXTERNAL-FORMAT>` of :ASCII and an
+      `<ELEMENT-TYPE>` of BASE-CHAR. Without Unicode support the
+      `<EXTERNAL-FORMAT>` can also be :ISO-8859-1, and the
+      `<ELEMENT-TYPE>` can also be [CHARACTER][type]. If
+      `<EXTERNAL-FORMAT>` and `<ELEMENT-TYPE>` are not compatible, or
+      the string is a `(SIMPLE-ARRAY CHARACTER (*))`, this data is
+      copied by the implementation as required.
+
+      Assigning a Lisp string to a C-STRING structure field or
+      variable stores the contents of the string to the memory already
+      pointed to by that variable. When a foreign object of type
+      `(* CHAR)` is assigned to a C-STRING, then the C-STRING pointer
+      is assigned to. This allows C-STRING pointers to be initialized.
+      For example:
+
+          (cl:in-package \"CL-USER\") ; which USEs package \"SB-ALIEN\"
+
+          (define-alien-type nil (struct foo (str c-string)))
+
+          (defun make-foo (str)
+            (let ((my-foo (make-alien (struct foo))))
+              (setf (slot my-foo 'str) (make-alien char (length str))
+                    (slot my-foo 'str) str)
+              my-foo))
+
+      Storing Lisp NIL in a C-STRING writes C `\\\\NULL` to the
+      variable."
+  "- `SB-ALIEN` also exports translations of these C type
+    specifiers as foreign type specifiers:
+
+      CHAR, SHORT, INT, LONG, UNSIGNED-CHAR, UNSIGNED-SHORT,
+      UNSIGNED-INT, UNSIGNED-LONG, FLOAT, DOUBLE, SIZE-T, OFF-T")
+
+(defsection @operations-on-foreign-values
+    (:title "Operations On Foreign Values")
+  "This section describes how to read foreign values as Lisp values,
+  how to coerce foreign values to different kinds of foreign values,
+  and how to dynamically allocate and free foreign variables."
+  (@accessing-foreign-values section)
+  (@coercing-foreign-values section)
+  (@foreign-dynamic-allocation section))
+
+(defsection @accessing-foreign-values (:title "Accessing Foreign Values")
+  (sb-alien:deref function)
+  (sb-alien:slot function)
+  (@untyped-memory section))
+
+(defsection @untyped-memory (:title "Untyped memory")
+  "As noted at the beginning of the chapter, the System Area Pointer
+  facilities allow untyped access to foreign memory. SAPs can be
+  converted to and from the usual typed foreign values using SAP-ALIEN
+  and ALIEN-SAP, and also to and from integers (raw machine
+  addresses). They should thus be used with caution; corrupting the
+  Lisp heap or other memory with SAPs is trivial."
+  (sb-sys:int-sap function)
+  (sb-sys:sap-ref-32 function)
+  (sb-sys:sap= function)
+  "Similarly named functions exist for accessing other sizes of word,
+  other comparisons, and other conversions. The reader is invited to
+  use APROPOS and DESCRIBE for more details:
+
+      (apropos \"sap\" :sb-sys)")
+
+(defsection @coercing-foreign-values (:title "Coercing Foreign Values")
+  (addr macro)
+  (cast macro)
+  (sap-alien macro)
+  (alien-sap function))
+
+(defsection @foreign-dynamic-allocation (:title "Foreign Dynamic Allocation")
+  "Lisp code can call the C standard library functions `malloc`
+  and `free` to dynamically allocate and deallocate foreign variables.
+  The Lisp code uses the same allocator as foreign C code, so it's
+  OK for foreign code to call `free` on the result of Lisp MAKE-ALIEN,
+  or for Lisp code to call FREE-ALIEN on foreign objects allocated by
+  C code."
+  (make-alien macro)
+  (make-alien-string function)
+  (free-alien function))
+
+(defsection @foreign-variables (:title "Foreign Variables")
+  "Both local (stack allocated) and external (C global) foreign
+  variables are supported."
+  (@local-foreign-variables section)
+  (@external-foreign-variables section))
+
+(defsection @local-foreign-variables (:title "Local Foreign Variables")
+  (with-alien macro))
+
+(defsection @external-foreign-variables (:title "External Foreign Variables")
+  "External foreign names are strings, and Lisp names are symbols. When
+  an external foreign value is represented using a Lisp variable, there
+  must be a way to convert from one name syntax into the other. The
+  macros EXTERN-ALIEN, DEFINE-ALIEN-VARIABLE and
+  DEFINE-ALIEN-ROUTINE use this conversion heuristic:
+
+  - Alien names are converted to Lisp names by uppercasing and
+    replacing underscores with hyphens.
+
+  - Conversely, Lisp names are converted to alien names by lowercasing
+    and replacing hyphens with underscores.
+
+  - Both the Lisp symbol and alien string names may be separately
+    specified by using a list of the form
+
+          (<alien-string> <lisp-symbol>)"
+  (define-alien-variable macro)
+  (get-errno function)
+  (extern-alien macro))
+
+(defsection @foreign-data-structure-examples
+    (:title "Foreign Data Structure Examples")
+  "Now that we have alien types, operations and variables, we can
+  manipulate foreign data structures. This C declaration
+
+      struct foo {
+          int a;
+          struct foo *b[100];
+      };
+
+  can be translated into the following alien type:
+
+      (define-alien-type nil
+        (struct foo
+          (a int)
+          (b (array (* (struct foo)) 100))))
+
+  Once the `FOO` alien type has been defined as above, the C
+  expression
+
+      struct foo f;
+      f.b[7].a;
+
+  can be translated in this way:
+
+      (with-alien ((f (struct foo)))
+        (slot (deref (slot f 'b) 7) 'a)
+        ;;
+        ;; Do something with f...
+        )
+
+  Or consider this example of an external C variable and some accesses:
+
+      struct c_struct {
+              short x, y;
+              char a, b;
+              int z;
+              c_struct *n;
+      };
+      extern struct c_struct *my_struct;
+      my_struct->x++;
+      my_struct->a = 5;
+      my_struct = my_struct->n;
+
+  which can be manipulated in Lisp like this:
+
+      (define-alien-type nil
+        (struct c-struct
+                (x short)
+                (y short)
+                (a char)
+                (b char)
+                (z int)
+                (n (* c-struct))))
+      (define-alien-variable \"my_struct\" (* c-struct))
+      (incf (slot my-struct 'x))
+      (setf (slot my-struct 'a) 5)
+      (setq my-struct (slot my-struct 'n))")
+
+(defsection @loading-shared-object-files (:title "Loading Shared Object Files")
+  "Foreign object files can be loaded into the running Lisp process by
+  calling LOAD-SHARED-OBJECT."
+  (load-shared-object function)
+  (unload-shared-object function))
+
+(defsection @foreign-function-calls (:title "Foreign Function Calls")
+  "The foreign function call interface allows a Lisp program to call
+  many functions written in languages that use the C calling convention.
+
+  Lisp sets up various signal handling routines and other environment
+  information when it first starts up, and expects these to be in
+  place at all times. The C functions called by Lisp should not change
+  the environment, especially the signal handlers: the signal handlers
+  installed by Lisp typically have interesting flags set (e.g to
+  request machine context information, or for signal delivery on an
+  alternate stack) which the Lisp runtime relies on for correct
+  operation. Precise details of how this works may change without
+  notice between versions; the source, or the brain of a friendly SBCL
+  developer, is the only documentation. Users of a Lisp built with the
+  :SB-THREAD feature should also read the section about threads,
+  @THREADING."
+  (alien-funcall function)
+  (alien-funcall-into function)
+  (define-alien-routine macro))
+
+;; <!-- FIXME: This is a \"changebar\" section from the CMU CL manual.
+;;      I (WHN 2002-07-14) am not very familiar with this content, so
+;;      I'm not immediately prepared to try to update it for SBCL, and
+;;      I'm not feeling masochistic enough to work to encourage this
+;;      kind of low-level hack anyway. However, I acknowledge that callbacks
+;;      are sometimes really really necessary, so I include the original
+;;      text in case someone is hard-core enough to benefit from it. If
+;;      anyone brings the information up to date for SBCL, it belong
+;;      either in the main manual or on a CLiki SBCL Internals page.
+;; LaTeX \subsection{Accessing Lisp Arrays}
+;; LaTeX
+;; LaTeX Due to the way \cmucl{} manages memory, the amount of memory that can
+;; LaTeX be dynamically allocated by \code{malloc} or \funref{make-alien} is
+;; LaTeX limited\footnote{\cmucl{} mmaps a large piece of memory for it's own
+;; LaTeX   use and this memory is typically about 8 MB above the start of the C
+;; LaTeX   heap.  Thus, only about 8 MB of memory can be dynamically
+;; LaTeX   allocated.}.
+;;
+;; Empirically determined to be considerably >8Mb on this x86 linux
+;; machine, but I don't know what the actual values are - dan 2003.09.01
+;;
+;; Note that this technique is used in SB-GROVEL in the SBCL contrib
+;;
+;; LaTeX
+;; LaTeX To overcome this limitation, it is possible to access the content of
+;; LaTeX Lisp arrays which are limited only by the amount of physical memory
+;; LaTeX and swap space available.  However, this technique is only useful if
+;; LaTeX the foreign function takes pointers to memory instead of allocating
+;; LaTeX memory for itself.  In latter case, you will have to modify the
+;; LaTeX foreign functions.
+;; LaTeX
+;; LaTeX This technique takes advantage of the fact that \cmucl{} has
+;; LaTeX specialized array types (\pxlref{specialized-array-types}) that match
+;; LaTeX a typical C array.  For example, a \code{(simple-array double-float
+;; LaTeX   (100))} is stored in memory in essentially the same way as the C
+;; LaTeX array \code{double x[100]} would be.  The following function allows us
+;; LaTeX to get the physical address of such a Lisp array:
+;; LaTeX \begin{example}
+;; LaTeX (defun array-data-address (array)
+;; LaTeX   \"Return the physical address of where the actual data of an array is
+;; LaTeX stored.
+;; LaTeX
+;; LaTeX ARRAY must be a specialized array type in CMU Lisp.  This means ARRAY
+;; LaTeX must be an array of one of the following types:
+;; LaTeX
+;; LaTeX                   double-float
+;; LaTeX                   single-float
+;; LaTeX                   (unsigned-byte 32)
+;; LaTeX                   (unsigned-byte 16)
+;; LaTeX                   (unsigned-byte  8)
+;; LaTeX                   (signed-byte 32)
+;; LaTeX                   (signed-byte 16)
+;; LaTeX                   (signed-byte  8)
+;; LaTeX \"
+;; LaTeX   (declare (type (or #+signed-array (array (signed-byte 8))
+;; LaTeX                      #+signed-array (array (signed-byte 16))
+;; LaTeX                      #+signed-array (array (signed-byte 32))
+;; LaTeX                      (array (unsigned-byte 8))
+;; LaTeX                      (array (unsigned-byte 16))
+;; LaTeX                      (array (unsigned-byte 32))
+;; LaTeX                      (array single-float)
+;; LaTeX                      (array double-float))
+;; LaTeX                  array)
+;; LaTeX            (optimize (speed 3) (safety 0))
+;; LaTeX            (ext:optimize-interface (safety 3)))
+;; LaTeX   ;; with-array-data will get us to the actual data.  However, because
+;; LaTeX   ;; the array could have been displaced, we need to know where the
+;; LaTeX   ;; data starts.
+;; LaTeX   (lisp::with-array-data ((data array)
+;; LaTeX                           (start)
+;; LaTeX                           (end))
+;; LaTeX     (declare (ignore end))
+;; LaTeX     ;; DATA is a specialized simple-array.  Memory is laid out like this:
+;; LaTeX     ;;
+;; LaTeX     ;;   byte offset    Value
+;; LaTeX     ;;        0         type code (should be 70 for double-float vector)
+;; LaTeX     ;;        4         4 * number of elements in vector
+;; LaTeX     ;;        8         1st element of vector
+;; LaTeX     ;;      ...         ...
+;; LaTeX     ;;
+;; LaTeX     (let ((addr (+ 8 (logandc1 7 (kernel:get-lisp-obj-address data))))
+;; LaTeX           (type-size (let ((type (array-element-type data)))
+;; LaTeX                        (cond ((or (equal type '(signed-byte 8))
+;; LaTeX                                   (equal type '(unsigned-byte 8)))
+;; LaTeX                               1)
+;; LaTeX                              ((or (equal type '(signed-byte 16))
+;; LaTeX                                   (equal type '(unsigned-byte 16)))
+;; LaTeX                               2)
+;; LaTeX                              ((or (equal type '(signed-byte 32))
+;; LaTeX                                   (equal type '(unsigned-byte 32)))
+;; LaTeX                               4)
+;; LaTeX                              ((equal type 'single-float)
+;; LaTeX                               4)
+;; LaTeX                              ((equal type 'double-float)
+;; LaTeX                               8)
+;; LaTeX                              (t
+;; LaTeX                               (error \"Unknown specialized array element type\"))))))
+;; LaTeX       (declare (type (unsigned-byte 32) addr)
+;; LaTeX                (optimize (speed 3) (safety 0) (ext:inhibit-warnings 3)))
+;; LaTeX       (system:int-sap (the (unsigned-byte 32)
+;; LaTeX                         (+ addr (* type-size start)))))))
+;; LaTeX \end{example}
+;; LaTeX
+;; LaTeX Assume we have the C function below that we wish to use:
+;; LaTeX \begin{example}
+;; LaTeX   double dotprod(double* x, double* y, int n)
+;; LaTeX   \{
+;; LaTeX     int k;
+;; LaTeX     double sum = 0;
+;; LaTeX
+;; LaTeX     for (k = 0; k < n; ++k) \{
+;; LaTeX       sum += x[k] * y[k];
+;; LaTeX     \}
+;; LaTeX   \}
+;; LaTeX \end{example}
+;; LaTeX The following example generates two large arrays in Lisp, and calls the C
+;; LaTeX function to do the desired computation.  This would not have been
+;; LaTeX possible using \code{malloc} or \code{make-alien} since we need about
+;; LaTeX 16 MB of memory to hold the two arrays.
+;; LaTeX \begin{example}
+;; LaTeX   (define-alien-routine \"dotprod\" double
+;; LaTeX     (x (* double-float) :in)
+;; LaTeX     (y (* double-float) :in)
+;; LaTeX     (n int :in))
+;; LaTeX
+;; LaTeX   (let ((x (make-array 1000000 :element-type 'double-float))
+;; LaTeX         (y (make-array 1000000 :element-type 'double-float)))
+;; LaTeX     ;; Initialize X and Y somehow
+;; LaTeX     (let ((x-addr (system:int-sap (array-data-address x)))
+;; LaTeX           (y-addr (system:int-sap (array-data-address y))))
+;; LaTeX       (dotprod x-addr y-addr 1000000)))
+;; LaTeX \end{example}
+;; LaTeX In this example, it may be useful to wrap the inner \code{let}
+;; LaTeX expression in an \code{unwind-protect} that first turns off garbage
+;; LaTeX collection and then turns garbage collection on afterwards.  This will
+;; LaTeX prevent garbage collection from moving \code{x} and \code{y} after we
+;; LaTeX have obtained the (now erroneous) addresses but before the call to
+;; LaTeX \code{dotprod} is made.
+;; LaTeX
+
+
+(defsection @calling-lisp-from-c (:title "Calling Lisp From C")
+  "SBCL supports the calling of Lisp functions using the C calling
+  convention. This is useful for both defining callbacks and for creating
+  an interface for calling into Lisp as a shared library directly from C.
+
+  The DEFINE-ALIEN-CALLABLE macro wraps Lisp code and creates a C
+  foreign function which can be called with the C calling convention.
+  On x86-64 and ARM64, callbacks may receive and return structures by
+  value."
+  (define-alien-callable macro)
+  "The ALIEN-CALLABLE-FUNCTION function returns the foreign callable
+  value associated with any name defined by DEFINE-ALIEN-CALLABLE, so
+  that we can, for example, pass the callable value to C as a
+  callback."
+  (alien-callable-function function)
+  "The WITH-ALIEN-CALLABLE macro wraps Lisp code and establishes
+  local C foreign functions which can be called with the C calling
+  convention. This macro is handy for passing callbacks which close over
+  Lisp values into C."
+  (with-alien-callable macro)
+  "Note that the garbage collector moves objects, and won't be able to fix
+  up any references in C variables. There are three mechanisms for
+  coping with this:
+
+  - SB-EXT:PURIFY moves all live Lisp data into static or read-only
+    areas such that it will never be moved (or freed) again in the
+    life of the Lisp session
+
+  - SB-SYS:WITH-PINNED-OBJECTS is a macro which arranges for some set
+    of objects to be pinned in memory for the dynamic extent of its
+    body forms. On ports which use the @GENERATIONAL-GC
+    (most, as of this writing) this affects exactly the specified
+    objects. On other ports it is implemented by turning off GC for
+    the duration (so could be said to have a whole-world granularity).
+
+  - Disable GC, using the SB-EXT:WITHOUT-GCING macro."
+  (@lisp-as-a-shared-library section))
+
+(defsection @lisp-as-a-shared-library (:title "Lisp as a Shared Library")
+  "SBCL supports the use of Lisp as a shared library that can be used by
+  C programs using the DEFINE-ALIEN-CALLABLE interface. See the
+  :CALLABLE-EXPORTS argument of SB-EXT:SAVE-LISP-AND-DIE for how to
+  save the Lisp image in a way that allows a C program to initialize
+  the Lisp runtime and the exported symbols. When SBCL is built as a
+  library, it exposes the symbol `initialize_lisp` which can be used
+  in conjunction with a core initializing global symbols to foreign
+  callables as function pointers and with object code allocating those
+  symbols to initialize the runtime properly. The arguments to
+  `initialize_lisp` are the same as the arguments to the main `sbcl`
+  program.
+
+  > _Note_: There is currently no way to run exit hooks or otherwise
+  > undo Lisp initialization gracefully from C.")
+
+(defsection @step-by-step-example-of-the-foreign-function-interface
+    (:title "Step-By-Step Example of the Foreign Function Interface")
+  "This section presents a complete example of an interface to a somewhat
+  complicated C function.
+
+  Suppose you have the following C function which you want to be able
+  to call from Lisp in the file `test.c`:
+
+      struct c_struct
+      {
+        int x;
+        char *s;
+      };
+
+      struct c_struct *c_function (i, s, r, a)
+          int i;
+          char *s;
+          struct c_struct *r;
+          int a[10];
+      {
+        int j;
+        struct c_struct *r2;
+
+        printf(\"i = %d\n\", i);
+        printf(\"s = %s\n\", s);
+        printf(\"r->x = %d\n\", r->x);
+        printf(\"r->s = %s\n\", r->s);
+        for (j = 0; j < 10; j++) printf(\"a[%d] = %d.\n\", j, a[j]);
+        r2 = (struct c_struct *) malloc (sizeof(struct c_struct));
+        r2->x = i + 5;
+        r2->s = \"a C string\";
+        return(r2);
+      };
+
+  It is possible to call this C function from Lisp using the file
+  `test.lisp` containing
+
+      (cl:defpackage \"TEST-C-CALL\" (:use \"CL\" \"SB-ALIEN\" \"SB-C-CALL\"))
+      (cl:in-package \"TEST-C-CALL\")
+
+      ;;; Define the record C-STRUCT in Lisp.
+      (define-alien-type nil
+          (struct c-struct
+                  (x int)
+                  (s c-string)))
+
+      ;;; Define the Lisp function interface to the C routine.  It returns a
+      ;;; pointer to a record of type C-STRUCT.  It accepts four parameters:
+      ;;; I, an int; S, a pointer to a string; R, a pointer to a C-STRUCT
+      ;;; record; and A, a pointer to the array of 10 ints.
+      ;;;
+      ;;; The INLINE declaration eliminates some efficiency notes about heap
+      ;;; allocation of alien values.
+      (declaim (inline c-function))
+      (define-alien-routine c-function
+          (* (struct c-struct))
+        (i int)
+        (s c-string)
+        (r (* (struct c-struct)))
+        (a (array int 10)))
+
+      ;;; a function which sets up the parameters to the C function and
+      ;;; actually calls it
+      (defun call-cfun ()
+        (with-alien ((ar (array int 10))
+                     (c-struct (struct c-struct)))
+          (dotimes (i 10)                     ; Fill array.
+            (setf (deref ar i) i))
+          (setf (slot c-struct 'x) 20)
+          (setf (slot c-struct 's) \"a Lisp string\")
+
+          (with-alien ((res (* (struct c-struct))
+                            (c-function 5 \"another Lisp string\" (addr c-struct) ar)))
+            (format t \"~&back from C function~%\")
+            (multiple-value-prog1
+                (values (slot res 'x)
+                        (slot res 's))
+
+              ;; Deallocate result. (after we are done referring to it:
+              ;; \"Pillage, *then* burn.\")
+              (free-alien res)))))
+
+  To execute the above example, it is necessary to compile the C
+  routine, e.g. with `cc -c test.c && ld -shared -o test.so test.o`.
+  In order to enable incremental loading with some linkers, you may
+  need to say `cc -G 0 -c test.c`.
+
+  Once the C code has been compiled, you can start up Lisp and load it
+  in: `sbcl`. Lisp should start up with its normal prompt.
+
+  Within Lisp, compile the Lisp file:
+
+      (compile-file \"test.lisp\")
+
+  This step can be done separately. You don't have to recompile every
+  time.
+
+  Within Lisp, load the foreign object file to define the necessary
+  symbols:
+
+      (load-shared-object \"test.so\")
+
+  Now you can load the compiled Lisp (fasl) file into Lisp:
+
+      (load \"test.fasl\")
+
+  And once the Lisp file is loaded, you can call the Lisp routine
+  that sets up the parameters and calls the C function:
+
+      (test-c-call::call-cfun)
+
+  The C routine should print the following information to standard output:
+
+      i = 5
+      s = another Lisp string
+      r->x = 20
+      r->s = a Lisp string
+      a[0] = 0.
+      a[1] = 1.
+      a[2] = 2.
+      a[3] = 3.
+      a[4] = 4.
+      a[5] = 5.
+      a[6] = 6.
+      a[7] = 7.
+      a[8] = 8.
+      a[9] = 9.
+
+  After return from the C function,
+  the Lisp wrapper function should print the following output:
+
+      back from C function
+
+  And upon return from the Lisp wrapper function,
+  before the next prompt is printed, the
+  Lisp read-eval-print loop should print the following return values:
+
+      10
+      \"a C string\"")

contrib/sb-manual/doc/intro.lisp

@@ -0,0 +1,502 @@
+(in-package :sb-manual)
+
+(defsection @introduction (:title "Introduction")
+  "SBCL is a mostly-conforming implementation of the ANSI Common Lisp
+  standard. This manual focuses on behavior which is specific to SBCL,
+  not on behavior which is common to all implementations of ANSI Common
+  Lisp."
+  (@ansi-conformance section)
+  (@extensions section)
+  (@idiosyncrasies section)
+  (@development-tools section)
+  (@more-sbcl-information section)
+  (@more-common-lisp-information section)
+  (@history-and-implementation-of-sbcl section))
+
+(defsection @ansi-conformance (:title "ANSI Conformance")
+  "Essentially every type of non-conformance is considered a bug. (The
+  exceptions involve internal inconsistencies in the standard.) See
+  @REPORTING-BUGS.
+
+  - PROG2 returns the primary value of its second form, as
+    specified in the _Arguments and Values_ section of the
+    specification for that operator, not that of its first form, as
+    specified in the _Description_.
+
+  - The STRING type is considered to be the union of all types
+    `(ARRAY C (SIZE))` for all non-`NIL` subtypes `C` of
+     [CHARACTER][type], excluding arrays specialized to the empty
+     type.
+
+  - The `:ORDER` long form option in DEFINE-METHOD-COMBINATION method
+    group specifiers accepts the value NIL as well as
+    :MOST-SPECIFIC-FIRST and :MOST-SPECIFIC-LAST, in order to allow
+    programmers to declare that the order of methods playing that role
+    in the method combination does not matter.")
+
+;;; FIXME: Document SERVE-EVENT?
+(defsection @extensions (:title "Extensions")
+  "SBCL comes with numerous extensions, some in core and some in modules
+  loadable with REQUIRE. Unfortunately, not all of these extensions
+  have proper documentation yet.
+
+  - __System Definition Tool:__ ASDF is a flexible and popular
+    protocol-oriented system definition tool by Daniel Barlow.
+
+  - __Foreign Function Interface:__ The `SB-ALIEN` package allows
+    interfacing with C-code, loading shared object files, etc. See
+    @FOREIGN-FUNCTION-INTERFACE.
+
+      @SB-GROVEL can be used to partially automate generation of
+      foreign function interface definitions.
+
+  - __Recursive Event Loop:__ SBCL provides a recursive event
+    loop (`SERVE-EVENT`) for doing non-blocking IO on multiple streams
+    without using threads.
+
+  - __Timeouts and Deadlines:__ SBCL allows restricting the execution
+    time of individual operations or parts of a computation using
+    :TIMEOUT arguments to certain blocking operations, synchronous
+    timeouts and asynchronous timeouts. The latter two affect operations
+    without explicit timeout support (such as standard functions and
+    macros). See @TIMEOUTS-AND-DEADLINES.
+
+  - __Metaobject Protocol:__ The `SB-MOP` package provides an
+    implementation of the metaobject protocol for the Common Lisp
+    Object System as described in _The Art of the Metaobject Protocol_
+    by Kiczales et al.
+
+  - __Extensible Sequences:__ SBCL allows users to define subclasses
+    of the SEQUENCE class. See @EXTENSIBLE-SEQUENCES.
+
+  - __Native Threads:__ SBCL has native threads on numerous platforms,
+    capable of taking advantage of SMP on multiprocessor machines. See
+    @THREADING.
+
+  - __Network Interface:__ The `SB-BSD-SOCKETS` module is a low-level
+    networking interface, providing both TCP and UDP sockets. See
+    @NETWORKING.
+
+  - __Introspective Facilities:__ The @SB-INTROSPECT module offers
+    numerous introspective extensions, including access to function
+    lambda-lists and a cross referencing facility.
+
+  - __Operating System Interface:__ The `SB-EXT` package contains a
+    number of functions for running external processes, accessing
+    environment variables, etc.
+
+      The @SB-POSIX module provides a lispy interface to standard
+      POSIX facilities.
+
+  - __Extensible Streams:__ The package `SB-GRAY` provides an
+    implementation of @GRAY-STREAMS.
+
+      The @SB-SIMPLE-STREAMS module is an implementation of the Simple
+      Streams API proposed by Franz Inc.
+
+  - __Profiling:__ The `SB-PROFILE` package provides an exact,
+    per-function @DETERMINISTIC-PROFILER.
+
+      The `SB-SPROF` module is SBCL's @STATISTICAL-PROFILER, capable
+      of call-graph generation and instruction level profiling, which
+      also supports allocation profiling.
+
+  - __Customization Hooks:__ SBCL contains a number of extra-standard
+    customization hooks that can be used to tweak the behaviour of the
+    system. See @CUSTOMIZATION-HOOKS-FOR-USERS.
+
+  - __sb-aclrepl:__ The @SB-ACLREPL module provides an Allegro-style
+    toplevel for SBCL, as an alternative to the classic CMUCL-style
+    one.
+
+  - __CLTL2 Compatibility Layer:__ The SB-CLTL2 module provides
+    SB-CLTL2:COMPILER-LET and environment access functionality
+    described in _Common Lisp The Language, 2nd Edition_ which were
+    removed from the language during the ANSI standardization process.
+
+  - __Executable Delivery:__ The :EXECUTABLE argument to
+    SB-EXT:SAVE-LISP-AND-DIE can produce a \"standalone\" executable
+    containing both an image of the current Lisp session and an SBCL
+    runtime.
+
+  - __Bitwise Rotation:__ The @SB-ROTATE-BYTE module provides an
+    efficient primitive for bitwise rotation of integers, an operation
+    required by e.g. numerous cryptographic algorithms but not
+    available as a primitive in ANSI Common Lisp.
+
+  - __Test Harness:__ The `SB-RT` module is a simple yet attractive
+    regression and unit-test framework.
+
+  - __MD5 Sums:__ The @SB-MD5 module provides an implementation of the
+    MD5 message digest algorithm for Common Lisp, using the
+    @MODULAR-ARITHMETIC optimizations provided by SBCL.")
+
+(defsection @idiosyncrasies (:title "Idiosyncrasies")
+  "The information in this section describes some of the ways that SBCL
+  deals with choices that the ANSI standard leaves to the
+  implementation."
+  (@declarations section)
+  (@fasl-format section)
+  (@compiler-only-implementation section)
+  (@defining-constants section)
+  (@style-warnings section))
+
+(defsection @declarations (:title "Declarations")
+  "Declarations are generally treated as assertions. This general
+  principle, and its implications, and the bugs which still keep the
+  compiler from quite satisfying this principle, are discussed in
+  @DECLARATIONS-AS-ASSERTIONS.")
+
+(defsection @fasl-format (:title "FASL format")
+  "SBCL fasl-format is binary compatible only with the exact SBCL version
+  it was generated with. While this is obviously suboptimal, it has
+  proven more robust than trying to maintain fasl compatibility across
+  versions: accidentally breaking things is far too easy, and can lead
+  to hard to diagnose bugs.
+
+  The following snippet handles fasl recompilation automatically for
+  ASDF-based systems, and makes a good candidate for inclusion in the
+  user or system initialization file (see @INITIALIZATION-FILES).
+
+      (require :asdf)
+
+      ;;; If a fasl was stale, try to recompile and load (once).
+      (defmethod asdf:perform :around ((o asdf:load-op)
+                                       (c asdf:cl-source-file))
+         (handler-case (call-next-method o c)
+            ;; If a fasl was stale, try to recompile and load (once).
+            (sb-ext:invalid-fasl ()
+               (asdf:perform (make-instance 'asdf:compile-op) c)
+               (call-next-method))))")
+
+(defsection @compiler-only-implementation
+    (:title "Compiler-only Implementation")
+  "SBCL is essentially a compiler-only implementation of Common Lisp.
+  That is, for all but a few special cases, EVAL creates a lambda
+  expression, calls COMPILE on the lambda expression to create a
+  compiled function, and then calls FUNCALL on the resulting function
+  object. A more traditional interpreter is also available on default
+  builds; it is usually only called internally. This is explicitly
+  allowed by the ANSI standard but leads to some oddities; e.g. at
+  default settings, FUNCTIONP and COMPILED-FUNCTION-P are equivalent,
+  and they collapse into the same function when SBCL is built without
+  the interpreter.")
+
+(defsection @defining-constants (:title "Defining Constants")
+  "SBCL is quite strict about ANSI's definition of DEFCONSTANT.
+  ANSI says that doing DEFCONSTANT of the same symbol more than once
+  is undefined unless the new value is EQL to the old value.
+  Conforming to this specification is a nuisance when the \"constant\"
+  value is only constant under some weaker test like STRING= or EQUAL.
+
+  It's especially annoying because, in SBCL, DEFCONSTANT takes effect
+  not only at load time but also at compile time, so that just
+  compiling and loading reasonable code like
+
+      (defconstant +foobyte+ '(1 4))
+
+  runs into this undefined behavior. Many implementations of Common
+  Lisp try to help the programmer around this annoyance by silently
+  accepting the undefined code and trying to do what the programmer
+  probably meant.
+
+  SBCL instead treats the undefined behavior as an error. Often such
+  code can be rewritten in portable ANSI Common Lisp which has the
+  desired behavior. E.g., the code above can be given an exactly
+  defined meaning by replacing DEFCONSTANT either with DEFPARAMETER or
+  with a customized macro which does the right thing, e.g.
+
+      (defmacro define-constant (name value &optional doc)
+        `(defconstant ,name (if (boundp ',name) (symbol-value ',name) ,value)
+                            ,@(when doc (list doc))))
+
+  or possibly along the lines of the SB-INT:DEFCONSTANT-EQX macro used
+  internally in the implementation of SBCL itself. In circumstances
+  where this is not appropriate, the programmer can handle the
+  condition type SB-EXT:DEFCONSTANT-UNEQL and choose either the
+  CONTINUE restart or ABORT restart as appropriate.")
+
+(defsection @style-warnings (:title "Style Warnings")
+  "SBCL gives style warnings about various kinds of perfectly legal code,
+  e.g.
+
+  - multiple DEFUNs of the same symbol in different units;
+
+  - special variables not named in the conventional `*foo*` style, and
+    lexical variables unconventionally named in the `*FOO*` style.
+
+  This causes friction with people who point out that other ways of
+  organizing code (especially avoiding the use of DEFGENERIC) are just
+  as aesthetically stylish. However, these warnings should be read not
+  as _warning, bad aesthetics detected, you have no style_ but as
+  _warning, this style keeps the compiler from understanding the code
+  as well as you might like_. That is, unless the compiler warns about
+  such conditions, there's no way for the compiler to warn about some
+  programming errors which would otherwise be easy to
+  overlook. (Related bug: The warning about multiple DEFUNs is
+  pointlessly annoying when you compile and then load a function
+  containing DEFUN wrapped in EVAL-WHEN, and ideally should be
+  suppressed in that case, but still isn't as of SBCL 0.7.6.)")
+
+(defsection @development-tools (:title "Development Tools")
+  (@editor-integration section)
+  (@language-reference section)
+  (@generating-executables section))
+
+(defsection @editor-integration (:title "Editor Integration")
+  "Though SBCL can be used running \"bare\", the recommended mode of
+  development is with an editor connected to SBCL, supporting not
+  only basic lisp editing (paren-matching, etc), but providing among
+  other features an integrated debugger, interactive compilation, and
+  automated documentation lookup.
+
+  Currently _SLIME_ (Superior Lisp Interaction Mode for Emacs)
+  together with Emacs is recommended for use with SBCL, though other
+  options exist as well. Historically, the ILISP package at
+  <http://ilisp.cons.org/> provided similar functionality, but it does
+  not support modern SBCL versions.
+
+  SLIME can be downloaded from <https://slime.common-lisp.dev/>.")
+
+(defsection @language-reference (:title "Language Reference")
+  "_\\CLHS_ (Common Lisp HyperSpec) is a hypertext version of the ANSI
+  standard, made freely available by LispWorks -- an invaluable
+  reference.
+
+  See <https://www.lispworks.com/documentation/HyperSpec/Front/index.htm>.")
+
+(defsection @generating-executables (:title "Generating Executables")
+  "SBCL can generate stand-alone executables. The generated executables
+  include the SBCL runtime itself, so no restrictions are placed on
+  program functionality. For example, a deployed program can call
+  COMPILE and LOAD, which requires the compiler to be present in the
+  executable. For further information, SB-EXT:SAVE-LISP-AND-DIE.")
+
+(defsection @more-sbcl-information (:title "More SBCL Information")
+  (@sbcl-homepage section)
+  (@online-documentation section)
+  (@additional-documentation-files section)
+  (@internals-documentation section))
+
+(defsection @sbcl-homepage (:title "SBCL Homepage")
+  "The SBCL website at <http://www.sbcl.org/> has some general
+  information, plus links to mailing lists devoted to SBCL, and to
+  archives of these mailing lists. Subscribing to the mailing lists
+  `sbcl-help` and `sbcl-announce` is recommended: both are fairly
+  low-volume, and help you keep abreast with SBCL development.")
+
+(defsection @online-documentation (:title "Online Documentation")
+  "Documentation for non-ANSI extensions for various commands is
+  available online from the SBCL executable itself. The extensions for
+  functions which have their own command prompts (e.g. the debugger,
+  and INSPECT) are documented in text available by typing `help` at
+  their command prompts. The extensions for functions which don't have
+  their own command prompt (such as TRACE) are described in their
+  documentation strings, unless your SBCL was compiled with an option
+  not to include documentation strings, in which case the
+  documentation strings are only readable in the source code.")
+
+(defsection @additional-documentation-files
+    (:title "Additional Documentation Files")
+  "Besides this user manual both SBCL source and binary distributions
+  include some other SBCL-specific documentation files, which should
+  be installed along with this manual on your system, e.g. in
+  `/usr/local/share/doc/sbcl/`.
+
+  - `COPYING`: Licence and copyright summary.
+
+  - `CREDITS`: Authorship information on various parts of SBCL.
+
+  - `INSTALL`: Covers installing SBCL from both source and binary
+     distributions on your system, and also has some installation
+     related troubleshooting information.
+
+  - `NEWS`: Summarizes changes between various SBCL versions.")
+
+(defsection @internals-documentation (:title "Internals Documentation")
+  "If you're interested in the development of the SBCL system itself,
+  then subscribing to
+  [sbcl-devel@lists.sourceforge.net](mailto:sbcl-devel@lists.sourceforge.net)
+  is a good idea.
+
+  SBCL internals documentation -- besides comments in the source -- is
+  available in the Web Archive:
+
+  <https://web.archive.org/web/20120814000933/http://sbcl-internals.cliki.net/index>.
+
+  Some low-level information describing the programming details of the
+  conversion from CMUCL to SBCL is available in the
+  `doc/FOR-CMUCL-DEVELOPERS` file.")
+
+(defsection @more-common-lisp-information
+    (:title "More Common Lisp Information")
+  (@internet-community section)
+  (@third-party-libraries section)
+  (@common-lisp-books section))
+
+(defsection @internet-community (:title "Internet Community")
+  "IRC channels on <https://libera.chat/>:
+
+  - `#common-lisp`: \"Common Lisp, the #1=(programmable . #1#)
+    programming language\"
+
+  - `#lispcafe`: \"The Lisp Café; sit down, have a drink, chat about
+    anything, and enjoy your stay. | <https://www.cliki.net/lispcafe> |
+    Be insuperable to each other\".
+
+  - `#sbcl`: \"Steel Bank Common Lisp Dev Hangout\"
+
+  You can use <https://web.libera.chat> or a normal IRC client.
+
+  Also, see <https://www.reddit.com/r/Common_Lisp/>, as well as
+  <https://www.lisp.org> and <https://cliki.net>, which contain
+  numerous pointers places in the net where lispers talks shop.")
+
+(defsection @third-party-libraries (:title "Third-party Libraries")
+  "For a wealth of information about free Common Lisp libraries and tools
+  we recommend checking out _CLiki_: <https://cliki.net/>.
+
+  The most popular library manager is Quicklisp:
+  <https://www.quicklisp.org/beta/>.")
+
+(defsection @common-lisp-books (:title "Common Lisp Books")
+  "If you're not a programmer and you're trying to learn, many
+  introductory Lisp books are available. However, we don't have any
+  standout favorites.
+
+  If you are an experienced programmer in other languages but need to
+  learn about Common Lisp, some books stand out:
+
+  - Practical Common Lisp, by Peter Seibel
+
+      An excellent introduction to the language, covering both the
+      basics and \"advanced topics\" like macros, CLOS, and packages.
+      Available both in print format and on the web:
+      <https://gigamonkeys.com/book/>.
+
+  - Paradigms Of Artificial Intelligence Programming, by Peter Norvig
+
+      Good information on general Common Lisp programming, and many
+      nontrivial examples. Whether or not your work is AI, it's a very
+      good book to look at.
+
+  - On Lisp, by Paul Graham
+
+      An in-depth treatment of macros, but not recommended as a first
+      Common Lisp book, since it is slightly pre-ANSI so you need to
+      be on your guard against non-standard usages, and since it
+      doesn't really even try to cover the language as a whole,
+      focusing solely on macros. Downloadable from
+      <https://www.paulgraham.com/onlisp.html>.
+
+  - Object-Oriented Programming In Common Lisp, by Sonya Keene
+
+      With the exception of _Practical Common Lisp_, most introductory
+      books don't emphasize CLOS. This one does. Even if you're very
+      knowledgeable about object oriented programming in the abstract,
+      it's worth looking at this book if you want to do any OO in
+      Common Lisp. Some abstractions in CLOS (especially multiple
+      dispatch) go beyond anything you'll see in most OO systems, and
+      there are a number of lesser differences as well. This book
+      tends to help with the culture shock.
+
+  - Art Of Metaobject Programming, by Gregor Kiczales et al.
+
+      Currently the prime source of information on the Common Lisp
+      Metaobject Protocol, which is supported by SBCL. Section
+      2 (Chapters 5 and 6) are freely available at
+      <http://mop.lisp.se/www.alu.org/mop/>.")
+
+(defsection @history-and-implementation-of-sbcl
+    (:title "History and Implementation of SBCL")
+  "You can work productively with SBCL without knowing or understanding
+  anything about where it came from, how it is implemented, or how it
+  extends the ANSI Common Lisp standard. However, a little knowledge
+  can be helpful in order to understand error messages, to
+  troubleshoot problems, to understand why some parts of the system
+  are better debugged than others, and to anticipate which known bugs,
+  known performance problems, and missing extensions are likely to be
+  fixed, tuned, or added.
+
+  SBCL is descended from CMUCL, which is itself descended from Spice
+  Lisp, including early implementations for the Mach operating system on
+  the IBM RT, back in the 1980s. Some design decisions from that time are
+  still reflected in the current implementation:
+
+  - The system expects to be loaded into a fixed-at-compile-time
+    location in virtual memory, and also expects the location of all
+    of its heap storage to be specified at compile time.
+
+  - The system overcommits memory, allocating large amounts of address
+    space from the system (often more than the amount of virtual
+    memory available) and then failing if it ends up using too much of
+    the allocated storage.
+
+  - The system is implemented as a C program which is responsible for
+    supplying low-level services and loading a Lisp `.core` file.
+
+  SBCL also inherited some newer architectural features from CMUCL.
+  The most important is that on some architectures it has a
+  @GENERATIONAL-GC, which has various implications (mostly good) for
+  performance. These are discussed in another chapter, @EFFICIENCY.
+
+  SBCL has diverged from CMUCL in that SBCL is now essentially a
+  compiler-only implementation of Common Lisp. This is a change in
+  implementation strategy, taking advantage of the freedom \"any of
+  these facilities might share the same execution strategy\"
+  guaranteed in CLHS `3.1` (Evaluation). It does not mean SBCL can't
+  be used interactively, and in fact the change is largely invisible
+  to the casual user, since SBCL still can and does execute code
+  interactively by compiling it on the fly. (It is visible if you know
+  how to look, like using COMPILED-FUNCTION-P; and it is visible in
+  the way that SBCL doesn't have many bugs which behave differently in
+  interpreted code than in compiled code.) What it means is that in
+  SBCL, the EVAL function only truly \"interprets\" a few easy kinds
+  of forms, such as symbols which are BOUNDP. More complicated forms
+  are evaluated by calling COMPILE and then calling FUNCALL on the
+  returned result.
+
+  The direct ancestor of SBCL is the x86 port of CMUCL. This port was in
+  some ways the most cobbled-together of all the CMUCL ports, since a
+  number of strange changes had to be made to support the register-poor
+  x86 architecture. Some things (like tracing and debugging) do not work
+  particularly well there. SBCL should be able to improve in these areas
+  (and has already improved in some other areas), but it takes a while.
+
+  On the x86 SBCL -- like the x86 port of CMUCL -- uses a
+  _@CONSERVATIVE-GC_. This means that it doesn't maintain a strict
+  separation between tagged and untagged data, instead treating some
+  untagged data (e.g. raw floating point numbers) as possibly-tagged
+  data and so not collecting any Lisp objects that they point to. This
+  has some negative consequences for average time efficiency (though
+  possibly no worse than the negative consequences of trying to
+  implement an exact GC on a processor architecture as register-poor
+  as the X86) and also has potentially unlimited consequences for
+  worst-case memory efficiency. In practice, @CONSERVATIVE-GCs work
+  reasonably well, not getting anywhere near the worst case. But they
+  can occasionally cause odd patterns of memory usage.
+
+  The fork from CMUCL was based on a major rewrite of the system
+  bootstrap process. CMUCL has for many years tolerated a very unusual
+  \"build\" procedure which doesn't actually build the complete system
+  from scratch, but instead progressively overwrites parts of a
+  running system with new versions. This quasi-build procedure can
+  cause various bizarre bootstrapping hangups, especially when a major
+  change is made to the system. It also makes the connection between
+  the current source code and the current executable more tenuous than
+  in other software systems -- it's easy to accidentally build a CMUCL
+  system containing characteristics not reflected in the current
+  version of the source code.
+
+  Other major changes since the fork from CMUCL include:
+
+  - SBCL has removed many CMUCL extensions, (e.g. IP networking,
+    remote procedure call, Unix system interface, and X11 interface)
+    from the core system. Most of these are available as contributed
+    modules (distributed with SBCL) or third-party modules instead.
+
+  - SBCL has deleted or deprecated some nonstandard features and code
+    complexity which helped efficiency at the price of
+    maintainability. For example, the SBCL compiler no longer
+    implements memory pooling internally (and so is simpler and more
+    maintainable, but generates more garbage and runs more slowly).")

contrib/sb-manual/doc/package-locks.lisp

@@ -0,0 +1,261 @@
+(in-package :sb-manual)
+
+(defsection @package-locks (:title "Package Locks"
+                            :concepts (("package" "lock")
+                                       ("lock," "package")))
+  "None of the following sections apply to SBCL built without package
+  locking support.
+
+  The interface described here is experimental: incompatible changes
+  in future SBCL releases are possible, even expected: the concept of
+  _implementation packages_ and the associated operators may be
+  renamed; more operations (such as naming restarts or catch tags) may
+  be added to the list of operations violating package locks."
+  (@package-lock-concepts section)
+  (@package-lock-dictionary section))
+
+(defsection @package-lock-concepts (:title "Package Lock Concepts")
+  "Package locks protect against unintentional modifications of a package:
+  they provide similar protection to user packages as is mandated to
+  `COMMON-LISP` package by the ANSI specification. They are not, and
+  should not be used as, a security measure.
+
+  Newly created packages are by default unlocked (see the :LOCK option
+  to DEFPACKAGE).
+
+  The package `COMMON-LISP` and SBCL internal implementation packages
+  are locked by default, including `SB-EXT`.
+
+  It may be beneficial to lock `COMMON-LISP-USER` as well, to ensure
+  that various libraries don't pollute it without asking, but this is
+  not currently done by default."
+  (@implementation-packages section)
+  (@package-lock-violations section)
+  (@package-locks-in-compiled-code section)
+  (@operations-violating-package-locks section))
+
+(defsection @implementation-packages (:title "Implementation Packages")
+  "Each package has a list of associated implementation packages. A
+  locked package, and the symbols whose home package it is, can be
+  modified without violating package locks only when *PACKAGE* is
+  bound to one of the implementation packages of the locked package.
+
+  Unless explicitly altered by DEFPACKAGE,
+  SB-EXT:ADD-IMPLEMENTATION-PACKAGE, or
+  SB-EXT:REMOVE-IMPLEMENTATION-PACKAGE, each package is its own
+  (only) implementation package.")
+
+(defsection @package-lock-violations (:title "Package Lock Violations")
+  (@lexical-bindings-and-declarations section)
+  (@other-operations section))
+
+(defsection @lexical-bindings-and-declarations
+    (:title "Lexical Bindings and Declarations")
+  "Lexical bindings or @DECLARATIONs that violate package locks cause a
+  compile-time warning, and a runtime PROGRAM-ERROR when the form that
+  violates package locks would be executed.
+
+  A complete listing of operators affected by this is: LET, LET*,
+  FLET, LABELS, MACROLET, SYMBOL-MACROLET, DECLARE.
+
+  Package locks affecting both lexical bindings and declarations can
+  be disabled locally with the SB-EXT:DISABLE-PACKAGE-LOCKS
+  declaration, and re-enabled with the SB-EXT:ENABLE-PACKAGE-LOCKS
+  declaration.
+
+  Example:
+
+      (in-package :locked)
+
+      (defun foo () ...)
+
+      (defmacro with-foo (&body body)
+        `(locally (declare (disable-package-locks locked:foo))
+           (flet ((foo () ...))
+             (declare (enable-package-locks locked:foo)) ; re-enable for body
+             ,@body)))")
+
+(defsection @other-operations (:title "Other Operations")
+  "If a non-lexical operation violates a package lock, a continuable
+  error that is of a subtype of SB-EXT:PACKAGE-LOCK-VIOLATION
+  (subtype of PACKAGE-ERROR) is signalled when the operation is
+  attempted.
+
+  Additional restarts may be established for continuable package lock
+  violations for interactive use.
+
+  The actual type of the error depends on circumstances that caused
+  the violation: operations on packages signal errors of type
+  SB-EXT:PACKAGE-LOCKED-ERROR, and operations on symbols signal errors
+  of type SB-EXT:SYMBOL-PACKAGE-LOCKED-ERROR.")
+
+(defsection @package-locks-in-compiled-code
+    (:title "Package Locks in Compiled Code")
+  "If file-compiled code contains interned symbols, then loading that
+  code into an image without the said symbols will not cause a package
+  lock violation, even if the packages in question are locked.
+
+  With the exception of interned symbols, behaviour is unspecified if
+  package locks affecting compiled code are not the same during
+  loading of the code or execution.
+
+  Specifically, code compiled with packages unlocked may or may not
+  fail to signal package-lock-violations even if the packages are
+  locked at runtime, and code compiled with packages locked may or may
+  not signal spurious package-lock-violations at runtime even if the
+  packages are unlocked.
+
+  In practice all this means that package-locks have a negligible
+  performance penalty in compiled code as long as they are not
+  violated.")
+
+(defsection @operations-violating-package-locks
+    (:title "Operations Violating Package Locks")
+  (@operations-on-packages section)
+  (@operations-on-symbols section))
+
+(defsection @operations-on-packages (:title "Operations on Packages")
+  "The following actions cause a package lock violation if the package
+  operated on is locked, and *PACKAGE* is not an implementation
+  package of that package, and the action would cause a change in the
+  state of the package (so e.g. exporting already external symbols is
+  never a violation). Package lock violations caused by these
+  operations signal errors of type SB-EXT:PACKAGE-LOCKED-ERROR.
+
+  - Shadowing a symbol in a package.
+
+  - Importing a symbol to a package.
+
+  - Uninterning a symbol from a package.
+
+  - Exporting a symbol from a package.
+
+  - Unexporting a symbol from a package.
+
+  - Changing the packages used by a package.
+
+  - Renaming a package.
+
+  - Deleting a package.
+
+  - Adding a new package local nickname to a package.
+
+  - Removing an existing package local nickname from a package.")
+
+(defsection @operations-on-symbols (:title "Operations on Symbols")
+  "Following actions cause a package lock violation if the home package
+  of the symbol operated on is locked, and *PACKAGE* is not an
+  implementation package of that package. Package lock violations
+  caused by these action signal errors of type
+  SB-EXT:SYMBOL-PACKAGE-LOCKED-ERROR.
+
+  These actions cause only one package lock violation per lexically
+  apparent violated package.
+
+  Example:
+
+
+      ;;; Packages FOO and BAR are locked.
+      ;;;
+      ;;; Two lexically apparent violated packages: exactly two
+      ;;; package-locked-errors will be signalled.
+
+      (defclass foo:point ()
+        ((x :accessor bar:x)
+         (y :accessor bar:y)))
+
+  - Binding or altering its value lexically or dynamically, or
+    establishing it as a symbol-macro.
+
+      Exceptions:
+
+      - If the symbol is not defined as a constant, global
+        symbol-macro or a global dynamic variable, it may be lexically
+        bound or established as a local symbol macro.
+
+      - If the symbol is defined as a global dynamic variable, it may
+        be assigned or bound.
+
+  - Defining, undefining, or binding it, or its setf name as a
+    function.
+
+      Exceptions:
+
+      - If the symbol is not defined as a function, macro, or special
+        operator it and its setf name may be lexically bound as a
+        function.
+
+  - Defining, undefining, or binding it as a macro or compiler macro.
+
+      Exceptions:
+
+      - If the symbol is not defined as a function, macro, or special
+        operator it may be lexically bound as a macro.
+
+  - Defining it as a type specifier or structure.
+
+  - Defining it as a declaration with a declaration proclamation.
+
+  - Declaring or proclaiming it special.
+
+  - Declaring or proclaiming its type or ftype.
+
+      Exceptions:
+
+      - If the symbol may be lexically bound, the type of that binding
+        may be declared.
+
+      - If the symbol may be lexically bound as a function, the ftype
+        of that binding may be declared.
+
+  - Defining a setf expander for it.
+
+  - Defining it as a method combination type.
+
+  - Using it as the CLASS-NAME argument to (SETF FIND-CLASS).
+
+  - Defining it as a hash table test using SB-EXT:DEFINE-HASH-TABLE-TEST.")
+
+(defsection @package-lock-dictionary (:title "Package Lock Dictionary")
+  (sb-ext:disable-package-locks declaration)
+  (sb-ext:enable-package-locks declaration)
+  (sb-ext:package-lock-violation condition)
+  (sb-ext:package-locked-error condition)
+  (sb-ext:symbol-package-locked-error condition)
+  (sb-ext:package-locked-error-symbol function)
+  (sb-ext:package-locked-p function)
+  (sb-ext:lock-package function)
+  (sb-ext:unlock-package function)
+  (sb-ext:package-implemented-by-list function)
+  (sb-ext:package-implements-list function)
+  (sb-ext:add-implementation-package function)
+  (sb-ext:remove-implementation-package function)
+  (sb-ext:without-package-locks macro)
+  (sb-ext:with-unlocked-packages macro)
+
+  "The DEFPACKAGE options are extended to include the following:
+
+  - :LOCK `<boolean>` (defaults to NIL)
+
+      If the argument to :LOCK is T, the package is locked, else it is
+      unlocked. Existing package are also affected.
+
+  - :IMPLEMENT `<package-designator>*`
+
+      The package is added as an implementation package to the
+      packages named. If :IMPLEMENT is not provided, it defaults to
+      the package itself.
+
+  Example:
+
+      (defpackage \"FOO\" (:export \"BAR\") (:lock t) (:implement))
+      (defpackage \"FOO-INT\" (:use \"FOO\") (:implement \"FOO\" \"FOO-INT\"))
+
+      ;;; is equivalent to
+
+      (defpackage \"FOO\") (:export \"BAR\"))
+      (lock-package \"FOO\")
+      (remove-implementation-package \"FOO\" \"FOO\")
+
+      (defpackage \"FOO-INT\" (:use \"BAR\"))
+      (add-implementation-package \"FOO-INT\" \"FOO\")")

contrib/sb-manual/doc/pathnames.lisp

@@ -0,0 +1,153 @@
+(in-package :sb-manual)
+
+(defsection @pathnames (:title "Pathnames"
+                        :concepts ("pathname"))
+  (@lisp-pathnames section)
+  (@native-filenames section))
+
+(defsection @lisp-pathnames (:title "Lisp Pathnames")
+  "There are many aspects of ANSI Common Lisp's pathname support
+  which are implementation-defined and so need documentation."
+  (@home-directory-specifiers section)
+  (@the-sys-logical-pathname-host section))
+
+;; FIXME: as a matter of ANSI conformance, we are required to document
+;; implementation-defined stuff, which for pathnames (chapter 19 of CLtS)
+;; includes:
+;;
+;; * Otherwise, the parsing of thing is implementation-defined.
+;;   (PARSE-NAMESTRING)
+;;
+;; * If thing contains an explicit host name and no explicit device name,
+;;   then it is implementation-defined whether parse-namestring will supply
+;;   the standard default device for that host as the device component of
+;;   the resulting pathname.  (PARSE-NAMESTRING)
+;;
+;; * The specific nature of the search is implementation-defined.
+;;   (LOAD-LOGICAL-PATHNAME-TRANSLATIONS)
+;;
+;; * Any additional elements are implementation-defined.
+;;   (LOGICAL-PATHNAME-TRANSLATIONS)
+;;
+;; * The matching rules are implementation-defined but should be consistent
+;;   with directory.  (PATHNAME-MATCH-P)
+;;
+;; * Any such additional translations are implementation-defined.
+;;   (TRANSLATE-LOGICAL-PATHNAMES)
+;;
+;; * ...or an implementation-defined portion of a component...
+;;   (TRANSLATE-PATHNAME)
+;;
+;; * The portion of source that is copied into the resulting pathname is
+;;   implementation-defined.  (TRANSLATE-PATHNAME)
+;;
+;; * During the copying of a portion of source into the resulting
+;;   pathname, additional implementation-defined translations of case or
+;;   file naming conventions might occur.  (TRANSLATE-PATHNAME)
+;;
+;; * In general, the syntax of namestrings involves the use of
+;;   implementation-defined conventions.  (19.1.1)
+;;
+;; * The nature of the mapping between structure imposed by pathnames and
+;;   the structure, if any, that is used by the underlying file system is
+;;   implementation-defined.  (19.1.2)
+;;
+;; * The mapping of the pathname components into the concepts peculiar to
+;;   each file system is implementation-defined.  (19.1.2)
+;;
+;; * Whether separator characters are permitted as part of a string in a
+;;   pathname component is implementation-defined;  (19.2.2.1.1)
+;;
+;; * Whether a value of :unspecific is permitted for any component on any
+;;   given file system accessible to the implementation is
+;;   implementation-defined.  (19.2.2.2.3)
+;;
+;; * Other symbols and integers have implementation-defined meaning.
+;;   (19.2.2.4.6)
+
+(defsection @home-directory-specifiers (:title "Home Directory Specifiers")
+  "SBCL accepts the keyword :HOME and a list of the form
+  `(:HOME` `\"username\")` as a directory component immediately
+  following :ABSOLUTE.
+
+  :HOME is represented in namestrings by `~/` and `(:HOME`
+  `\"username\")` by `~username/` at the start of the namestring.
+  Tilde-characters elsewhere in namestrings represent themselves.
+
+  Home directory specifiers are resolved to home directory of the
+  current or specified user by SB-EXT:NATIVE-NAMESTRING, which is used
+  by the implementation to translate pathnames before passing them on
+  to operating system specific routines.
+
+  Using `(:HOME` `\"user\")` form on Windows signals an error.")
+
+(defsection @the-sys-logical-pathname-host
+    (:title "The SYS Logical Pathname Host")
+  ;; The existence and meaning of SYS: logical pathnames is
+  ;; implementation-defined (CLHS 19.3.1.1.1).
+  "The @LOGICAL-PATHNAME host named by `\"SYS\"` exists in SBCL.
+  Its LOGICAL-PATHNAME-TRANSLATIONS may be set by the site or the user
+  applicable to point to the locations of the system's sources; in
+  particular, the core system's source files match the logical
+  pathname `\"SYS:SRC;**;*.*.*\"`, and the contributed modules' source
+  files match `\"SYS:CONTRIB;**;*.*.*\"`."
+  (sb-ext:set-sbcl-source-location function))
+
+(defsection @native-filenames (:title "Native Filenames")
+  "In some circumstances, what is wanted is a Lisp pathname object which
+  corresponds to a string produced by the Operating System. In this
+  case, some of the default parsing rules are inappropriate: most
+  filesystems do not have a native understanding of wild pathnames;
+  such functionality is often provided by shells above the OS, often
+  in mutually-incompatible ways.
+
+  To allow the user to deal with this, the following functions are
+  provided: SB-EXT:PARSE-NATIVE-NAMESTRING and SB-EXT:NATIVE-PATHNAME
+  return the closest equivalent Lisp pathname to a given string
+  (appropriate for the Operating System), while
+  SB-EXT:NATIVE-NAMESTRING converts a non-wild pathname designator to
+  the equivalent native namestring, if possible. Some Lisp pathname
+  concepts (such as the :BACK directory component) have no direct
+  equivalents in most Operating Systems; the behaviour of
+  SB-EXT:NATIVE-NAMESTRING is unspecified if an inappropriate pathname
+  designator is passed to it. Additionally, note that conversion from
+  pathname to native filename and back to pathname should not be
+  expected to preserve equivalence under EQUAL."
+  (sb-ext:parse-native-namestring function)
+  (sb-ext:native-pathname function)
+  (sb-ext:native-namestring function)
+  "Because some file systems permit the names of directories to be
+  expressed in multiple ways, it is occasionally necessary to parse a
+  native file name as a directory name or to produce a native file
+  name that names a directory as a file. For these cases,
+  PARSE-NATIVE-NAMESTRING accepts the keyword argument
+  :AS-DIRECTORY to force a filename to parse as a directory, and
+  SB-EXT:NATIVE-NAMESTRING accepts the keyword argument :AS-FILE
+  to force a pathname to unparse as a file. For example,
+
+      ; On Unix, the directory \"/tmp/\" can be denoted by \"/tmp/\" or \"/tmp\".
+      ; Under the default rules for native filenames, these parse and
+      ; unparse differently.
+      (defvar *p*)
+      (setf *p* (parse-native-namestring \"/tmp/\")) => #P\"/tmp/\"
+      (pathname-name *p*) => NIL
+      (pathname-directory *p*) => (:ABSOLUTE \"tmp\")
+      (native-namestring *p*) => \"/tmp/\"
+
+      (setf *p* (parse-native-namestring \"/tmp\")) => #P\"/tmp\"
+      (pathname-name *p*) => \"tmp\"
+      (pathname-directory *p*) => (:ABSOLUTE)
+      (native-namestring *p*) => \"/tmp\"
+
+      ; A non-NIL AS-DIRECTORY argument to PARSE-NATIVE-NAMESTRING forces
+      ; both the second string to parse the way the first does.
+      (setf *p* (parse-native-namestring \"/tmp\"
+                                         nil *default-pathname-defaults*
+                                         :as-directory t)) => #P\"/tmp/\"
+      (pathname-name *p*) => NIL
+      (pathname-directory *p*) => (:ABSOLUTE \"tmp\")
+
+      ; A non-NIL AS-FILE argument to NATIVE-NAMESTRING forces the pathname
+      ; parsed from the first string to unparse as the second string.
+      (setf *p* (parse-native-namestring \"/tmp/\")) => #P\"/tmp/\"
+      (native-namestring *p* :as-file t) => \"/tmp\"")

contrib/sb-manual/doc/profiling.lisp

@@ -0,0 +1,162 @@
+(in-package :sb-manual)
+
+(defsection @profiling (:title "Profiling"
+                        :concepts ("profiling"))
+  "SBCL includes both a deterministic profiler, that can collect
+  statistics on individual functions, and a more \"modern\",
+  statistical profiler.
+
+  Inlined functions do not appear in the results reported by either."
+  (@deterministic-profiler section)
+  (@statistical-profiler section))
+
+(defsection @deterministic-profiler
+    (:title "Deterministic Profiler"
+     :concepts (("profiling," "deterministic")))
+  "The package `SB-PROFILE` provides a classic, per-function-call
+  profiler.
+
+  > __Warning__: When profiling code executed by multiple threads in
+  > parallel, the consing attributed to each function is inaccurate."
+  (sb-profile:profile macro)
+  (sb-profile:unprofile macro)
+  (sb-profile:report function)
+  (sb-profile:reset function))
+
+(defsection @statistical-profiler (:title "Statistical Profiler"
+                                   :concepts (("profiling," "statistical")))
+  "The `SB-SPROF` module, loadable by
+
+      (require :sb-sprof)
+
+  provides an alternate profiler which works by taking samples of the
+  program execution at regular intervals, instead of instrumenting
+  functions as SB-PROFILE:PROFILE does. You might find `SB-SPROF` more
+  useful than the deterministic profiler when profiling functions in the
+  `COMMON-LISP` package, SBCL internals, or code where the instrumenting
+  overhead is excessive.
+
+  Additionally `SB-SPROF` includes a limited deterministic profiler
+  which can be used for reporting the amounts of calls to some functions
+  during
+
+  __Example usage:__
+
+      (in-package :cl-user)
+
+      (require :sb-sprof)
+
+      (declaim (optimize speed))
+
+      (defun cpu-test-inner (a i)
+        (logxor a
+                (* i 5)
+                (+ a i)))
+
+      (defun cpu-test (n)
+        (let ((a 0))
+          (dotimes (i (expt 2 n) a)
+            (setf a (cpu-test-inner a i)))))
+
+      ;;;; CPU profiling
+
+      ;;; Take up to 1000 samples of running (CPU-TEST 26), and give a flat
+      ;;; table report at the end. Profiling will end one the body has been
+      ;;; evaluated once, whether or not 1000 samples have been taken.
+      (sb-sprof:with-profiling (:max-samples 1000
+                                :report :flat
+                                :loop nil)
+        (cpu-test 26))
+
+      ;;; Record call counts for functions defined on symbols in the CL-USER
+      ;;; package.
+      (sb-sprof:profile-call-counts \"CL-USER\")
+
+      ;;; Take 1000 samples of running (CPU-TEST 24), and give a flat
+      ;;; table report at the end. The body will be re-evaluated in a loop
+      ;;; until 1000 samples have been taken. A sample count will be printed
+      ;;; after each iteration.
+      (sb-sprof:with-profiling (:max-samples 1000
+                                :report :flat
+                                :loop t
+                                :show-progress t)
+        (cpu-test 24))
+
+      ;;;; Allocation profiling
+
+      (defun foo (&rest args)
+        (mapcar (lambda (x) (float x 1d0)) args))
+
+      (defun bar (n)
+        (declare (fixnum n))
+        (apply #'foo (loop repeat n collect n)))
+
+      (sb-sprof:with-profiling (:max-samples 10000
+                                :mode :alloc
+                                :report :flat)
+        (bar 1000))
+
+  __Output:__
+
+  The flat report format will show a table of all functions that the
+  profiler encountered on the call stack during sampling, ordered by
+  the number of samples taken while executing that function.
+
+                 Self        Total        Cumul
+        Nr  Count     %  Count     %  Count     %    Calls  Function
+      ------------------------------------------------------------------------
+         1     69  24.4     97  34.3     69  24.4 67108864  CPU-TEST-INNER
+         2     64  22.6     64  22.6    133  47.0        -  SB-VM::GENERIC-+
+         3     39  13.8    256  90.5    172  60.8        1  CPU-TEST
+         4     31  11.0     31  11.0    203  71.7        -  SB-KERNEL:TWO-ARG-XOR
+
+  For each function, the table will show three absolute and relative
+  sample counts. The `Self` column shows samples taken while directly
+  executing that function. The `Total` column shows samples taken
+  while executing that function or functions called from it (sampled
+  to a platform-specific depth). The `Cumul` column shows the sum of
+  all `Self` columns up to and including that line in the table.
+
+  Additionally the `Calls` column will record the amount of calls that
+  were made to the function during the profiling run. This value will
+  only be reported for functions that have been explicitly marked for
+  call counting with SB-SPROF:PROFILE-CALL-COUNTS.
+
+  The profiler also hooks into the disassembler such that instructions
+  which have been sampled are annotated with their relative frequency
+  of sampling. This information is not stored across different
+  sampling runs.
+
+      ;      6CF:       702E             JO L4              ; 6/242 samples
+      ;      6D1:       D1E3             SHL EBX, 1
+      ;      6D3:       702A             JO L4
+      ;      6D5: L2:   F6C303           TEST BL, 3         ; 2/242 samples
+      ;      6D8:       756D             JNE L8
+      ;      6DA:       8BC3             MOV EAX, EBX       ; 5/242 samples
+      ;      6DC: L3:   83F900           CMP ECX, 0         ; 4/242 samples
+
+  __Platform support__
+
+  Allocation profiling is only supported on SBCL builds that use the
+  @GENERATIONAL-GC. Tracking of call stacks at a depth of more than
+  two levels is only supported on x86 and x86-64.
+
+  __Macros__"
+  (sb-sprof:with-profiling macro)
+  (sb-sprof:with-sampling macro)
+  "__Functions__"
+  (sb-sprof:map-traces function)
+  (sb-sprof:sample-pc function)
+  (sb-sprof:report function)
+  (sb-sprof:reset function)
+  (sb-sprof:start-profiling function)
+  (sb-sprof:stop-profiling function)
+  (sb-sprof:profile-call-counts function)
+  (sb-sprof:unprofile-call-counts function)
+  "__Variables__"
+  (sb-sprof:*max-samples* variable)
+  (sb-sprof:*sample-interval* variable)
+  "__Credits__
+
+  `SB-SPROF` is an SBCL port, with enhancements, of Gerd Moellmann's
+  statistical profiler for CMUCL.")

contrib/sb-manual/doc/sbcl.lisp

@@ -0,0 +1,53 @@
+(in-package :sb-manual)
+
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (defun documentation-generation-date-string (&key long)
+    (multiple-value-bind (second minute hour day month year)
+        (decode-universal-time (get-universal-time))
+      (if long
+          (format nil "~D-~2,'0D-~2,'0D ~2,'0D:~2,'0D:~2,'0D"
+                  year month day hour minute second)
+          (format nil "~D-~2,'0D" year month))))
+  (defun package-exists-p/reader (name)
+    (if (find-package name) '(and) '(or))))
+
+(defsection @sbcl-manual (:title "SBCL Manual")
+  ;; This docstring is not used in the Texinfo version (see
+  ;; EMIT-TEXINFO-FOR-SECTION).
+  #.(format nil "This is an unofficial rendering of the SBCL Manual
+  using [MGL-PAX][pax::@pax-manual] with the same content as the
+  official version at <https://www.sbcl.org/> but with heavy linking
+  internally, to the CLHS, and to the source code on
+  [GitHub](https://github.com/sbcl/sbcl).
+
+  The output is for SBCL version `~A`, generated _~A_. See
+  <https://fixnum.com> for this document in other formats.
+
+  This manual is part of the SBCL software system. See the
+  `\\\\README` file for more information. The manual is largely
+  derived from the manual for the [CMUCL](https://cmucl.org/) system,
+  which was produced at Carnegie Mellon University and later released
+  into the public domain. The manual is in the public domain and is
+  provided with absolutely no warranty. See the `\\\\COPYING` and
+  `\\\\CREDITS` files for more information."
+            (lisp-implementation-version)
+            (documentation-generation-date-string :long t))
+  (@support-and-bugs section)
+  (@introduction section)
+  (@starting-and-stopping section)
+  (@compiler section)
+  (@debugger section)
+  (@efficiency section)
+  (@beyond-the-ansi-standard section)
+  (@external-formats section)
+  (@foreign-function-interface section)
+  (@pathnames section)
+  (@streams section)
+  (@package-locks section)
+  (@threading section)
+  (@timers section)
+  #+#.(sb-manual::package-exists-p/reader '#:sb-bsd-sockets)
+  (@networking section)
+  (@profiling section)
+  (@contributed-modules section)
+  (@deprecation section))

contrib/sb-manual/doc/start-stop.lisp

@@ -0,0 +1,340 @@
+(in-package :sb-manual)
+
+(defsection @starting-and-stopping (:title "Starting and Stopping")
+  (@starting-sbcl section)
+  (@stopping-sbcl section)
+  (@command-line-options section)
+  (@initialization-files section)
+  (@initialization-and-exit-hooks section))
+
+(defsection @starting-sbcl (:title "Starting SBCL")
+  (@running-from-shell section)
+  (@running-from-emacs section)
+  (@shebang-scripts section))
+
+(defsection @running-from-shell (:title "Running from Shell")
+  "To run SBCL, type `sbcl` at the command line.
+
+  You should end up in the toplevel _REPL_ (read-eval-print loop),
+  where you can interact with SBCL by typing expressions.
+
+      $ sbcl
+      This is SBCL 0.8.13.60, an implementation of ANSI Common Lisp.
+      More information about SBCL is available at <http://www.sbcl.org/>.
+
+      SBCL is free software, provided as is, with absolutely no warranty.
+      It is mostly in the public domain; some portions are provided under
+      BSD-style licenses.  See the CREDITS and COPYING files in the
+      distribution for more information.
+      * (+ 2 2)
+      4
+      * (exit)
+      $
+
+  Also see @COMMAND-LINE-OPTIONS and @STOPPING-SBCL.")
+
+(defsection @running-from-emacs (:title "Running from Emacs")
+  "To run SBCL as an `inferior-lisp` from Emacs, in your `.emacs` do
+  something like:
+
+      ;;; The SBCL binary and command-line arguments
+      (setq inferior-lisp-program \"/usr/local/bin/sbcl --noinform\")
+
+  For more information on using SBCL with Emacs, see
+  @EDITOR-INTEGRATION.")
+
+(defsection @shebang-scripts (:title "Shebang Scripts")
+  "Standard Unix tools that are interpreters follow a common command line
+  protocol that is necessary to work with \"shebang scripts\". SBCL
+  supports this via the `--script` command line option (see
+  @COMMAND-LINE-OPTIONS).
+
+  Example file (`hello.lisp`):
+
+      #!/usr/local/bin/sbcl --script
+      (write-line \"Hello, World!\")
+
+  Usage from the command line:
+
+      $ ./hello.lisp
+      Hello, World!
+
+  Note that SBCL skips the shebang line when it reads the file:
+
+      $ sbcl --script hello.lisp
+      Hello, World!")
+
+(defsection @stopping-sbcl (:title "Stopping SBCL")
+  (@exit section)
+  (@end-of-file section)
+  (@saving-a-core-image section)
+  (@exit-on-errors section))
+
+(defsection @exit (:title "Exit")
+  "SBCL can be stopped at any time by calling SB-EXT:EXIT,
+  optionally returning a specified numeric value to the calling
+  process. See @THREADING for information about terminating individual
+  threads."
+  (sb-ext:exit function))
+
+(defsection @end-of-file (:title "End of File")
+  "By default SBCL also exits on end of input, caused either by user
+  pressing `Control-D` on an attached terminal, or end of input when
+  using SBCL as part of a shell pipeline.")
+
+(defsection @saving-a-core-image (:title "Saving a Core Image")
+  "SBCL has the ability to save its state as a file for later
+  execution. This functionality is important for its bootstrapping
+  process, and is also provided as an extension to the user."
+  (sb-ext:save-lisp-and-die function)
+  ;; When Swank is loaded, it sets this variable.
+  (sb-ext:*save-hooks* (variable nil))
+  "In cases where the standard initialization files have already been loaded
+  into the saved core, and alternative ones should be used (or none at
+  all), SBCL allows customizing the initfile pathname computation."
+  (sb-ext:*sysinit-pathname-function* variable)
+  (sb-ext:*userinit-pathname-function* variable)
+  "To facilitate distribution of SBCL applications using external
+  resources, the filesystem location of the SBCL core file being used
+  is available from Lisp."
+  (sb-ext:*core-pathname* (variable "<site-specific>")))
+
+(defsection @exit-on-errors (:title "Exit on Errors")
+  "SBCL can also be configured to exit if an unhandled error occurs,
+  which is mainly useful for acting as part of a shell pipeline; doing
+  so under most other circumstances would mean giving up large parts
+  of the flexibility and robustness of Common Lisp. See
+  @DEBUGGER-ENTRY and the command line option `--disable-debugger` in
+  @RUNTIME-OPTIONS.")
+
+(defsection @command-line-options (:title "Command Line Options")
+  "Command line options can be considered an advanced topic; for ordinary
+  interactive use, no command line arguments should be necessary.
+
+  In order to understand the command line argument syntax for SBCL, it
+  is helpful to understand that the SBCL system is implemented as two
+  components, a low-level runtime environment written in \\C and a
+  higher-level system written in Common Lisp itself. Some command line
+  arguments are processed during the initialization of the low-level
+  runtime environment, some command line arguments are processed
+  during the initialization of the Common Lisp system, and any
+  remaining command line arguments are made available to user code via
+  SB-EXT:*POSIX-ARGV*.
+
+  The full, unambiguous syntax for invoking SBCL at the command line
+  is:
+
+      sbcl <runtime-option>* --end-runtime-options \\
+           <toplevel-option>* --end-toplevel-options \\
+           <user-option>*
+
+  For convenience, `--end-runtime-options` and
+  `--end-toplevel-options` can be omitted, which can be convenient
+  when you are running the program interactively, and you can see that
+  no ambiguities are possible with the option values you are using.
+  Omitting these elements is probably a bad idea for any batch file
+  where any of the options are under user control, since it makes it
+  impossible for SBCL to detect erroneous command line input, so that
+  erroneous command line arguments will be passed on to the user
+  program even if they was intended for the runtime system or the Lisp
+  system."
+  (@runtime-options section)
+  (@toplevel-options section))
+
+(defsection @runtime-options (:title "Runtime Options")
+  "- `--core <corefilename>`
+
+      Run the specified Lisp core file instead of the default. Note
+      that if the Lisp core file is a user-created core file, it may
+      run a nonstandard toplevel which does not recognize the standard
+      toplevel options.
+
+  - `--dynamic-space-size <megabytes>`
+
+      Size of the dynamic space reserved on startup in megabytes.
+      Default value is platform dependent.
+
+  - `--control-stack-size <megabytes>`
+
+      Size of control stack reserved for each thread in megabytes.
+      Default value is 2.
+
+  - `--tls-limit <positive integer>`
+
+      Maximum number of thread-local symbols in threaded builds.
+      Default value is 4096.
+
+  - `--noinform`
+
+      Suppress the printing of any banner or other informational
+      message at startup. This makes it easier to write Lisp programs
+      which work cleanly in Unix pipelines. See also the `--noprint`
+      and `--disable-debugger` options.
+
+  - `--disable-ldb`
+
+      Disable the low-level debugger. Only effective if SBCL is
+      compiled with @LDB. ~DISABLING-LDB
+
+  - `--lose-on-corruption`
+
+      There are some dangerous low-level errors (for instance, control
+      stack exhausted, memory fault) that (or whose handlers) can
+      corrupt the image. By default, SBCL prints a warning, then tries
+      to continue and handle the error in Lisp, but this will not
+      always work, and SBCL may malfunction or even hang. With this
+      option, upon encountering such an error, SBCL will exit instead
+      of invoking @LDB (if present and enabled ~ENABLING-LDB).
+
+  - `--script <filename>`
+
+      As a _runtime_ option, this is equivalent to `--noinform`
+      `--disable-ldb` `--lose-on-corruption`
+      `--end-runtime-options` `--script` `<filename>`. See
+      the description of `--script` as a _toplevel_ option below.
+      If there are no other command line arguments following
+      `--script`, the filename argument can be omitted.
+
+  - `--merge-core-pages`
+
+      When platform support is present, provide hints to the operating
+      system that identical pages may be shared between processes
+      until they are written to. This can be useful to reduce the
+      memory usage on systems with multiple SBCL processes started
+      from similar but differently-named core files, or from
+      compressed cores. Without platform support, do nothing. By
+      default only compressed cores trigger hinting.
+
+  - `--no-merge-core-pages`
+
+      Ensures that no sharing hint is provided to the operating
+      system.
+
+  - `--help`
+
+      Print some basic information about SBCL, then exit.
+
+  - `--version`
+
+      Print SBCL's version information, then exit.
+
+  In the future, runtime options may be added to control behaviour
+  such as lazy allocation of memory.
+
+  Runtime options, including any `--end-runtime-options` option, are
+  stripped out of the command line before the Lisp toplevel logic gets
+  a chance to see it.")
+
+(defsection @toplevel-options (:title "Toplevel Options")
+  "The following options are processed and removed by the default
+  toplevel (see SB-EXT:SAVE-LISP-AND-DIE).
+
+  - `--sysinit <filename>`
+
+      Load `FILENAME` instead of the default system initialization
+      file (see @INITIALIZATION-FILES).
+
+  - `--no-sysinit`
+
+      Don't load a system-wide initialization file. If this option is
+      given, the `--sysinit` option is ignored.
+
+  - `--userinit <filename>`
+
+      Load `FILENAME` instead of the default user initialization file
+      (see @INITIALIZATION-FILES.)
+
+  - `--no-userinit`
+
+      Don't load a user initialization file. If this option is given,
+      the `--userinit` option is ignored.
+
+  - `--eval <command>`
+
+      After executing any initialization file, but before starting the
+      read-eval-print loop on standard input, read and evaluate
+      `COMMAND`. More than one `--eval` option can be used, and all
+      will be read and executed, in the order they appear on the
+      command line.
+
+  - `--load <filename>`
+
+      This is equivalent to `--eval '(load \"<filename>\")'`. The
+      special syntax is intended to reduce quoting headaches when
+      invoking SBCL from shell scripts.
+
+  - `--noprint`
+
+      When ordinarily the toplevel \"read-eval-print loop\" would be
+      executed, execute a \"read-eval loop\" instead, i.e. don't print
+      a prompt and don't echo results. Combined with the `--noinform`
+      runtime option, this makes it easier to write Lisp \"scripts\"
+      which work cleanly in Unix pipelines.
+
+  - `--disable-debugger`
+
+      By default when SBCL encounters an error, it enters the builtin
+      debugger, allowing interactive diagnosis and possible
+      intercession. This option disables the debugger, causing errors
+      to print a backtrace and exit with status 1 instead. When given,
+      this option takes effect before loading of initialization files
+      or processing `--eval` and `--load` options. See
+      SB-EXT:DISABLE-DEBUGGER and @DEBUGGER-ENTRY.
+
+  - `--script <filename>`
+
+      Implies `--no-userinit` `--no-sysinit` `--disable-debugger`
+      `--end-toplevel-options`.
+
+      Causes the system to load the specified file instead of entering
+      the read-eval-print-loop, and exit afterwards. If the file
+      begins with a shebang line, it is ignored.
+
+      If there are no other command line arguments following, the
+      filename can be omitted: this causes the script to be loaded
+      from standard input instead. Shebang lines in standard input
+      script are currently _not_ ignored.
+
+      In either case, if there is an unhandled error (e.g. end of
+      file, or a broken pipe) on either standard input, standard
+      output, or standard error, the script silently exits with code
+      0. This allows e.g. safely piping output from SBCL to `head -n1`
+      or similar.
+
+      Additionally, the option sets *COMPILE-VERBOSE* and
+      *LOAD-VERBOSE* to NIL while loading the file to avoid
+      potentially verbose diagnostic messages printed on the standard
+      output.")
+
+(defsection @initialization-files (:title "Initialization Files")
+  "SBCL processes initialization files with READ and EVAL,
+  not LOAD; hence initialization files can be used to set startup
+  *PACKAGE* and *READTABLE*, and for proclaiming a global optimization
+  policy.
+
+  - __System Initialization File:__ Defaults to `$SBCL_HOME/sbclrc`,
+    or if that doesn't exist to `/etc/sbclrc`. Can be overridden with
+    the command line option `--sysinit` or `--no-sysinit` (see
+    @TOPLEVEL-OPTIONS).
+
+      The system initialization file is intended for system
+      administrators and software packagers to configure locations of
+      installed third party modules, etc.
+
+  - __User Initialization File:__ Defaults to `$HOME/.sbclrc`. Can be
+    overridden with the command line option `--userinit` or
+    `--no-userinit` (see @TOPLEVEL-OPTIONS).
+
+      The user initialization file is intended for personal
+      customizations, such as loading certain modules at startup,
+      defining convenience functions to use in the @REPL,
+      handling automatic recompilation of FASLs (see @FASL-FORMAT),
+      etc.
+
+  Neither initialization file is required.")
+
+(defsection @initialization-and-exit-hooks
+    (:title "Initialization and Exit Hooks")
+  "SBCL provides hooks into the system initialization and exit."
+  (sb-ext:*init-hooks* variable)
+  (sb-ext:*exit-hooks* variable))

contrib/sb-manual/doc/streams.lisp

@@ -0,0 +1,322 @@
+(in-package :sb-manual)
+
+(defsection @streams (:title "Streams")
+  "Streams which read or write Lisp character data from or to the outside
+  world -- files, sockets or other external entities -- require the
+  specification of a conversion between the external, binary data and
+  the Lisp characters. In ANSI Common Lisp, this is done by specifying
+  the :EXTERNAL-FORMAT argument when the stream is created. The major
+  information required is an _encoding_, specified by a keyword naming
+  that encoding; however, it is also possible to specify refinements
+  to that encoding as additional options to the external format
+  designator.
+
+  In addition, SBCL supports various extensions of ANSI Common Lisp
+  streams:
+
+  - _Bivalent Streams_: A type of stream that can read and write both
+    CHARACTER and `(UNSIGNED-BYTE 8)` values.
+
+  - _Gray Streams_: User-overloadable CLOS classes whose instances can
+    be used as Lisp streams (e.g. passed as the first argument to
+    FORMAT).
+
+  - _Simple Streams_: The bundled contrib module `SB-SIMPLE-STREAMS`
+    implements a subset of the Franz Allegro simple-streams proposal."
+  (@stream-external-formats section)
+  (@bivalent-streams section)
+  (@gray-streams section)
+  (@sb-simple-streams section))
+
+(defsection @stream-external-formats (:title "Stream External Formats")
+  "The function STREAM-EXTERNAL-FORMAT returns the canonical name of
+  the @EXTERNAL-FORMAT (see @EXTERNAL-FORMATS) used by the stream for
+  character-based input and/or output.
+
+  When constructing file streams, for example using OPEN or
+  WITH-OPEN-FILE, the external format to use is specified via the
+  :EXTERNAL-FORMAT argument which accepts an external format
+  designator (see @EXTERNAL-FORMAT-DESIGNATORS).")
+
+(defsection @bivalent-streams (:title "Bivalent Streams")
+  "A _bivalent stream_ can be used to read and write both
+  CHARACTER and `(UNSIGNED-BYTE 8)` values. A bivalent stream is
+  created by calling OPEN with the argument :ELEMENT-TYPE
+  :DEFAULT. On such a stream, both binary and character data can be
+  read and written with the usual input and output functions.
+
+  Streams are _not_ created bivalent by default for performance
+  reasons. Bivalent streams are incompatible with `FAST-READ-CHAR`, an
+  internal optimization in SBCL's stream machinery that bulk-converts
+  octets to characters and implements a fast path through READ-CHAR.")
+
+(defsection @gray-streams (:title "Gray Streams")
+  "The Gray Streams interface is a widely supported extension that
+  provides for definition of CLOS-extensible stream classes. Gray
+  stream classes are implemented by adding methods to generic
+  functions analogous to Common Lisp's standard I/O functions.
+  Instances of Gray stream classes may be used with any I/O operation
+  where a non-Gray stream can, provided that all required methods have
+  been implemented suitably."
+  (@gray-streams-classes section)
+  (@methods-common-to-all-streams section)
+  (@input-stream-methods section)
+  (@character-input-stream-methods section)
+  (@output-stream-methods section)
+  (@character-output-stream-methods section)
+  (@binary-stream-methods section)
+  (@gray-streams-examples section))
+
+(defsection @gray-streams-classes (:title "Gray Streams classes")
+  "The defined Gray Stream classes are these:"
+  (sb-gray:fundamental-stream class)
+  (sb-gray:fundamental-input-stream class)
+  "The function INPUT-STREAM-P will return true of any generalized
+  instance of SB-GRAY:FUNDAMENTAL-INPUT-STREAM."
+  (sb-gray:fundamental-output-stream class)
+  "The function OUTPUT-STREAM-P will return true of any generalized
+  instance of SB-GRAY:FUNDAMENTAL-OUTPUT-STREAM."
+  (sb-gray:fundamental-binary-stream class)
+  "Note that instantiable subclasses of SB-GRAY:FUNDAMENTAL-BINARY-STREAM
+  should provide (or inherit) an applicable method for the generic
+  function STREAM-ELEMENT-TYPE."
+  (sb-gray:fundamental-character-stream class)
+  (sb-gray:fundamental-binary-input-stream class)
+  (sb-gray:fundamental-binary-output-stream class)
+  (sb-gray:fundamental-character-input-stream class)
+  (sb-gray:fundamental-character-output-stream class))
+
+(defsection @methods-common-to-all-streams
+    (:title "Methods common to all streams")
+  "These generic functions can be specialized on any generalized instance
+  of fundamental-stream."
+  (stream-element-type generic-function)
+  (close generic-function)
+  (sb-gray:stream-file-position generic-function))
+
+(defsection @input-stream-methods (:title "Input stream methods")
+  "These generic functions may be specialized on any generalized instance
+  of fundamental-input-stream."
+  (sb-gray:stream-clear-input generic-function)
+  (sb-gray:stream-read-sequence generic-function))
+
+(defsection @character-input-stream-methods
+    (:title "Character input stream methods")
+  "These generic functions are used to implement subclasses of
+  SB-GRAY:FUNDAMENTAL-INPUT-STREAM:"
+  (sb-gray:stream-peek-char generic-function)
+  (sb-gray:stream-read-char-no-hang generic-function)
+  (sb-gray:stream-read-char generic-function)
+  (sb-gray:stream-read-line generic-function)
+  (sb-gray:stream-listen generic-function)
+  (sb-gray:stream-unread-char generic-function))
+
+(defsection @output-stream-methods (:title "Output stream methods")
+  "These generic functions are used to implement subclasses of
+  SB-GRAY:FUNDAMENTAL-OUTPUT-STREAM:"
+  (sb-gray:stream-clear-output generic-function)
+  (sb-gray:stream-finish-output generic-function)
+  (sb-gray:stream-force-output generic-function)
+  (sb-gray:stream-write-sequence generic-function))
+
+(defsection @character-output-stream-methods
+    (:title "Character output stream methods")
+  "These generic functions are used to implement subclasses of
+  SB-GRAY:FUNDAMENTAL-CHARACTER-OUTPUT-STREAM:"
+  (sb-gray:stream-advance-to-column generic-function)
+  (sb-gray:stream-fresh-line generic-function)
+  (sb-gray:stream-line-column generic-function)
+  (sb-gray:stream-line-length generic-function)
+  (sb-gray:stream-start-line-p generic-function)
+  (sb-gray:stream-terpri generic-function)
+  (sb-gray:stream-write-char generic-function)
+  (sb-gray:stream-write-string generic-function))
+
+(defsection @binary-stream-methods (:title "Binary stream methods")
+  "The following generic functions are available for subclasses of
+  SB-GRAY:FUNDAMENTAL-BINARY-STREAM:"
+  (sb-gray:stream-read-byte generic-function)
+  (sb-gray:stream-write-byte generic-function))
+
+(defsection @gray-streams-examples (:title "Gray Streams Examples")
+  "Below are two classes of stream that can be conveniently defined as
+  wrappers for Common Lisp streams. These are meant to serve as
+  examples of minimal implementations of the protocols that must be
+  followed when defining Gray streams. Realistic uses of the Gray
+  Streams API would implement the various methods that can do I/O in
+  batches, such as SB-GRAY:STREAM-READ-LINE,
+  SB-GRAY:STREAM-WRITE-STRING, SB-GRAY:STREAM-READ-SEQUENCE, and
+  SB-GRAY:STREAM-WRITE-SEQUENCE."
+  (@character-counting-input-stream section)
+  (@output-prefixing-character-stream section))
+
+(defsection @character-counting-input-stream
+    (:title "Character Counting Input Stream")
+  "  It is occasionally handy for programs that process input files to
+  count the number of characters and lines seen so far, and the number
+  of characters seen on the current line, so that useful messages may
+  be reported in case of parsing errors, etc. Here is a character
+  input stream class that keeps track of these counts. Note that all
+  character input streams must implement SB-GRAY:STREAM-READ-CHAR and
+  SB-GRAY:STREAM-UNREAD-CHAR.
+
+      (defclass wrapped-stream (fundamental-stream)
+        ((stream :initarg :stream :reader stream-of)))
+
+      (defmethod stream-element-type ((stream wrapped-stream))
+        (stream-element-type (stream-of stream)))
+
+      (defmethod close ((stream wrapped-stream) &key abort)
+        (close (stream-of stream) :abort abort))
+
+      (defclass wrapped-character-input-stream
+          (wrapped-stream fundamental-character-input-stream)
+        ())
+
+      (defmethod stream-read-char ((stream wrapped-character-input-stream))
+        (read-char (stream-of stream) nil :eof))
+
+      (defmethod stream-unread-char ((stream wrapped-character-input-stream)
+                                     char)
+        (unread-char char (stream-of stream)))
+
+      (defclass counting-character-input-stream
+          (wrapped-character-input-stream)
+        ((char-count :initform 1 :accessor char-count-of)
+         (line-count :initform 1 :accessor line-count-of)
+         (col-count :initform 1 :accessor col-count-of)
+         (prev-col-count :initform 1 :accessor prev-col-count-of)))
+
+      (defmethod stream-read-char ((stream counting-character-input-stream))
+        (with-accessors ((inner-stream stream-of) (chars char-count-of)
+                         (lines line-count-of) (cols col-count-of)
+                         (prev prev-col-count-of)) stream
+            (let ((char (call-next-method)))
+              (cond ((eql char :eof)
+                     :eof)
+                    ((char= char #\Newline)
+                     (incf lines)
+                     (incf chars)
+                     (setf prev cols)
+                     (setf cols 1)
+                     char)
+                    (t
+                     (incf chars)
+                     (incf cols)
+                     char)))))
+
+      (defmethod stream-unread-char ((stream counting-character-input-stream)
+                                     char)
+        (with-accessors ((inner-stream stream-of) (chars char-count-of)
+                         (lines line-count-of) (cols col-count-of)
+                         (prev prev-col-count-of)) stream
+            (cond ((char= char #\Newline)
+                   (decf lines)
+                   (decf chars)
+                   (setf cols prev))
+                  (t
+                   (decf chars)
+                   (decf cols)
+                   char))
+            (call-next-method)))
+
+  The default methods for SB-GRAY:STREAM-READ-CHAR-NO-HANG,
+  SB-GRAY:STREAM-PEEK-CHAR, SB-GRAY:STREAM-LISTEN,
+  SB-GRAY:STREAM-CLEAR-INPUT, SB-GRAY:STREAM-READ-LINE, and
+  SB-GRAY:STREAM-READ-SEQUENCE should be sufficient (though the last
+  two will probably be slower than methods that forwarded directly).
+
+  Here's a sample use of this class:
+
+      (with-input-from-string (input \"1 2
+       3 :foo  \")
+        (let ((counted-stream (make-instance 'counting-character-input-stream
+                               :stream input)))
+          (loop for thing = (read counted-stream) while thing
+             unless (numberp thing) do
+               (error \"Non-number ~S (line ~D, column ~D)\" thing
+                      (line-count-of counted-stream)
+                      (- (col-count-of counted-stream)
+                         (length (format nil \"~S\" thing))))
+             end
+             do (print thing))))
+
+  Output:
+
+      1
+      2
+      3
+      Non-number :FOO (line 2, column 5)
+        [Condition of type SIMPLE-ERROR]")
+
+(defsection @output-prefixing-character-stream
+    (:title "Output Prefixing Character Stream")
+  "One use for a wrapped output stream might be to prefix each line of
+  text with a timestamp, e.g. for a logging stream. Here's a simple
+  stream that does this, though without any fancy line-wrapping. Note
+  that all character output stream classes must implement
+  SB-GRAY:STREAM-WRITE-CHAR and SB-GRAY:STREAM-LINE-COLUMN.
+
+      (defclass wrapped-stream (fundamental-stream)
+        ((stream :initarg :stream :reader stream-of)))
+
+      (defmethod stream-element-type ((stream wrapped-stream))
+        (stream-element-type (stream-of stream)))
+
+      (defmethod close ((stream wrapped-stream) &key abort)
+        (close (stream-of stream) :abort abort))
+
+      (defclass wrapped-character-output-stream
+          (wrapped-stream fundamental-character-output-stream)
+        ((col-index :initform 0 :accessor col-index-of)))
+
+      (defmethod stream-line-column ((stream wrapped-character-output-stream))
+        (col-index-of stream))
+
+      (defmethod stream-write-char ((stream wrapped-character-output-stream)
+                                    char)
+        (with-accessors ((inner-stream stream-of) (cols col-index-of)) stream
+          (write-char char inner-stream)
+          (if (char= char #\Newline)
+              (setf cols 0)
+              (incf cols))))
+
+      (defclass prefixed-character-output-stream
+          (wrapped-character-output-stream)
+        ((prefix :initarg :prefix :reader prefix-of)))
+
+      (defgeneric write-prefix (prefix stream)
+        (:method ((prefix string) stream) (write-string prefix stream))
+        (:method ((prefix function) stream) (funcall prefix stream)))
+
+      (defmethod stream-write-char ((stream prefixed-character-output-stream)
+                                    char)
+        (with-accessors ((inner-stream stream-of) (cols col-index-of)
+                         (prefix prefix-of)) stream
+          (when (zerop cols)
+            (write-prefix prefix inner-stream))
+          (call-next-method)))
+
+  As with the example input stream, this implements only the minimal
+  protocol. A production implementation should also provide methods
+  for at least SB-GRAY:STREAM-WRITE-STRING,
+  SB-GRAY:STREAM-WRITE-SEQUENCE.
+
+  And here's a sample use of this class:
+
+      (flet ((format-timestamp (stream)
+               (apply #'format stream \"[~2@*~2,' D:~1@*~2,'0D:~0@*~2,'0D] \"
+                      (multiple-value-list (get-decoded-time)))))
+        (let ((output (make-instance 'prefixed-character-output-stream
+                                     :stream *standard-output*
+                                     :prefix #'format-timestamp)))
+          (loop for string in '(\"abc\" \"def\" \")ghi\") do
+               (write-line string output)
+               (sleep 1))))
+
+  Output:
+
+      [ 0:30:05] abc
+      [ 0:30:06] def
+      [ 0:30:07] ghi
+      NIL")

contrib/sb-manual/doc/support-and-bugs.lisp

@@ -0,0 +1,125 @@
+(in-package :sb-manual)
+
+(defsection @support-and-bugs (:title "Getting Support and Reporting Bugs")
+  (@volunteer-support section)
+  (@commercial-support section)
+  (@reporting-bugs section))
+
+(defsection @volunteer-support (:title "Volunteer Support")
+  "Your primary source of SBCL support should probably be the mailing
+  list `sbcl-help`: in addition to other users SBCL developers monitor
+  this list and are available for advice. As an anti-spam measure
+  subscription is required for posting:
+
+  <https://lists.sourceforge.net/lists/listinfo/sbcl-help>
+
+  Remember that the people answering your question are volunteers, so
+  you stand a much better chance of getting a good answer if you ask a
+  good question.
+
+  Before sending mail, check the list archives at either
+
+  <http://sourceforge.net/mailarchive/forum.php?forum_name=sbcl-help>
+
+  or
+
+  <http://news.gmane.org/gmane.lisp.steel-bank.general>
+
+  to see if your question has been answered already. Checking the bug
+  database is also worth it (see @REPORTING-BUGS), to see if the issue
+  is already known.
+
+  For general advice on asking good questions, see
+
+  <http://www.catb.org/~esr/faqs/smart-questions.html>.")
+
+(defsection @commercial-support (:title "Commercial Support")
+  "There is no formal organization developing SBCL, but if you need a
+  paid support arrangement or custom SBCL development, we maintain the
+  list of companies and consultants below. Use it to identify service
+  providers with appropriate skills and interests, and contact them
+  directly.
+
+  The SBCL project cannot verify the accuracy of the information or
+  the competence of the people listed, and they have provided their
+  own blurbs below: you must make your own judgement of suitability
+  from the available information - refer to the links they provide,
+  the CREDITS file, mailing list archives, CVS commit messages, and so
+  on. Please feel free to ask for advice on the sbcl-help list.
+
+  (At present, no companies or consultants wish to advertise paid
+  support or custom SBCL development in this manual).")
+
+(defsection @reporting-bugs (:title "Reporting Bugs")
+  "SBCL uses Launchpad to track bugs. The bug database is available at
+
+  <https://bugs.launchpad.net/sbcl>
+
+  Reporting bugs there requires registering at Launchpad. However,
+  bugs can also be reported on the mailing list `sbcl-bugs`, which is
+  moderated but does _not_ require subscribing. Simply send email to
+  [`sbcl-bugs@lists.sourceforge.net`](mailto:sbcl-bugs@lists.sourceforge.net)
+  and the bug will be checked and added to Launchpad by SBCL
+  maintainers.
+
+  See the `\\\\HACKING` file on how to send patches."
+  (@how-to-report-bugs-effectively section)
+  (@how-to-report-signal-related-bugs section))
+
+(defsection @how-to-report-bugs-effectively
+    (:title "How to Report Bugs Effectively")
+  "Please include enough information in a bug report that someone reading
+  it can reproduce the problem, i.e. don't write
+
+      Subject: apparent bug in PRINT-OBJECT (or *PRINT-LENGTH*?)
+      PRINT-OBJECT doesn't seem to work with *PRINT-LENGTH*. Is this a bug?
+
+  but instead
+
+      Subject: apparent bug in PRINT-OBJECT (or *PRINT-LENGTH*?)
+      In sbcl-1.2.3 running under OpenBSD 4.5 on my Alpha box, when
+      I compile and load the file
+         (DEFSTRUCT (FOO (:PRINT-OBJECT (LAMBDA (X Y)
+                                          (LET ((*PRINT-LENGTH* 4))
+                                            (PRINT X Y)))))
+           X Y)
+      then at the command line type
+         (MAKE-FOO)
+      the program loops endlessly instead of printing the object.
+
+  A more in-depth discussion on reporting bugs effectively can be
+  found at
+
+  <http://www.chiark.greenend.org.uk/~sgtatham/bugs.html>.")
+
+(defsection @how-to-report-signal-related-bugs
+    (:title "How to Report Signal-related Bugs")
+  "If you run into a signal related bug, you are getting fatal errors
+  such as `signal N is [un]blocked` or just hangs, and you want to
+  send a useful bug report then:
+
+  - Compile SBCL with @LDB enabled (feature `:SB-LDB`, see
+    `base-target-features.lisp-expr`).
+
+  - Isolate a smallish test case, run it.
+
+  - If it just hangs kill it with `SIGABRT`: `kill -ABRT <pidof sbcl>`.
+
+  - Print the backtrace from ldb by typing `ba`.
+
+  - Attach gdb: `gdb -p <pidof sbcl>` and get backtraces for all
+    threads: `thread apply all ba`.
+
+  - If multiple threads are in play then still in gdb, try to get Lisp
+    backtrace for all threads: `thread apply all call
+    backtrace_from_fp($ebp, 100, 0)`. Substitute `$ebp` with `$rbp` on
+    x86-64. The backtraces will appear in the stdout of the SBCL
+    process.
+
+  - Send a report with the backtraces and the output (both stdout and
+    stderr) produced by SBCL.
+
+  - Don't forget to include OS and SBCL version.
+
+  - If available, include information on outcome of the same test with
+    other versions of SBCL, OS, ...")

contrib/sb-manual/doc/threading.lisp

@@ -0,0 +1,334 @@
+(in-package :sb-manual)
+
+(defsection @threading (:title "Threading")
+  "SBCL supports a fairly low-level threading interface that maps onto
+  the host operating system's concept of threads or lightweight
+  processes. This means that threads may take advantage of hardware
+  multiprocessing on machines that have more than one CPU, but it does
+  not allow Lisp control of the scheduler. This is found in the
+  `SB-THREAD` package.
+
+  Threads are part of the default build on x86[-64]/ARM64 Linux and
+  Windows.
+
+  They are also supported on: x86[-64] Darwin (Mac OS X), x86[-64]
+  FreeBSD, x86 SunOS (Solaris), PPC Linux, ARM64 Linux, RISC-V Linux.
+  On these platforms threads must be explicitly enabled at build-time,
+  see `INSTALL` for directions."
+  (@threading-basics section)
+  (@special-variables section)
+  (@atomic-operations section)
+  (@mutex-support section)
+  (@semaphores section)
+  (@waitqueue/condition-variables section)
+  (@barriers section)
+  (@sessions/debugging section)
+  (@foreign-threads section)
+  (@implementation-on-linux-x86oids section))
+
+(defsection @threading-basics (:title "Threading Basics")
+  "```
+  (make-thread (lambda () (write-line \"Hello, world\")))
+  ```"
+  (@thread-objects section)
+  (@running-threads section)
+  (@asynchronous-operations section)
+  (@miscellaneous-operations section)
+  (@error-conditions section))
+
+(defsection @thread-objects (:title "Thread Objects")
+  (sb-thread:thread structure)
+  (sb-thread:*current-thread* variable)
+  (sb-thread:list-all-threads function)
+  (sb-thread:thread-alive-p function)
+  (sb-thread:thread-name function)
+  (sb-thread:main-thread-p function)
+  (sb-thread:main-thread function))
+
+(defsection @running-threads (:title "Running Threads")
+  (sb-thread:make-thread function)
+  (sb-thread:return-from-thread macro)
+  (sb-thread:abort-thread function)
+  (sb-thread:join-thread function)
+  (sb-thread:thread-yield function))
+
+(defsection @asynchronous-operations (:title "Asynchronous Operations")
+  (sb-thread:interrupt-thread function)
+  (sb-thread:terminate-thread function))
+
+(defsection @miscellaneous-operations (:title "Miscellaneous Operations")
+  (sb-thread:symbol-value-in-thread function))
+
+(defsection @error-conditions (:title "Error Conditions")
+  (sb-thread:thread-error condition)
+  (sb-thread:thread-error-thread function)
+  (sb-thread:symbol-value-in-thread-error condition)
+  (sb-thread:interrupt-thread-error condition)
+  (sb-thread:join-thread-error condition))
+
+(defsection @special-variables (:title "Special Variables")
+  "The interaction of special variables with multiple threads is mostly
+  as one would expect, with behaviour very similar to other
+  implementations.
+
+  - Global special values are visible across all threads.
+
+  - Bindings (e.g. using LET) are local to the thread.
+
+  - Threads do not inherit dynamic bindings from the parent thread.
+
+  The last point means that
+
+      (defparameter *x* 0)
+      (let ((*x* 1))
+        (sb-thread:make-thread (lambda () (print *x*))))
+
+  prints `0` and not `1`.
+
+  Note, however, that there is a hard limit on the number of distinct
+  symbols that can be bound dynamically in threaded builds (see
+  `--tls-limit` in @RUNTIME-OPTIONS). Exceeding this limit triggers
+  the low-level error `Thread local storage exhausted.`")
+
+(defsection @atomic-operations (:title "Atomic Operations")
+  "Following atomic operations are particularly useful for implementing
+  lockless algorithms."
+  (sb-ext:atomic-decf macro)
+  (sb-ext:atomic-incf macro)
+  (sb-ext:atomic-pop macro)
+  (sb-ext:atomic-push macro)
+  (sb-ext:atomic-update macro)
+  (sb-ext:compare-and-swap macro)
+  "Our SB-EXT:COMPARE-AND-SWAP is user-extensible by defining functions
+  named `(CAS <PLACE>)`, allowing users to add CAS support to new
+  places."
+  (sb-ext:cas macro)
+  (sb-ext:get-cas-expansion function))
+
+(defsection @mutex-support (:title "Mutex Support")
+  "Mutexes are used for controlling access to a shared resource. One
+  thread is allowed to hold the mutex, others which attempt to take it
+  will be made to wait until it's free. Threads are woken in the order
+  that they go to sleep.
+
+      (defpackage :demo (:use \"CL\" \"SB-THREAD\" \"SB-EXT\"))
+
+      (in-package :demo)
+
+      (defvar *a-mutex* (make-mutex :name \"my lock\"))
+
+      (defun thread-fn ()
+        (format t \"Thread ~A running ~%\" *current-thread*)
+        (with-mutex (*a-mutex*)
+          (format t \"Thread ~A got the lock~%\" *current-thread*)
+          (sleep (random 5)))
+        (format t \"Thread ~A dropped lock, dying now~%\" *current-thread*))
+
+      (make-thread #'thread-fn)
+      (make-thread #'thread-fn)"
+  (sb-thread:mutex structure)
+  (sb-thread:with-mutex macro)
+  (sb-thread:with-recursive-lock macro)
+  (sb-thread:make-mutex function)
+  (sb-thread:mutex-name function)
+  (sb-thread:mutex-owner function)
+  (sb-thread:mutex-value function)
+  (sb-thread:grab-mutex function)
+  (sb-thread:release-mutex function))
+
+(defsection @semaphores (:title "Semaphores")
+  "Semaphores are among other things useful for keeping track of a
+  countable resource, e.g. messages in a queue, and sleep when the
+  resource is exhausted."
+  (sb-thread:semaphore structure)
+  (sb-thread:make-semaphore function)
+  (sb-thread:signal-semaphore function)
+  (sb-thread:wait-on-semaphore function)
+  (sb-thread:try-semaphore function)
+  (sb-thread:semaphore-count function)
+  (sb-thread:semaphore-name function)
+  (sb-thread:semaphore-notification structure)
+  (sb-thread:make-semaphore-notification function)
+  (sb-thread:semaphore-notification-status function)
+  (sb-thread:clear-semaphore-notification function))
+
+(defsection @waitqueue/condition-variables
+    (:title "Waitqueue/condition variables")
+  "These are based on the POSIX condition variable design, hence the
+  annoyingly CL-conflicting name. For use when you want to check a
+  condition and sleep until it's true. For example: you have a shared
+  queue, a writer process checking _queue is empty_ and one or more
+  readers that need to know when _queue is not empty_. It sounds
+  simple but is astonishingly easy to deadlock if another process runs
+  when you weren't expecting it to.
+
+  There are three components:
+
+  - the condition itself (not represented in code)
+
+  - the condition variable (a.k.a. waitqueue) which proxies for it
+
+  - a lock to hold while testing the condition
+
+  Important stuff to be aware of:
+
+  - when calling condition-wait, you must hold the mutex.
+    condition-wait will drop the mutex while it waits, and obtain it
+    again before returning for whatever reason;
+
+  - likewise, you must be holding the mutex around calls to
+    SB-THREAD:CONDITION-NOTIFY;
+
+  - a process may return from SB-THREAD:CONDITION-WAIT in several
+    circumstances: it is not guaranteed that the underlying condition
+    has become true. You must check that the resource is ready for
+    whatever you want to do to it.
+
+          (defvar *buffer-queue* (make-waitqueue))
+          (defvar *buffer-lock* (make-mutex :name \"buffer lock\"))
+
+          (defvar *buffer* (list nil))
+
+          (defun reader ()
+            (with-mutex (*buffer-lock*)
+              (loop
+               (condition-wait *buffer-queue* *buffer-lock*)
+               (loop
+                (unless *buffer* (return))
+                (let ((head (car *buffer*)))
+                  (setf *buffer* (cdr *buffer*))
+                  (format t \"reader ~A woke, read ~A~%\"
+                          *current-thread* head))))))
+
+          (defun writer ()
+            (loop
+             (sleep (random 5))
+             (with-mutex (*buffer-lock*)
+               (let ((el (intern
+                          (string (code-char
+                                   (+ (char-code #\A) (random 26)))))))
+                 (setf *buffer* (cons el *buffer*)))
+               (condition-notify *buffer-queue*))))
+
+          (make-thread #'writer)
+          (make-thread #'reader)
+          (make-thread #'reader)"
+  (sb-thread:waitqueue structure)
+  (sb-thread:make-waitqueue function)
+  (sb-thread:waitqueue-name function)
+  (sb-thread:condition-wait function)
+  (sb-thread:condition-notify function)
+  (sb-thread:condition-broadcast function))
+
+(defsection @barriers (:title "Barriers")
+  "These are based on the Linux kernel barrier design, which is in turn
+  based on the Alpha CPU memory model. They are presently implemented for
+  x86, x86-64, PPC, ARM64, and RISC-V systems, and behave as compiler
+  barriers on all other CPUs.
+
+  In addition to explicit use of the SB-THREAD:BARRIER macro, the
+  following functions and macros also serve as :MEMORY barriers:
+
+  - SB-EXT:ATOMIC-DECF, SB-EXT:ATOMIC-INCF, SB-EXT:ATOMIC-PUSH,
+    and SB-EXT:ATOMIC-POP
+
+  - SB-EXT:COMPARE-AND-SWAP
+
+  - SB-THREAD:GRAB-MUTEX, SB-THREAD:RELEASE-MUTEX,
+    SB-THREAD:WITH-MUTEX and SB-THREAD:WITH-RECURSIVE-LOCK
+
+  - SB-THREAD:SIGNAL-SEMAPHORE, SB-THREAD:TRY-SEMAPHORE and
+    SB-THREAD:WAIT-ON-SEMAPHORE
+
+  - SB-THREAD:CONDITION-WAIT, SB-THREAD:CONDITION-NOTIFY and
+    SB-THREAD:CONDITION-BROADCAST."
+  (sb-thread:barrier macro))
+
+(defsection @sessions/debugging (:title "Sessions/Debugging")
+  "If the user has multiple views onto the same Lisp image (for example,
+  using multiple terminals, or a windowing system, or network access)
+  they are typically set up as multiple _sessions_ such that each view
+  has its own collection of foreground, background, and stopped
+  threads. A thread which wishes to create a new session can use
+  SB-THREAD:WITH-NEW-SESSION to remove itself from the current
+  session (which it shares with its parent and siblings) and create a
+  fresh one."
+  (sb-thread:with-new-session macro)
+  #-win32
+  (sb-thread:make-listener-thread function)
+  "Within a single session, threads arbitrate between themselves for
+  the user's attention. A thread may be in one of three notional
+  states: foreground, background, or stopped. When a background
+  process attempts to print a repl prompt or to enter the debugger, it
+  will stop and print a message saying that it has stopped. The user
+  at his leisure may switch to that thread to find out what it needs.
+  If a background thread enters the debugger, selecting any restart
+  will put it back into the background before it resumes. Arbitration
+  for the input stream is managed by calls to
+  SB-THREAD:GET-FOREGROUND (which may block) and
+  SB-THREAD:RELEASE-FOREGROUND."
+  (sb-thread:get-foreground function)
+  (sb-thread:release-foreground function))
+
+(defsection @foreign-threads (:title "Foreign threads")
+  "Direct calls to `pthread_create(3)` (instead of SB-THREAD:MAKE-THREAD)
+  create threads that SBCL is not aware of, these are called foreign
+  threads. Currently, it is not possible to run Lisp code in such
+  threads. This means that the Lisp side signal handlers cannot work.
+  The best solution is to start foreign threads with signals blocked,
+  but since third party libraries may create threads, it is not always
+  feasible to do so. As a workaround, upon receiving a signal in a
+  foreign thread, SBCL changes the thread's sigmask to block all
+  signals that it wants to handle and resends the signal to the
+  current process which should land in a thread that does not block
+  it, that is, a Lisp thread.
+
+  The resignalling trick cannot work for synchronously triggered signals
+  (`SIGSEGV` and co), take care not to trigger any. Resignalling for
+  synchronously triggered signals in foreign threads is subject to
+  `--lose-on-corruption`, see @RUNTIME-OPTIONS.")
+
+(defsection @implementation-on-linux-x86oids
+    (:title "Implementation on Linux x86oids")
+  "Threading is implemented using pthreads and some Linux specific bits
+  like futexes.
+
+  On x86, the per-thread local bindings for special variables is
+  achieved using the `%fs` segment register to point to a per-thread
+  storage area. This may cause interesting results if you link to
+  foreign code that expects threading or creates new threads, and the
+  thread library in question uses %fs in an incompatible way. On
+  x86-64 the r12 register has a similar role.
+
+  Queues require the `futex(2)` system call to be available: this is
+  the reason for the NPTL requirement. We test at runtime that this
+  system call exists.
+
+  Garbage collection is done with the existing Conservative
+  Generational GC. Allocation is done in small (typically 8k) regions:
+  each thread has its own region so this involves no stopping.
+  However, when a region fills, a lock must be obtained while another
+  is allocated, and when a collection is required, all processes are
+  stopped. This is achieved by sending them signals, which may make
+  for interesting behaviour if they are interrupted in system calls.
+  The streams interface is believed to handle the required system call
+  restarting correctly, but this may be a consideration when making
+  other blocking calls e.g. from foreign library code.
+
+  Large amounts of the SBCL library have not been inspected for
+  thread-safety.  Some of the obviously unsafe areas have large locks
+  around them, so compilation and fasl loading, for example, cannot be
+  parallelized.  Work is ongoing in this area.
+
+  A new thread by default is created in the same POSIX process group and
+  session as the thread it was created by.  This has an impact on
+  keyboard interrupt handling: pressing your terminal's intr key
+  (typically `Control-C`) will interrupt all processes in the
+  foreground process group, including Lisp threads that SBCL considers
+  to be notionally _background_. This is undesirable, so background
+  threads are set to ignore the `SIGINT` signal.
+
+  `SB-THREAD:MAKE-LISTENER-THREAD` in addition to creating a new Lisp
+  session makes a new POSIX session, so that pressing `Control-C` in
+  one window will not interrupt another listener - this has been found
+  to be embarrassing.")

contrib/sb-manual/doc/timers.lisp

@@ -0,0 +1,41 @@
+(in-package :sb-manual)
+
+(defsection @timers (:title "Timers")
+  "SBCL supports a system-wide event scheduler implemented on top of
+  `setitimer(2)` that also works with threads but does not require a
+  separate scheduler thread.
+
+  The following example schedules a timer that writes `Hello, world`
+  after two seconds.
+
+      (schedule-timer (make-timer (lambda ()
+                                    (write-line \"Hello, world\")
+                                    (force-output)))
+                      2)
+
+  It should be noted that writing timer functions requires special
+  care, as the dynamic environment in which they run is unpredictable:
+  dynamic variable bindings, locks held, etc, all depend on whatever
+  code was running when the timer fired. The following example should
+  serve as a cautionary tale:
+
+      (defvar *foo* nil)
+
+      (defun show-foo ()
+        (format t \"~&foo=~S~%\" *foo*)
+        (force-output t))
+
+      (defun demo ()
+        (schedule-timer (make-timer #'show-foo) 0.5)
+        (schedule-timer (make-timer #'show-foo) 1.5)
+        (let ((*foo* t))
+          (sleep 1.0))
+        (let ((*foo* :surprise!))
+          (sleep 2.0)))"
+  (sb-ext:timer structure)
+  (sb-ext:make-timer function)
+  (sb-ext:timer-name function)
+  (sb-ext:timer-scheduled-p function)
+  (sb-ext:schedule-timer function)
+  (sb-ext:unschedule-timer function)
+  (sb-ext:list-all-timers function))

contrib/sb-manual/docstring.lisp

@@ -0,0 +1,70 @@
+(in-package :sb-manual)
+
+;;;; Determining the package for parsing docstrings
+;;;;
+;;;; The package for parsing is the package that was in effect when
+;;;; the docstring of the definition was read. This is basically the
+;;;; nearest IN-PACKAGE above the definition in the sources.
+;;;;
+;;;; With these semantics, when editing a docstring in Slime, if M-.
+;;;; works on an uppercase symbol name, then you can expect it to be
+;;;; codified by MARKDOWN-TO-TEXINFO. For symbols without a
+;;;; definition, you can use TAB-completion to check, but it's better
+;;;; to actually load PAX and check (see
+;;;; PAX::@BROWSING-LIVE-DOCUMENTATION).
+
+;;; To avoid conflicting with PAX's heuristics, DOCSTRING-PACKAGE
+;;; always returns a non-NIL package. We use a reimplementation of
+;;; DREF-EXT:DEFINITION-PROPERTY for DREF-EXT:DOCSTRING*, which we
+;;; feed to DOCSTRING-PACKAGE-OVERRIDES-TO-PAX in USE-PAX.
+
+;;; These map the SYMBOL-PACKAGE of a definition's XREF-NAME to the
+;;; docstring package.
+(defparameter *package-to-docstring-package*
+  '(("COMMON-LISP" "SB-IMPL")
+    ("SB-ACLREPL" "SB-ACLREPL")
+    ("SB-ALIEN" "SB-ALIEN")
+    ("SB-BSD-SOCKETS" "SB-BSD-SOCKETS")
+    ("SB-CONCURRENCY" "SB-CONCURRENCY")
+    ("SB-COVER" "SB-COVER")
+    ("SB-DEBUG" "SB-DEBUG")
+    ("SB-EXT" "SB-IMPL")
+    ("SB-GRAY" "SB-GRAY")
+    ("SB-GROVEL" "SB-GROVEL")
+    ("SB-INTROSPECT" "SB-INTROSPECT")
+    ("SB-MANUAL" "SB-MANUAL")
+    ("SB-MD5" "SB-MD5")
+    ("SB-POSIX" "SB-POSIX")
+    ("SB-SEQUENCE" "SB-IMPL")
+    ("SB-PROFILE" "SB-PROFILE")
+    ("SB-ROTATE-BYTE" "SB-ROTATE-BYTE")
+    ("SB-UNICODE" "SB-UNICODE")
+    ("SB-SPROF" "SB-SPROF")
+    ("SB-SYS" "SB-IMPL")
+    ("SB-THREAD" "SB-THREAD")))
+
+;;; The package-wide docstring packages are almost correct, but there
+;;; are a couple of definitions in random files.
+(defparameter *definition-to-docstring-package*
+  '(((with-compilation-unit macro) "SB-C")
+    ((sb-ext:restrict-compiler-policy function) "SB-C")
+    ((trace macro) "SB-DEBUG")))
+
+;;; For when this file is recompiled in interactive development after
+;;; a later, explicit call to USE-PAX
+(eval-when (:load-toplevel :execute)
+  (when *using-pax*
+    (convert-docstring-package-overrides-to-pax)))
+
+(defun docstring-package (xref)
+  (let* ((name (xref-name xref))
+         (key (list name (xref-locative xref))))
+    (or (find-package
+         (or (second (find key *definition-to-docstring-package*
+                           :key #'first :test #'equal))
+             (when (symbolp name)
+               (second (find (package-name (symbol-package name))
+                             *package-to-docstring-package*
+                             :key #'first :test #'equal)))))
+        (assert nil () "Cannot determine package of the docstring of ~S."
+                xref))))

contrib/sb-manual/late.lisp

@@ -0,0 +1,5 @@
+(in-package :sb-manual)
+
+(eval-when (:load-toplevel :execute)
+  (when (find-package '#:mgl-pax)
+    (use-pax)))

contrib/sb-manual/make-pax-docs.lisp

@@ -0,0 +1,99 @@
+;;;; Generate the SBCL manual in various formats in doc/manual/ with PAX
+
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (require :mgl-pax/full)
+  (require :sb-manual))
+
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (sb-manual::use-pax))
+
+(in-package :sb-manual)
+
+(defvar *git-forge-uri*)
+(defvar *git-root*)
+(defvar *output-dir*)
+
+(defvar *directory* (truename (make-pathname :name nil :type nil
+                                             :defaults *load-truename*)))
+
+(defun sbcl-pages* (format)
+  (let ((source-uri-fn (when (and (not (eq format :plain))
+                                  *git-forge-uri*)
+                         (pax:make-git-source-uri-fn nil *git-forge-uri*
+                                                     :git-root *git-root*)))
+        (output-file (ecase format
+                       ((:plain) "sbcl-manual.txt")
+                       ((:markdown) "sbcl-manual.md")
+                       ((:pdf) "sbcl-manual.pdf")
+                       ((:html) "html/sbcl-manual.html"))))
+    `((:objects (, @sbcl-manual)
+       :output (,(merge-pathnames output-file *output-dir*)
+                :if-does-not-exist :create
+                :if-exists :supersede
+                ,@(when (eq format :pdf)
+                    '(:element-type (unsigned-byte 8)))
+                :ensure-directories-exist t)
+       ,@(when source-uri-fn
+           `(:source-uri-fn ,source-uri-fn))))))
+
+;;; Adjust the width of section numbers in the PDF table of contents, so
+;;; that e.g "17.10.13" doesn't protude from its box.
+(defparameter *pandoc-pdf-adjusted-table-of-contents
+  "\\makeatletter
+% \\@dottedtocline{level}{indent_before_number}{width_of_number_box}
+\\renewcommand*\\l@subsection{\\@dottedtocline{2}{1.5em}{2.8em}}
+\\renewcommand*\\l@subsubsection{\\@dottedtocline{3}{4.3em}{4.0em}}
+\\makeatother")
+
+(defun muffle-uninteresting-warnings (condition)
+  (let ((string (princ-to-string condition)))
+    ;; We know that there is no source location for declarations.
+    (when (and (search "No source location" string)
+               (search "DECLARATION" string))
+      (muffle-warning condition))))
+
+(defun make-pax-docs (&optional git-forge-uri)
+  (let ((*git-forge-uri* (or (and (plusp (length git-forge-uri))
+                                  git-forge-uri)
+                             "https://github.com/sbcl/sbcl"))
+        (*git-root* (truename (merge-pathnames "../../" *directory*)))
+        (*output-dir* (merge-pathnames "output/" *directory*))
+        (pax:*document-downcase-uppercase-code* t)
+        (pax:*document-url-versions* '(1))
+        (pax:*document-pandoc-pdf-header-includes*
+          (format nil "~A~%~A~%"
+                  pax:*document-pandoc-pdf-header-includes*
+                  *pandoc-pdf-adjusted-table-of-contents))
+        (pax:*document-pandoc-pdf-options*
+          '(("-V" "papersize=a4")
+            ("-V" "margin-left=0.8in")
+            ("-V" "margin-right=0.8in")
+            ("-V" "margin-top=1.0in")
+            ("-V" "margin-bottom=1.0in")
+            ("-V" "fontfamily=XCharter")
+            ("-V" "fontsize=10pt")
+            ("-V" "colorlinks=true")
+            ("-V" "linkcolor=blue")
+            ("-V" "urlcolor=Maroon")
+            ("-V" "toccolor=blue")
+            #+nil
+            "--verbose")))
+    (handler-bind ((warning #'muffle-uninteresting-warnings))
+      (format t "Git root: ~A~%Git forge URI: ~A~%Output dir: ~A~%"
+              *git-root* *git-forge-uri* *output-dir*)
+      (format t "Generating manual in plain text format~%")
+      (pax:document @sbcl-manual :pages (sbcl-pages* :plain) :format :plain)
+      (format t "Generating manual in Markdown format~%")
+      (pax:document @sbcl-manual :pages (sbcl-pages* :markdown)
+                    :format :markdown)
+      (format t "Generating manual in PDF format~%")
+      (pax:document @sbcl-manual :pages (sbcl-pages* :pdf) :format :pdf)
+      (format t "Generating manual in HTML format~%")
+      (pax:update-asdf-system-html-docs
+       @sbcl-manual "sb-manual"
+       :pages (sbcl-pages* :html)
+       :target-dir (merge-pathnames "html/" *output-dir*)
+       :style :charter))))
+
+#+nil
+(make-pax-docs)

contrib/sb-manual/make-pax-docs.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# This software is part of the SBCL system. See the README file for
+# more information.
+#
+# This software is in the public domain and is provided with
+# absolutely no warranty. See the COPYING and CREDITS files for
+# more information.
+
+BASE=`dirname "$0"`
+
+git_forge_uri="$1"
+
+if [ -z "$2" ] ; then
+    SBCL_TOP="$BASE/../.."
+    sbclsystem=$SBCL_TOP/src/runtime/sbcl
+    sbclcore=$SBCL_TOP/output/sbcl.core
+    if [ -f $sbclsystem ] && [ -f $sbclcore ]
+    then
+        SBCLRUNTIME="$sbclsystem --core $sbclcore"
+        SBCL_HOME=$SBCL_TOP/obj/sbcl-home/; export SBCL_HOME
+    else
+        SBCLRUNTIME="`command -v sbcl`"
+    fi
+    . $SBCL_TOP/output/build-config
+else
+    SBCLRUNTIME="$2"
+fi
+
+# We *don't* add --no-sysinit and --no-userinit because we rely on the
+# user to have set things up so that PAX can be loaded.
+${SBCLRUNTIME} \
+    --noinform --noprint --disable-debugger \
+    --load ${BASE}/make-pax-docs.lisp \
+    --eval "(sb-manual::make-pax-docs \"${git_forge_uri}\")" \
+    --quit

contrib/sb-manual/manual.lisp

@@ -0,0 +1,82 @@
+(in-package :sb-manual)
+
+(defsection @sb-manual (:title "sb-manual")
+  "The `SB-MANUAL` module has the SBCL user manual in forms
+  mimicking `PAX:DEFSECTION`:
+
+      (defsection @example (:title \"Example\")
+        \"This is an example, but see the real @SB-MANUAL.\"
+        (print function)
+        (@subexample section))
+
+  The names of the variables holding the documentation are exported
+  from the `SB-MANUAL` package. Since sections are basically
+  variables, in Slime, `\\\\M-.` on `\"@SB-MANUAL\"`, `\"print\"`, or
+  on `\"@subexample\"` will take you to the respective definition.
+  This makes it easy to navigate the documentation. Normal Lisp
+  definition docstrings and section docstrings reference sections
+  following the usual convention of uppercasing the name. Docstrings
+  are in a subset of Markdown and use very little markup in general,
+  so they are easy to read directly in the source.
+
+  The official manual in Info, HTML and PDF formats is generated via
+  Texinfo generated from these definitions."
+  (@using-pax section)
+  (@browsing-live-with-pax section)
+  (@fancy-documentation-with-pax section))
+
+(defsection @using-pax (:title "Using \\PAX")
+  "However, `SB-MANUAL::DEFSECTION` is but a dummy implementation of
+  `PAX:DEFSECTION` to avoid a hard dependency on \\PAX.
+
+  See the `MGL-PAX` ASDF:SYSTEM or <https://github.com/melisgl/mgl-pax/>.
+
+  When \\PAX is loaded, the dummy `DEFSECTION` definitions are made
+  real, so that \\PAX can work with them."
+  (use-pax function))
+
+(defsection @browsing-live-with-pax (:title "Browsing Live with \\PAX")
+  "With \\PAX, you can browse the manual live. The documentation of this
+  feature is available at
+  [online](https://melisgl.github.io/mgl-pax-world/pax-manual.html#MGL-PAX:@BROWSING-LIVE-DOCUMENTATION%20MGL-PAX:SECTION).
+
+  If you are browsing this manual live right now, here is the
+  equivalent live link: `PAX::@BROWSING-LIVE-DOCUMENTATION`.
+
+  Notable features:
+
+  - Autolinks within the manual: if SB-EXT:EXIT is mentioned, then
+    it's linked to its documentation. You basically get links to where
+    `\\\\M-.` would go in the sources.
+
+  - Autolinks to the \\CLHS.
+
+  - View the documentation of any Lisp definition or section without
+    generating the entire manual.
+
+  - Locatives (e.g. the `\"[function]\"` in `\"- [function]
+    SB-EXT:EXIT\"`) are also links in live browsing: they tell Slime
+    to visit the definition.
+
+      For this to work, you need to allow Slime to evaluate Elisp sent
+      from SBCL:
+
+          (setq slime-enable-evaluate-in-emacs t)
+
+      and maybe your window manager focus stealing configuration needs
+      tweaking as well.
+
+  Live browsing can greatly reduce the latency of Edit-Compile-View
+  Loop, when working on documentation.")
+
+(defsection @fancy-documentation-with-pax
+    (:title "Fancy Documentation with \\PAX")
+  "\\PAX can generate dead documentation, too. In the SBCL sources,
+  `contrib/sb-manual/make-pax-docs.sh` generates the manual in plain
+  text, Markdown, PDF, and HTML formats. These differ from those
+  generated via Texinfo in that they are autolinked (like when
+  @BROWSING-LIVE-WITH-PAX).
+
+  Also, you can generate documentation yourself with e.g.
+
+      (pax:document sb-manual:@sbcl-manual :format :markdown)")

contrib/sb-manual/markdown.lisp

@@ -0,0 +1,897 @@
+;;;; A Markdown-to-Texinfo converter for the SBCL manual.
+
+;;;; This software is part of the SBCL software system. SBCL is in the
+;;;; public domain and is provided with absolutely no warranty. See
+;;;; the COPYING file for more information.
+;;;;
+;;;; Written by Rudi Schlatte <rudi@constantly.at>, mangled by
+;;;; Nikodemus Siivola. Brought closer to Markdown, extended with
+;;;; section linking and concept indexing by Gabor Melis.
+
+(in-package :sb-manual)
+
+(defvar *concept-keys-to-prepend*)
+
+;;; MARKDOWN-TO-TEXINFO converts a strict subset of Markdown to
+;;; Texinfo. It also codifies (marks up as code) and downcases
+;;; uppercase symbols (those that actually exist in the image), and
+;;; autolinks references to sections, attempting to approximate PAX
+;;; semantics.
+;;;
+;;; Note that for writing docstrings, you need to know two more pieces:
+;;;
+;;; - See REINDENT-DOCSTRING for how the docstring relates to the
+;;;   Markdown string passed to MARKDOWN-TO-TEXINFO.
+;;;
+;;; - See DOCSTRING-PACKAGE to understand what *PACKAGE* is when
+;;;   MARKDOWN-TO-TEXINFO is called. This is package in effect when
+;;;   the docstring was READ. If it's wrong, you will see missed
+;;;   opportunities for codification and linking.
+;;;
+;;;
+;;; Markdown Formatting
+;;; -------------------
+;;;
+;;; The supported Markdown constructs are:
+;;;
+;;; - Emphasis: _italic_ -> @emph{italic}
+;;;
+;;; - Strong emphasis: __bold__ -> @strong{bold}
+;;;
+;;; - Inline code: `monospace` -> @code{monospace}
+;;;
+;;; - Itemized lists (like this one). List items can span multiple
+;;;   lines.
+;;;
+;;;     - Nested lists are indented 4 spaces. A blank line is required
+;;;       before the first one.
+;;;
+;;; - Indented code blocks are indented with 4 extra spaces after a
+;;;   blank line:
+;;;
+;;;     Like this:
+;;;
+;;;         void main();
+;;;
+;;; - Fenced code blocks are indented at the normal level after a
+;;;   blank line:
+;;;
+;;;     ```
+;;;     void main();
+;;;     ```
+;;;
+;;;     Use fenced code blocks only when you have consecutive code
+;;;     blocks, which would be collapsed into a single code block when
+;;;     indented.
+;;;
+;;; - Blockquotes:
+;;;
+;;;     > _Note_: They can span multiple lines and anything can be
+;;;     > nested in them. Rendered indented, typically with a vertical
+;;;     > bar on the left.
+;;;
+;;; - Note that ``abc'' is *not* supported and *will* screw up the
+;;;   rendering of the Markdown. This is because it's impossible to
+;;;   reconcile it with backticks: consider the possible semantics of
+;;;
+;;;         ``x'' and ``y''
+;;;
+;;;
+;;; Codification and Downcasing
+;;; ---------------------------
+;;
+;;; Summary: Some text is automatically codified (e.g. FOO -> `FOO`)
+;;; and most code is downcased.
+;;;
+;;; We approximate the semantics of PAX::@CODIFICATION with the
+;;; settings PAX:*DOCUMENT-UPPERCASE-IS-CODE* and
+;;; PAX:*DOCUMENT-DOWNCASE-UPPERCASE-CODE* both true.
+;;;
+;;; - Fully-qualified all-uppercase string representatation of symbols
+;;;   are codified (SB-EXT:CAS, :XYZ).
+;;;
+;;; - All-uppercase SYMBOL-NAMEs accessible in *PACKAGE*.
+;;;
+;;; - When at least 3 uppercase characters are followed by a lowercase
+;;;   character (e.g. SETFable), then the uppercase prefix is codified
+;;;   with the previous rules.
+;;;
+;;; When there is no corresponding symbol, the Markdown backtick
+;;; syntax (`PRINT`) can be used to codify.
+;;;
+;;; When there are no lowercase nor #\" characters in inline code (as
+;;; opposed to code blocks), be it auto-codified or explicitly
+;;; backticked, it's downcased.
+;;;
+;;; When there is a corresponding symbol, but codification or
+;;; downcasing should not happen, use backslash escapes.
+;;;
+;;; Escaping (following PAX::@OVERVIEW-OF-ESCAPING):
+;;;
+;;;   PRINT     -> @code{print}    (Should be autolinked, unimplemented)
+;;;   \PRINT    -> @code{print}    (Prevent autolinking)
+;;;   \\PRINT   -> PRINT           (Prevent autolinking and codification)
+;;;   `PRINT`   -> @code{print}    (Should be autolinked, unimplemented)
+;;;   `\PRINT`  -> @code{print}    (Prevent autolinking)
+;;;   `\\PRINT` -> @code{PRINT}    (Prevent autolinking and downcasing)
+;;;
+;;; Note that in docstrings, the backslashes need to be doubled.
+;;;
+;;;
+;;; Linking
+;;; -------
+;;;
+;;; - <http...> -> @url{http...}
+;;;
+;;; - [label](uri) -> @uref{uri, label}
+;;;
+;;; - [label][id] -> label
+;;;
+;;;     This just strips Markdown reference links. These are used by
+;;;     PAX to disambiguate, e.g. "[FUNCTION][type]" links to the
+;;;     FUNCTION class only while FUNCTION links to both the class and
+;;;     macro.
+;;;
+;;; - SECTION references (see DEFSECTION):
+;;;
+;;;     @SECTION-NAME -> @ref{<section name>}
+;;;
+;;; - CONCEPT references (see DEFINE-CONCEPT):
+;;;
+;;;     - pure concept: @CONCEPT-NAME -> "" (no output)
+;;;     - titled concept: @CONCEPT-NAME -> <title>
+;;;
+;;; TODO:
+;;;
+;;; - Maybe implement glossary-terms (for books, "safe type", etc).
+;;;
+;;; Also, see SB-PCL::NORMALIZE-SBCL-DOCSTRING, an expedient docstring
+;;; to plain text converter that supports the subset of this
+;;; functionality necessary for the docstrings in SBCL core.
+(defun markdown-to-texinfo (string &optional lambda-list)
+  (let ((*texinfo-local-variables* (flatten lambda-list))
+        (lines (string-lines string))
+        (line-number 0)
+        (current-paragraph nil))
+    (declare (special *texinfo-local-variables*))
+    (flet ((flush-paragraph ()
+             (when current-paragraph
+               (let* ((*concept-keys-to-prepend* ())
+                      (string (process-inline-markdown
+                               (format nil "~{~A~^~%~}"
+                                       (nreverse current-paragraph)))))
+                 (write-concept-keys *concept-keys-to-prepend* t)
+                 (write-string string))
+               (terpri)
+               (setf current-paragraph nil))))
+      (loop while (< line-number (length lines))
+            for line = (svref lines line-number)
+            do (let ((*concept-keys-to-prepend* ()))
+                 (multiple-value-bind (count collected)
+                     (parse-markdown-block lines line-number 0)
+                   (cond
+                     (count
+                      (flush-paragraph)
+                      (write-concept-keys *concept-keys-to-prepend* t)
+                      (dolist (c collected)
+                        (write-line c))
+                      (incf line-number count))
+                     ((blankp line)
+                      (flush-paragraph)
+                      (write-line line)
+                      (incf line-number))
+                     (t
+                      (push line current-paragraph)
+                      (incf line-number))))))
+      (flush-paragraph))))
+
+
+;;;; Utilities
+
+(defun flatten (list)
+  (cond ((null list)
+         nil)
+        ((consp (car list))
+         (nconc (flatten (car list)) (flatten (cdr list))))
+        ((null (cdr list))
+         (cons (car list) nil))
+        (t
+         (cons (car list) (flatten (cdr list))))))
+
+(defun flatten-to-string (list)
+  (format nil "~{~A~^-~}" (flatten list)))
+
+(defun internedp (symbol-name package)
+  (nth-value 1 (find-symbol symbol-name package)))
+
+(defun external-symbol-p (symbol &optional (package (symbol-package symbol)))
+  (and package
+       (multiple-value-bind (symbol* status)
+           (find-symbol (symbol-name symbol) package)
+         (and (eq status :external)
+              (eq symbol symbol*)))))
+
+
+;;;; Texinfo escaping
+
+(defparameter *texinfo-special-chars* "@{}")
+
+(defun escape-texinfo (string)
+  (with-output-to-string (s)
+    (loop for char across string
+          do (when (find char *texinfo-special-chars*)
+               (write-char #\@ s))
+             (write-char char s))))
+
+(defun unescape-texinfo (string)
+  (with-output-to-string (s)
+    (let ((prev-escape-p nil))
+      (loop for char across string
+            do (cond (prev-escape-p
+                      (write-char char s)
+                      (setq prev-escape-p nil))
+                     ((char= char #\@)
+                      (setq prev-escape-p t))
+                     (t
+                      (write-char char s)))))))
+
+(progn
+  (assert (equal (escape-texinfo "@code{x}") "@@code@{x@}"))
+  (assert (equal (unescape-texinfo "@@code@{x@}") "@code{x}")))
+
+
+;;;; Codification (following PAX::@CODIFICATION)
+
+(defvar *lower-case-chars* "abcdefghijklmnopqrstuvwxyz")
+
+(defun codifiable-bounds (word)
+  (when (codifiable-word-p word)
+    ;; PAX::@NAMES-IN-RAW-NAMES is involved. We only try two simple
+    ;; cases to get a PAX::@NAME.
+    (flet ((try-name (start end)
+             (let ((name (subseq word start end)))
+               (multiple-value-bind (symbol foundp)
+                   (read-symbol-without-interning name)
+                 (when (and foundp (interesting-name-p word symbol))
+                   (return-from codifiable-bounds (values start end)))))))
+      ;; 1. Trim the lower-case characters
+      (let* ((name (string-left-trim *lower-case-chars* word))
+             (name-start (- (length word) (length name)))
+             (name (string-right-trim *lower-case-chars* name))
+             (name-end (+ name-start (length name))))
+        (try-name name-start name-end))
+      ;; 2. Find the upper-case core
+      (multiple-value-bind (name-start name-end) (uppercase-core-bounds word)
+        (when name-start
+          (try-name name-start name-end))))))
+
+(defun codifiable-word-p (string)
+  (uppercase-core-bounds string))
+
+(defun read-symbol-without-interning (string)
+  (if (and (plusp (length string))
+           (char= (aref string 0) #\:))
+      (find-symbol (subseq string 1) :keyword)
+      (let ((pos (position #\: string)))
+        (if pos
+            (let* ((package-name (subseq string 0 pos))
+                   (symbol-name (subseq string (1+ pos)))
+                   (double-colon-p
+                     (and (plusp (length symbol-name))
+                          (char= (aref symbol-name 0) #\:))))
+              (when double-colon-p
+                (setq symbol-name (subseq symbol-name 1)))
+              (if package-name
+                  (when (find-package package-name)
+                    (multiple-value-bind (symbol status)
+                        (find-symbol symbol-name package-name)
+                      (when (or double-colon-p
+                                (eq status :external))
+                        (values symbol status))))
+                  (find-symbol symbol-name *package*)))
+            (find-symbol string *package*)))))
+
+;;; Approximating PAX::@INTERESTING. This is only called when we
+;;; already found the interned SYMBOL.
+(defun interesting-name-p (word symbol)
+  (or (<= 3 (length word))
+      (external-symbol-p symbol)
+      (has-local-reference-p symbol)))
+
+(defun uppercase-core-bounds (string)
+  (let* ((first-uppercase-pos (position-if #'upper-case-p string))
+         (last-uppercase-pos (position-if #'upper-case-p string
+                                          :from-end t)))
+    (when (and first-uppercase-pos
+               (if (= last-uppercase-pos first-uppercase-pos)
+                   (notany #'lower-case-p string)
+                   (not (find-if #'lower-case-p string
+                                 :start (1+ first-uppercase-pos)
+                                 :end last-uppercase-pos))))
+      (values first-uppercase-pos (1+ last-uppercase-pos)))))
+
+(defvar *texinfo-local-variables* ())
+
+(defun has-local-reference-p (name)
+  (find name *texinfo-local-variables*))
+
+#+nil
+(progn
+  (assert (equal (multiple-value-list (codifiable-bounds "PRINT"))
+                 '(0 5)))
+  (assert (equal (multiple-value-list (codifiable-bounds "T"))
+                 '(0 1)))
+  (if (internedp "A" *package*)
+      (assert (equal (multiple-value-list (codifiable-bounds "A"))
+                     '(0 1)))
+      (assert (null (codifiable-bounds "A"))))
+  (assert (equal (multiple-value-list (codifiable-bounds "*FEATURES*"))
+                 '(0 10))))
+
+;;; We parse words (e.g. nonREADable) and find symbols in them.
+(defparameter *word-characters*
+  (format nil "abcdefghijklmnopqrstuvwxyz~
+               ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789~
+               *~~@:-+&=<>#'"))
+
+(defparameter *word-delimiters* " ',.!?;()[]{}")
+
+;;; Return a list of index pairs of symbol-like parts of LINE.
+(defun locate-symbols (line)
+  (let (result)
+    (flet ((grab (word-start word-end)
+             (let ((word (subseq line word-start word-end)))
+               (multiple-value-bind (name-start name-end)
+                   (codifiable-bounds word)
+                 (when name-start
+                   (push (list (+ word-start name-start)
+                               (+ word-start name-end))
+                         result)))))
+           (got-symbol-p (start)
+             (let ((end (when (< start (length line))
+                          (position-if (lambda (c)
+                                         (or (whitespacep c)
+                                             (find c *word-delimiters*)))
+                                       line :start start))))
+               (when end
+                 (every (lambda (char) (find char *word-characters*))
+                        (subseq line start end))))))
+      (do ((begin nil)
+           (maybe-begin t)
+           (i 0 (1+ i)))
+          ((>= i (length line))
+           ;; symbol at end of line
+           (when begin
+             (grab begin i))
+           (nreverse result))
+        (cond
+          ((and begin
+                (or (whitespacep (char line i))
+                    (find (char line i) *word-delimiters*)
+                    ;; For e.g. "T:"
+                    (and (char= (char line i) #\:)
+                         (or (= (1+ i) (length line))
+                             (whitespacep (char line (1+ i)))))))
+           ;; symbol end
+           (grab begin i)
+           (setf begin nil
+                 maybe-begin t))
+          ((and begin (not (find (char line i) *word-characters*)))
+           ;; Not a symbol: abort
+           (setf begin nil))
+          ((and maybe-begin (not begin)
+                (find (char line i) *word-characters*))
+           ;; potential symbol begin at this position
+           (setf begin i
+                 maybe-begin nil))
+          ((or (whitespacep (char line i))
+               (find (char line i) *word-delimiters*))
+           ;; potential symbol begin after this position
+           (setf maybe-begin t))
+          ((and (eql #\( (char line i)) (got-symbol-p (1+ i)))
+           ;; a type designator, or a function call as part of the text?
+           (multiple-value-bind (exp end)
+               (let ((*package* (find-package :cl-user)))
+                 (ignore-errors (read-from-string line nil nil :start i)))
+             (when exp
+               (grab i end)
+               (setf begin nil
+                     maybe-begin nil
+                     i (1- end)))))
+          (t
+           ;; Not reading a symbol, not at potential start of symbol
+           (setf maybe-begin nil)))))))
+
+(unwind-protect
+     (progn
+       (defsection @test-section (:title "test section"))
+       (defsection @test5 (:title "Test5"))
+       (assert (equal (locate-symbols "PRINT") '((0 5))))
+       (assert (equal (locate-symbols "CL:PRINT") '((0 8))))
+       (assert (equal (locate-symbols "*FEATURES*") '((0 10))))
+       (assert (equal (locate-symbols "SETFable") '((0 4))))
+       (assert (equal (locate-symbols "SETF-able") '((0 4))))
+       (assert (equal (locate-symbols "nonREADable") '((3 7))))
+       (assert (equal (locate-symbols "NOSUCHSYMBOL-able") '()))
+       (assert (equal (locate-symbols "ASDF-like") '()))
+       (assert (equal (locate-symbols "@TEST-SECTION") '((0 13))))
+       (assert (equal (locate-symbols "SB-MANUAL:@TEST-SECTION") '((0 23))))
+       (assert (equal (locate-symbols "@NOSUCHSECTION") '()))
+       (assert (equal (locate-symbols "@TEST5") '((0 6))))
+       (assert (equal (locate-symbols ":IR1-CONVERT") '((0 12)))))
+  (makunbound '@test-section)
+  (makunbound '@test5))
+
+
+;;;; Processing Markdown inline elements
+
+;;; Format symbols either as Texinfo @code{}, link sections, and
+;;; handle concepts.
+(defun codify-and-link (line)
+  (with-output-to-string (result)
+    (let ((last 0))
+      (dolist (symbol/index (locate-symbols line))
+        ;; Flush unwritten text since the end of the previous symbol.
+        (write-string (escape-texinfo (subseq line last
+                                              (first symbol/index)))
+                      result)
+        (let* ((symbol-name (apply #'subseq line symbol/index))
+               (symbol (read-from-string symbol-name)))
+          (cond ((doc-name-p symbol :section)
+                 (format result "@ref{~A}"
+                         (texinfo-node-id (symbol-value symbol))))
+                ((doc-name-p symbol :concept)
+                 (let* ((concept (symbol-value symbol))
+                        (title (doctitle concept)))
+                   (when title
+                     (format result "~A" (escape-texinfo title)))
+                   (setq *concept-keys-to-prepend*
+                         (append *concept-keys-to-prepend*
+                                 (multiplexing-concept-keys concept)))))
+                (t
+                 ;; We could use for @var{} if
+                 ;; (HAS-LOCAL-REFERENCE-P SYMBOL).
+                 (format result "@code{~A}"
+                         (escape-texinfo (maybe-downcase symbol-name))))))
+        (setf last (second symbol/index)))
+      (write-string (escape-texinfo (subseq line last)) result))))
+
+(defun write-concept-keys (keys stream)
+  (dolist (key (remove-duplicates keys :test #'equal))
+    (typecase key
+      (list
+       ;; We don't use @subentry because with it Texinfo always
+       ;; presents it as as hierarchical list even if it has only one
+       ;; branch.
+       (format stream "~&@cindex~{ ~A~}~%" key))
+      (symbol
+       (assert (and (boundp key)
+                    (typep (symbol-value key) (dummy 'concept)))
+               () "Variable ~S does not hold a concept." key)
+       (write-concept-keys (concept-keys (symbol-value key)) stream))
+      (t
+       (format stream "~&@cindex ~A~%" key)))))
+
+(defvar *downcase-uppercase-code* t)
+
+(defun maybe-downcase (string)
+  (if *downcase-uppercase-code*
+      (string-downcase string)
+      string))
+
+(defun texinfo-node-id (section)
+  (let ((name (symbol-name (section-name section))))
+    (assert (char= (char name 0) #\@))
+    (let ((name (subseq name 1)))
+      (assert (null (find-if (lambda (char)
+                               (find char *texinfo-special-chars*))
+                             name))
+              () "Section name ~S contains special texinfo characters." name)
+      (substitute #\Space #\- (string-downcase name)))))
+
+(when (and (not *using-pax*)
+           *downcase-uppercase-code*)
+  (defsection @test-section (:title "Test Section"))
+  (unwind-protect
+       (progn
+         (assert (equal (codify-and-link "@TEST-SECTION") "@ref{test section}"))
+         (assert (equal (codify-and-link "@NOSUCHSECTION") "@@NOSUCHSECTION"))
+         (assert (equal (codify-and-link ":START") "@code{:start}"))
+         (assert (equal (codify-and-link "[:START") "[@code{:start}"))
+         (assert (equal (codify-and-link "{:START") "@{@code{:start}")))
+    (makunbound '@test-section)))
+
+;;; Translate backticks, emphasis and codification escapes, while
+;;; delegating normal text to CODIFY-AND-LINK.
+(defun process-inline-markdown (string)
+  (let ((len (length string))
+        (i 0)
+        (codifiable-buffer (make-string-output-stream))
+        (out (make-string-output-stream)))
+    (labels ((out (string)
+               (write-string string out))
+             (out-escaped (string)
+               (out (escape-texinfo (string string))))
+             (buffer-codifiable-char (char)
+               (write-char char codifiable-buffer))
+             (flush-codifiable-buffer ()
+               (let ((codifiable (get-output-stream-string codifiable-buffer)))
+                 (when (plusp (length codifiable))
+                   (out (codify-and-link codifiable))))))
+      (loop while (< i len)
+            for char = (char string i)
+            do (cond
+                 ;; Escapes: \FOO
+                 ((char= char #\\)
+                  (flush-codifiable-buffer)
+                  (incf i)
+                  (when (< i len)
+                    (out-escaped (char string i))
+                    (incf i)
+                    ;; Protect the rest of the contiguous word from
+                    ;; CODIFY-AND-LINK.
+                    (loop
+                      while (and (< i len)
+                                 (not (or (whitespacep (char string i))
+                                          (find (char string i)
+                                                ",.!?;()'[]{}\""))))
+                      do (out-escaped (char string i))
+                         (incf i))
+                    (decf i)))
+                 ;; Backticks: `CODE` with PAX downcasing and escape rules
+                 ((char= char #\`)
+                  (flush-codifiable-buffer)
+                  (incf i)
+                  (let ((code-buffer (make-string-output-stream)))
+                    (loop while (and (< i len) (char/= (char string i) #\`))
+                          do (write-char (char string i) code-buffer)
+                             (incf i))
+                    (let* ((code-str (get-output-stream-string code-buffer))
+                           (slash-count (loop for c across code-str
+                                              while (char= c #\\)
+                                              count t))
+                           ;; Consume up to 2 leading backslashes as PAX escapes
+                           (actual-code (subseq code-str (min slash-count 2))))
+                      (out "@code{")
+                      (if (< slash-count 2)
+                          ;; 0 or 1 backslash: Downcase if there are
+                          ;; no lowercase letters (1 backslash turns
+                          ;; off autolinking, which is naturally
+                          ;; handled by bypassing CODIFY-AND-LINK).
+                          (if (and (not (find-if #'lower-case-p actual-code))
+                                   (not (find #\" actual-code)))
+                              (out-escaped (maybe-downcase actual-code))
+                              (out-escaped actual-code))
+                          ;; 2 backslashes turn off autolinking AND downcasing.
+                          (out-escaped actual-code))
+                      (out "}"))))
+                 ;; Strong emphasis: __foo__ -> @strong{foo}
+                 ((and (char= char #\_)
+                       (< (1+ i) len)
+                       (char= (char string (1+ i)) #\_))
+                  (let ((close-pos (search "__" string :start2 (+ i 2))))
+                    (if close-pos
+                        (progn
+                          (flush-codifiable-buffer)
+                          (format out "@strong{~A}"
+                                  (process-inline-markdown
+                                   (subseq string (+ i 2) close-pos)))
+                          (setf i (+ close-pos 1)))
+                        (buffer-codifiable-char char))))
+                 ;; Emphasis: _foo_ -> @emph{foo}
+                 ((char= char #\_)
+                  (let ((close-pos nil)
+                        (j (1+ i)))
+                    (loop while (< j len)
+                          do (if (char= (char string j) #\_)
+                                 (if (and (< (1+ j) len)
+                                          (char= (char string (1+ j)) #\_))
+                                     ;; Skip double underscores so
+                                     ;; they don't falsely close a
+                                     ;; single underscore.
+                                     (incf j 2)
+                                     (progn
+                                       (setf close-pos j)
+                                       (return)))
+                                 (incf j)))
+                    (if close-pos
+                        (progn
+                          (flush-codifiable-buffer)
+                          (format out "@emph{~A}"
+                                  (process-inline-markdown
+                                   (subseq string (1+ i) close-pos)))
+                          (setf i close-pos))
+                        (buffer-codifiable-char char))))
+                 ;; Markdown autolinks: <http...> -> @url{http...}
+                 ((and (char= char #\<)
+                       (<= (+ i 5) len)
+                       (string-equal string "http" :start1 (1+ i)
+                                     :end1 (+ i 5)))
+                  (let ((close-pos (position #\> string :start (1+ i))))
+                    (if close-pos
+                        (progn
+                          (flush-codifiable-buffer)
+                          (out "@url{")
+                          (out-escaped (subseq string (1+ i) close-pos))
+                          (out "}")
+                          (setf i close-pos))
+                        (buffer-codifiable-char char))))
+                 ;; Markdown explicit links: [label](url) -> @uref{url, label}
+                 ;;
+                 ;; Markdown reflinks: [label][id] -> label
+                 ((char= char #\[)
+                  (let* ((close-bracket (position #\] string :start (1+ i)))
+                         (next-char (when (and close-bracket
+                                               (< (1+ close-bracket) len))
+                                      (char string (1+ close-bracket))))
+                         (open-paren (when (eql next-char #\()
+                                       (1+ close-bracket)))
+                         (close-paren (when open-paren
+                                        (position #\) string
+                                                  :start (1+ open-paren))))
+                         (open-bracket2 (when (eql next-char #\[)
+                                          (1+ close-bracket)))
+                         (close-bracket2
+                           (when open-bracket2
+                             (position #\] string :start (1+ open-bracket2)))))
+                    (cond
+                      ;; Explicit link: [label](url)
+                      (close-paren
+                       (flush-codifiable-buffer)
+                       (out "@uref{")
+                       (out-escaped (subseq string (1+ open-paren)
+                                            close-paren))
+                       (out ", ")
+                       (out (process-inline-markdown
+                             (subseq string (1+ i) close-bracket)))
+                       (out "}")
+                       (setf i close-paren))
+                      ;; Reflink: [label][id]
+                      (close-bracket2
+                       (flush-codifiable-buffer)
+                       ;; Process the name, drop the id
+                       (out (process-inline-markdown
+                             (subseq string (1+ i) close-bracket)))
+                       (setf i close-bracket2))
+                      ;; Not a recognized link structure, treat as a
+                      ;; normal character
+                      (t
+                       (buffer-codifiable-char char)))))
+                 (t
+                  (buffer-codifiable-char char)))
+               (incf i))
+      (flush-codifiable-buffer)
+      (get-output-stream-string out))))
+
+(when *downcase-uppercase-code*
+  (assert (equal (process-inline-markdown "`abc`") "@code{abc}"))
+  (assert (equal (process-inline-markdown "_abc_") "@emph{abc}"))
+  (assert (equal (process-inline-markdown "__abc__") "@strong{abc}"))
+  (assert (equal (process-inline-markdown "_PRINT_") "@emph{@code{print}}"))
+  (assert (equal (process-inline-markdown "<httpabc>") "@url{httpabc}"))
+  (assert (equal (process-inline-markdown "`N`") "@code{n}"))
+  (assert (equal (process-inline-markdown "`N`th") "@code{n}th"))
+  (assert (equal (process-inline-markdown "[x](uri)") "@uref{uri, x}"))
+  (assert (equal (process-inline-markdown "[`x`](uri)") "@uref{uri, @code{x}}"))
+  (assert (equal (process-inline-markdown "[function][type]") "function"))
+  (assert (equal (process-inline-markdown "[`function`][type]")
+                 "@code{function}")))
+
+
+;;;; Processing Markdown block elements
+
+;;; Collect lines enclosed in Markdown ``` fences. Returns the number
+;;; of lines consumed and a list of lines.
+(defun collect-fenced-code (lines starting-line base-indent)
+  (let* ((first-line (svref lines starting-line))
+         (trimmed (string-left-trim " " first-line)))
+    (when (and (>= (length trimmed) 3)
+               (string= (subseq trimmed 0 3) "```"))
+      (let ((lang (string-trim " " (subseq trimmed 3)))
+            (consumed 1)
+            (result nil))
+        (loop for index from (1+ starting-line) below (length lines)
+              for line = (svref lines index)
+              for line-trimmed = (string-left-trim " " line)
+              do (incf consumed)
+              if (and (>= (length line-trimmed) 3)
+                      (string= (subseq line-trimmed 0 3) "```"))
+                do (loop-finish)        ; Closing fence found
+              else
+                ;; Strip up to the base indentation of the environment
+                do (push (if (and (indentation line)
+                                  (>= (indentation line) base-indent))
+                             (subseq line base-indent)
+                             line)
+                         result))
+        (let ((env (if (string-equal lang "lisp") "lisp" "example")))
+          (values consumed
+                  `(,(format nil "@~A" env)
+                    ,@(mapcar #'escape-texinfo (nreverse result))
+                    ,(format nil "@end ~A" env))))))))
+
+;;; Collect lines that start with a Markdown blockquote marker (">").
+;;; A blockquote must be preceded by a blank line or be the first
+;;; line. The marker can be indented up to 3 characters on top of
+;;; BASE-INDENT. By leveraging string streams and passing the stripped
+;;; content recursively back to MARKDOWN-TO-TEXINFO, we maintain full
+;;; support for nested blocks, lists, and inline text wrapping.
+(defun collect-blockquote (lines starting-line base-indent)
+  (unless (and (> starting-line 0)
+               (not (blankp (svref lines (1- starting-line)))))
+    (let* ((first-line (svref lines starting-line))
+           (first-indent (indentation first-line)))
+      (when (and first-indent
+                 (<= first-indent (+ base-indent 3))
+                 (< first-indent (length first-line))
+                 (char= (char first-line first-indent) #\>))
+        (let ((n-lines 0)
+              (stripped-lines nil))
+          (loop for index from starting-line below (length lines)
+                for line = (svref lines index)
+                for indent = (indentation line)
+                do (cond
+                     ((and indent
+                           (<= indent (+ base-indent 3))
+                           (< indent (length line))
+                           (char= (char line indent) #\>))
+                      (let* ((start (1+ indent))
+                             (content-start
+                               (if (and (< start (length line))
+                                        (char= (char line start) #\Space))
+                                   (1+ start)
+                                   start)))
+                        (push (subseq line content-start) stripped-lines)
+                        (incf n-lines)))
+                     (t
+                      (loop-finish))))
+          ;; Trim trailing empty lines
+          (loop while (and stripped-lines (string= (car stripped-lines) ""))
+                do (pop stripped-lines) (decf n-lines))
+          (when stripped-lines
+            (let ((inner-texinfo
+                    ;; Process the stripped sub-document cleanly using
+                    ;; the main loop to handle paragraphs, nesting,
+                    ;; and formatting automatically.
+                    (with-output-to-string (*standard-output*)
+                      (markdown-to-texinfo
+                       (format nil "~{~A~^~%~}" (nreverse stripped-lines))
+                       *texinfo-local-variables*))))
+              (values n-lines
+                      `("@quotation"
+                        ,@(coerce (string-lines inner-texinfo) 'list)
+                        "@end quotation")))))))))
+
+;;; Collect lines indented with an extra 4 character on top of
+;;; BASE-INDENT. An indented code block must be preceeded by a blank
+;;; line or be the first line.
+(defun collect-indented-code (lines starting-line base-indent)
+  (unless (and (> starting-line 0)
+               (not (blankp (svref lines (1- starting-line)))))
+    (let ((indent (indentation (svref lines starting-line))))
+      (when (and indent (>= indent (+ base-indent 4)))
+        (let ((n-lines 0)
+              (result nil))
+          (loop for index from starting-line below (length lines)
+                for line = (svref lines index)
+                for line-indent = (indentation line)
+                do (cond
+                     ((blankp line)
+                      ;; Blank lines are allowed inside indented code blocks.
+                      (push "" result)
+                      (incf n-lines))
+                     ((>= line-indent (+ base-indent 4))
+                      (push (subseq line (+ base-indent 4)) result)
+                      (incf n-lines))
+                     (t
+                      ;; Indentation dropped, so the code block ends.
+                      (loop-finish))))
+          ;; Trim trailing empty lines.
+          (loop while (and result (string= (car result) ""))
+                do (pop result) (decf n-lines))
+          (if result
+              (values n-lines `("@example"
+                                ,@(mapcar #'escape-texinfo (nreverse result))
+                                "@end example"))
+              nil))))))
+
+;;; Return the indent if the line starts with a Markdown list marker
+;;; (#\- or \*) followed by a space.
+(defun maybe-itemize-offset (line)
+  (let ((indent (indentation line)))
+    (when indent
+      (let ((trimmed (string-left-trim " " line)))
+        (when (and (>= (length trimmed) 2)
+                   (member (char trimmed 0) '(#\- #\*))
+                   (char= (char trimmed 1) #\Space))
+          indent)))))
+
+;;; Collect a bulleted list.
+(defun collect-markdown-itemize (lines starting-line base-indent)
+  (let ((this-offset (maybe-itemize-offset (svref lines starting-line))))
+    (when (and this-offset (= this-offset base-indent))
+      (let ((result nil)
+            (lines-consumed 0)
+            (child-base (+ base-indent 4))
+            (current-paragraph nil)
+            (item-pending-p nil))
+        (flet ((flush-paragraph ()
+                 (if current-paragraph
+                     (let ((processed (process-inline-markdown
+                                       (format nil "~{~A~^~%~}"
+                                               (nreverse current-paragraph)))))
+                       (if item-pending-p
+                           (push (format nil "@item ~A" processed) result)
+                           (push processed result))
+                       (setf current-paragraph nil)
+                       (setf item-pending-p nil))
+                     (when item-pending-p
+                       (push "@item" result)
+                       (setf item-pending-p nil)))))
+          (loop for line-number = starting-line then (+ starting-line
+                                                        lines-consumed)
+                while (< line-number (length lines))
+                for line = (svref lines line-number)
+                for indent = (indentation line)
+                for offset = (maybe-itemize-offset line)
+                do (cond
+                     ((blankp line)
+                      ;; Blank lines inside lists are buffered
+                      (flush-paragraph)
+                      (push "" result)
+                      (incf lines-consumed))
+                     ;; New Item in the same list
+                     ((and offset (= offset base-indent))
+                      (flush-paragraph)
+                      (setf item-pending-p t)
+                      (let ((item-text (subseq line (+ offset 2))))
+                        (unless (blankp item-text)
+                          (push item-text current-paragraph)))
+                      (incf lines-consumed))
+                     ;; Indented block/text inside the list item (>= 4 spaces)
+                     ((and indent (>= indent child-base))
+                      (flush-paragraph)
+                      (multiple-value-bind (sub-consumed sub-result)
+                          (parse-markdown-block lines line-number child-base)
+                        (if sub-consumed
+                            (progn
+                              (setf result (append (reverse sub-result) result))
+                              (incf lines-consumed sub-consumed))
+                            ;; Fallback: normal text continuing the item body
+                            (progn
+                              (push (subseq line child-base) current-paragraph)
+                              (incf lines-consumed)))))
+                     ;; Normal text continuing the item body
+                     ((and indent (> indent base-indent))
+                      (push line current-paragraph)
+                      (incf lines-consumed))
+                     ;; If we get here, the line is NOT a new bullet,
+                     ;; and it less than 4 spaces of relative
+                     ;; indentation, so the list is over.
+                     (t
+                      (loop-finish))))
+          (flush-paragraph)
+          ;; Trim trailing empty lines, so they return to the outer scope.
+          (loop while (and result (string= (car result) ""))
+                do (pop result) (decf lines-consumed))
+          (values lines-consumed `("@itemize" ,@(reverse result)
+                                   "@end itemize")))))))
+
+;;; Parse the line at INDEX in LINES as a Markdown block. Return the
+;;; number of lines consumed and the parse.
+(defun parse-markdown-block (lines index base-indent)
+  (let ((line (svref lines index)))
+    (multiple-value-bind (n-lines-consumed result)
+        (collect-fenced-code lines index base-indent)
+      (when n-lines-consumed
+        (return-from parse-markdown-block (values n-lines-consumed result))))
+    (multiple-value-bind (n-lines-consumed result)
+        (collect-blockquote lines index base-indent)
+      (when n-lines-consumed
+        (return-from parse-markdown-block (values n-lines-consumed result))))
+    (cond
+      ((and (indentation line) (>= (indentation line) (+ base-indent 4)))
+       (collect-indented-code lines index base-indent))
+      ((maybe-itemize-offset line)
+       (collect-markdown-itemize lines index (maybe-itemize-offset line)))
+      (t nil))))

contrib/sb-manual/package.lisp

@@ -0,0 +1,9 @@
+(locally (declare (sb-ext:muffle-conditions sb-int:package-at-variance))
+  (handler-bind ((sb-int:package-at-variance #'muffle-warning))
+    (defpackage :sb-manual
+      (:use :cl :sb-alien)
+      (:export #:use-pax)
+      #+sb-doc
+      (:import-from #:sb-pcl
+       #:string-lines #:whitespacep #:indentation #:blankp
+       #:reindent-docstring))))

contrib/sb-manual/pax.lisp

@@ -0,0 +1,329 @@
+;;;; PAX stubs
+;;;;
+;;;; Contribs cannot depend on external libraries, so we fake as much
+;;;; of PAX and DRef as necessary. USE-PAX switches to the real
+;;;; implementation.
+;;;;
+;;;; If PAX is not loaded, the dummy DEFSECTION below still gives us
+;;;; the ability to use M-. on section names in docstrings as they are
+;;;; just variables, which makes navigating the documentation faster.
+;;;;
+;;;; When PAX is loaded, we have PAX::@BROWSING-LIVE-DOCUMENTATION for
+;;;; low-latency, interactive documentation work and
+;;;; PAX::@GENERATING-DOCUMENTATION for autolinked documentation
+;;;; (also, see make-pax-docs.sh and http://fixnum.com).
+;;;;
+;;;; For browsing, use this setup:
+;;;;
+;;;;     (setq pax:*document-downcase-uppercase-code* t
+;;;;           pax:*browse-context* :pax-world)
+
+(in-package :sb-manual)
+
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (defvar *using-pax* nil)
+  ;; A list of (LOCAL-SYMBOL PACKAGE) elements. Originally,
+  ;; LOCAL-SYMBOL has home package SB-MANUAL. For example, the element
+  ;; (SECTION :PAX) causes PAX:SECTION to be SHADOWING-IMPORTed in
+  ;; USE-PAX.
+  (defvar *dummies* ()))
+
+(defmacro defun-dummy ((name package) lambda-list &body body)
+  `(eval-when (:compile-toplevel :load-toplevel :execute)
+     (pushnew '(,name ,package) *dummies* :test #'equal)
+     (declaim (notinline ,name))
+     (unless *using-pax*
+       (defun ,name ,lambda-list ,@body))))
+
+(defmacro defmacro-dummy ((name package) lambda-list &body body)
+  (unless *using-pax*
+    `(eval-when (:compile-toplevel :load-toplevel :execute)
+       (pushnew '(,name ,package) *dummies* :test #'equal)
+       (unless *using-pax*
+         (defmacro ,name ,lambda-list ,@body)))))
+
+(defparameter *extra-dummies*
+  '((argument :pax)
+    (macro :dref)
+    (setf-function :dref)
+    (setf-generic-function :dref)
+    (section :pax)
+    (concept :pax)
+    (clhs :pax)
+    (docstring :dref)))
+
+(defun dummy (symbol)
+  (let ((*package* (find-package :sb-manual)))
+    (read-from-string (symbol-name symbol))))
+
+;;; We might want to populate this with GUESS-PACKAGE-FROM-ARGLIST.
+(defvar *definition-to-docstring-package*)
+(defvar *package-to-docstring-package*)
+
+(defun resolve-lazy-doc (&rest args)
+  (declare (ignore args))
+  (use-pax))
+
+(defun lazy-doc-name-p (symbol &optional kind)
+  (when (boundp symbol)
+    (let ((value (symbol-value symbol)))
+      (and (listp value)
+           (listp (first value))
+           (eq (caar value) :%pax-lazy-doc)
+           (or (null kind)
+               (eq (third (first value)) kind))))))
+
+(defun use-pax ()
+  "Ensure that exported variables are `PAX:SECTION`s.
+  It is an error if the `MGL-PAX` library is not loaded.
+
+  Calling this function explicitly is rarely necessary because it is
+  called automatically:
+
+  - when `SB-MANUAL` is loaded, if PAX is present;
+
+  - when `PAX:DOCUMENT` (more precisely, `DREF:LOCATE`) is called on
+    an `SB-MANUAL` section.
+
+  The latter feature requires v0.4.12 of PAX. See the `MGL-PAX`
+  asdf:system."
+  (unless *using-pax*
+    (assert (find-package '#:mgl-pax) ()
+            "The MGL-PAX package does not exist. Load PAX first.")
+    ;; Replace dummies with the real symbols.
+    (loop for (name package) in (append *dummies* *extra-dummies*)
+          do (let ((new-symbol (read-from-string
+                                (format nil "~A::~A" package name))))
+               (when (and (fboundp new-symbol)
+                          (null (macro-function new-symbol)))
+                 (setf (fdefinition name) (fdefinition new-symbol)))
+               (shadowing-import new-symbol :sb-manual)))
+    ;; Ensure PAX:DEFSECTION exports only SECTIONs.
+    (eval-string
+     "(defmethod pax:exportable-reference-p
+          ((package (eql (find-package 'sb-manual)))
+           symbol locative-type locative-args)
+        (eq locative-type 'section))")
+    ;; Reevaluate DEFSECTION and DEFINE-CONCEPT forms with PAX.
+    (do-symbols (symbol :sb-manual)
+      (when (lazy-doc-name-p symbol)
+        (let ((value (symbol-value symbol))
+              (source-location
+                (sb-int:info :source-location :variable symbol)))
+          (eval (subst-dummies (second value)))
+          (setf (sb-int:info :source-location :variable symbol)
+                source-location))))
+    (convert-docstring-package-overrides-to-pax)
+    (eval-string
+     "(pax:register-doc-in-pax-world
+       'sb-manual:@sbcl-manual @sbcl-manual
+       `((:objects
+          (, @sbcl-manual)
+          :source-uri-fn ,(pax:make-git-source-uri-fn
+                           nil \"https://github.com/sbcl/sbcl\"
+                           :git-root (asdf:system-relative-pathname
+                                      :sb-manual \"../../../\")))))")
+    (setq *using-pax* t)))
+
+;;; Convert *DEFINITION-TO-DOCSTRING-PACKAGE* to
+;;; DREF:DEFINITION-PROPERTIES and *PACKAGE-TO-DOCSTRING-PACKAGE* to
+;;; DREF:DEFINITION-PROPERTIES. See DREF-EXT:DOCSTRING*.
+(defun convert-docstring-package-overrides-to-pax ()
+  (loop for ((name locative) package) in *definition-to-docstring-package*
+        do (eval-format
+            "(setf (dref-ext:definition-property (dref:xref '~S '~S)~
+                                                 'docstring)
+                   (list nil (find-package ~S)))"
+            name (subst-dummies locative) package))
+  (setq *definition-to-docstring-package*
+        (subst-dummies *definition-to-docstring-package*))
+  (loop for (from-package to-package) in *package-to-docstring-package*
+        do (eval-format
+            "(setf (dref-ext:definition-property `(:package ,(find-package ~S))
+                                                 'docstring)
+                   (list nil (find-package ~S)))"
+            from-package to-package)))
+
+(defun eval-string (string)
+  (let ((*package* (find-package :sb-manual)))
+    (eval (read-from-string string))))
+
+(defun eval-format (format-control &rest format-args)
+  (let ((*package* (find-package :sb-manual)))
+    (eval (read-from-string (apply #'format nil format-control format-args)))))
+
+(defun subst-dummies (tree)
+  (let ((new-tree tree))
+    (loop
+      for (name package) in (append *dummies* *extra-dummies*)
+      do (let ((new-name (read-from-string
+                          (format nil "~A::~A" package name))))
+           (setq new-tree (subst new-name name new-tree))))
+    new-tree))
+
+
+;;; The main job DEFSECTION is to associate a title and a list of
+;;; "entries" with a name. An entry is either a docstring or a
+;;; reference to a definition of the form (<NAME> <DEFINITION-TYPE>).
+;;;
+;;; Concepts and index keys may also be used in DEFSECTION forms:
+;;;
+;;;     (defsection @the-repl (:title "The REPL" :concepts (@repl))
+;;;        "The REPL is ...")
+;;;
+;;; When processed, this becomes
+;;;
+;;;    @cindex Read-Eval-Print Loop
+;;;    @cindex REPL
+;;;    @node the repl
+;;;    @section The REPL
+;;;    The REPL is ...
+;;;
+;;; Note that it doesn't matter in :KEYS of DEFSECTION whether a
+;;; concept is pure or titled.
+;;;
+;;; Note that referencing sections in docstrings does not cause
+;;; indexing. This is due to examples like
+;;;
+;;;     We now move on to the next section, @XXX.
+;;;     See @XXX, for other considerations.
+;;;
+;;; In general, mentioning and linking a section is not the same as
+;;; being about it, and only the latter deserves a concept index
+;;; entry.
+;;;
+;;; See also PAX:DEFSECTION.
+(defmacro-dummy (defsection pax)
+                (name (&key (package *package*) (export t) title concepts)
+                      &body entries)
+  (let ((value
+          `((:%pax-lazy-doc ,name :section resolve-lazy-doc)
+            (defsection ,name (:package ,package :export ,export :title ,title
+                               :concepts ,concepts)
+              ,@entries))))
+    `(progn
+       (defparameter ,name ',value)
+       ,@(when export
+           `((export ',name :sb-manual))))))
+
+(defun-dummy (section-name :pax) (section)
+  (second (second section)))
+
+(defun-dummy (section-title :pax) (section)
+  (getf (third (second section)) :title))
+
+(defun-dummy (concept-keys :pax) (section)
+  (resolve-concept-symbols (getf (third (second section)) :concepts)))
+
+(defun-dummy (section-package :pax) (section)
+  (find-package (getf (third (second section)) :package)))
+
+;;; This is a list of (NAME LOCATIVE) elements with our dummy DEFSECTION.
+(defun-dummy (section-entries :pax) (section)
+  (nthcdr 3 (second section)))
+
+(defun-dummy (xref-name :dref) (xref)
+  (first xref))
+
+(defun-dummy (xref-locative :dref) (xref)
+  (normalize-locative (second xref)))
+
+(defun normalize-locative (locative)
+  (if (and (listp locative)
+           (null (cdr locative)))
+      (first locative)
+      locative))
+
+(defun-dummy (xref-locative-type :dref) (xref)
+  (first (sb-c::ensure-list (second xref))))
+
+
+;;; DEFINE-CONCEPT names a set of index keys. When MARKDOWN-TO-TEXINFO
+;;; encounters the name of a concept in a docstring, it emits Texinfo
+;;; `@cindex' lines for its keys. There are two kinds of concepts.
+;;;
+;;; When a `pure' concept (that has no title) is processed in a
+;;; docstring by MARKDOWN-TO-TEXINFO, it produces no visible output,
+;;; but it emits Texinfo @cindex lines for its KEYS. It is thus _not_
+;;; part of the normal flow of text.
+;;;
+;;;     (define-concept ~repl (:keys ("Read-Eval-Print Loop" "REPL")))
+;;;
+;;; Example use:
+;;;
+;;;     The REPL ~REPL is interactive.
+;;;
+;;; This becomes
+;;;
+;;;     @cindex Read-Eval-Print Loop
+;;;     @cindex REPL
+;;;     The REPL  is interactive.
+;;;
+;;; When a `titled' concept is processed, it is replaced by its title.
+;;; Use this in the normal flow of text like section names. Note that
+;;; other concepts can be referenced in KEYS. The definition below is
+;;; the titled synonym of ~REPL.
+;;;
+;;;     (define-concept @repl (:title "REPL" :keys (~repl)))
+;;;
+;;; Example use:
+;;;
+;;;     The @REPL is interactive.
+;;;
+;;; This becomes
+;;;
+;;;     @cindex Read-Eval-Print Loop
+;;;     @cindex REPL
+;;;     The REPL is interactive.
+;;;
+;;; By convention, names of pure concepts start with #\~, while those
+;;; of titled concepts start with the usual #\@. The tilde is to draw
+;;; attention to that the name is not part of the normal text.
+;;;
+;;; Finally, index keys can be hierarchical:
+;;;
+;;;     (define-concept ~defining-macros (:keys (("defining" "macros")
+;;;                                              ("macros," "defining"))))
+;;;
+;;; This is processed into
+;;;
+;;;     @cindex defining @subentry macros
+;;;     @cindex macros, @subentry defining
+;;;
+;;; Note Texinfo's @sortas is not supported yet. PAX implements that
+;;; by allowing any subkey string to be a (<NAME> . <SORT-AS-STRING>)
+;;; cons.
+;;;
+;;; See also PAX:DEFINE-CONCEPT.
+(defmacro-dummy (define-concept pax) (name (&key title keys))
+  (let ((value
+          `((:%pax-lazy-doc ,name :concept resolve-lazy-doc)
+            (define-concept ,name (:title ,title :keys ,keys)))))
+    `(defparameter ,name ',value)))
+
+(defun-dummy (multiplexing-concept-keys :pax) (concept)
+  (resolve-concept-symbols (getf (third (second concept)) :keys)))
+
+(defun-dummy (doctitle :pax) (concept)
+  (getf (third (second concept)) :title))
+
+(defun resolve-concept-symbols (list)
+  (if (find-if #'symbolp list)
+      (loop for x in list
+            append (if (symbolp x)
+                       (multiplexing-concept-keys (find-concept x))
+                       (list x)))
+      list))
+
+(defun find-concept (name)
+  (if (doc-name-p name :concept)
+      (symbol-value name)
+      (error "Undefined ~S ~S." 'concept name)))
+
+(defun doc-name-p (symbol kind)
+  (if *using-pax*
+      (and (boundp symbol)
+           (typep (symbol-value symbol) (dummy (ecase kind
+                                                 (:section 'section)
+                                                 (:concept 'concept)))))
+      (lazy-doc-name-p symbol kind)))

contrib/sb-manual/sb-manual.asd

@@ -0,0 +1,44 @@
+(error "Can't build contribs with ASDF")
+
+(defsystem "sb-manual"
+  :serial t
+  :components ((:file "package")
+               (:file "pax")
+               (:file "docstring")
+               (:file "markdown" :if-feature :sb-doc)
+               (:file "texinfo" :if-feature :sb-doc)
+               (:file "manual")
+               (:module "doc/"
+                :serial t
+                :components ((:file "concepts")
+                             (:file "sbcl")
+                             (:file "support-and-bugs")
+                             (:file "intro")
+                             (:file "start-stop")
+                             (:file "compiler")
+                             (:file "debugger")
+                             (:file "efficiency")
+                             (:file "beyond-ansi")
+                             (:file "external-formats")
+                             (:file "ffi")
+                             (:file "pathnames")
+                             (:file "streams")
+                             (:file "package-locks")
+                             (:file "threading")
+                             (:file "timers")
+                             (:file "../../sb-bsd-sockets/manual")
+                             (:file "profiling")
+                             (:file "contrib-modules")
+                             (:file "../../sb-aclrepl/manual")
+                             (:file "../../sb-concurrency/manual")
+                             (:file "../../sb-cover/manual")
+                             (:file "../../sb-grovel/manual")
+                             (:file "../../sb-introspect/manual")
+                             (:file "../../sb-md5/manual")
+                             (:file "../../sb-posix/manual")
+                             (:file "../../sb-queue/manual")
+                             (:file "../../sb-rotate-byte/manual")
+                             (:file "../../sb-simd/manual")
+                             (:file "../../sb-simple-streams/manual")
+                             (:file "deprecation")))
+               (:file "late")))

contrib/sb-manual/sb-manual.texinfo

@@ -0,0 +1,119 @@
+@c Generated by the sb-manual contrib. Do not edit.
+
+@node sb manual
+@section sb-manual
+
+@menu
+* Using PAX: using pax.
+* Browsing Live with PAX: browsing live with pax.
+* Fancy Documentation with PAX: fancy documentation with pax.
+@end menu
+
+The @code{sb-manual} module has the SBCL user manual in forms
+mimicking @code{pax:defsection}:
+
+@example
+(defsection @@example (:title "Example")
+  "This is an example, but see the real @@SB-MANUAL."
+  (print function)
+  (@@subexample section))
+@end example
+
+The names of the variables holding the documentation are exported
+from the @code{sb-manual} package. Since sections are basically
+variables, in Slime, @code{M-.} on @code{"@@SB-MANUAL"}, @code{"print"}, or
+on @code{"@@subexample"} will take you to the respective definition.
+This makes it easy to navigate the documentation. Normal Lisp
+definition docstrings and section docstrings reference sections
+following the usual convention of uppercasing the name. Docstrings
+are in a subset of Markdown and use very little markup in general,
+so they are easy to read directly in the source.
+
+The official manual in Info, HTML and PDF formats is generated via
+Texinfo generated from these definitions.
+
+@node using pax
+@subsection Using PAX
+
+However, @code{sb-manual::defsection} is but a dummy implementation of
+@code{pax:defsection} to avoid a hard dependency on PAX.
+
+See the @code{mgl-pax} @code{asdf:system} or @url{https://github.com/melisgl/mgl-pax/}.
+
+When PAX is loaded, the dummy @code{defsection} definitions are made
+real, so that PAX can work with them.
+
+@anchor{Function sb-manual use-pax}
+@ffindex @sortas{use-pax sb-manual} use-pax [sb-manual]
+@deffn{Function} sb-manual:use-pax
+Ensure that exported variables are @code{pax:section}s.
+It is an error if the @code{mgl-pax} library is not loaded.
+
+Calling this function explicitly is rarely necessary because it is
+called automatically:
+
+@itemize
+@item when @code{sb-manual} is loaded, if @code{pax} is present;
+
+@item when @code{pax:document} (more precisely, @code{dref:locate}) is called on
+  an @code{sb-manual} section.
+@end itemize
+
+The latter feature requires v0.4.12 of @code{pax}. See the @code{mgl-pax}
+asdf:system.
+@end deffn
+@node browsing live with pax
+@subsection Browsing Live with PAX
+
+With PAX, you can browse the manual live. The documentation of this
+feature is available at
+@uref{https://melisgl.github.io/mgl-pax-world/pax-manual.html#MGL-PAX:@@BROWSING-LIVE-DOCUMENTATION%20MGL-PAX:SECTION, online}.
+
+If you are browsing this manual live right now, here is the
+equivalent live link: @code{pax::@@browsing-live-documentation}.
+
+Notable features:
+
+@itemize
+@item Autolinks within the manual: if @code{sb-ext:exit} is mentioned, then
+  it's linked to its documentation. You basically get links to where
+  @code{M-.} would go in the sources.
+
+@item Autolinks to the CLHS.
+
+@item View the documentation of any Lisp definition or section without
+  generating the entire manual.
+
+@item Locatives (e.g. the @code{"[function]"} in @code{"- [function]
+  SB-EXT:EXIT"}) are also links in live browsing: they tell Slime
+  to visit the definition.
+
+For this to work, you need to allow Slime to evaluate Elisp sent
+from SBCL:
+
+@example
+(setq slime-enable-evaluate-in-emacs t)
+@end example
+
+and maybe your window manager focus stealing configuration needs
+tweaking as well.
+@end itemize
+
+Live browsing can greatly reduce the latency of Edit-Compile-View
+Loop, when working on documentation.
+
+@node fancy documentation with pax
+@subsection Fancy Documentation with PAX
+
+PAX can generate dead documentation, too. In the SBCL sources,
+@code{contrib/sb-manual/make-pax-docs.sh} generates the manual in plain
+text, Markdown, PDF, and HTML formats. These differ from those
+generated via Texinfo in that they are autolinked (like when
+@ref{browsing live with pax}).
+
+Also, you can generate documentation yourself with e.g.
+
+@example
+(pax:document sb-manual:@@sbcl-manual :format :markdown)
+@end example
+

contrib/sb-manual/texinfo.lisp

@@ -0,0 +1,315 @@
+(in-package :sb-manual)
+
+(defun xref-defined-p (xref)
+  (let ((name (xref-name xref))
+        (locative-type (xref-locative-type xref)))
+    (case locative-type
+      ((function generic-function)
+       (ignore-errors (fdefinition name)))
+      ((variable)
+       (member (sb-int:info :variable :kind name)
+               '(:global :special :constant)))
+      ((declaration)
+       (find name (sb-cltl2:declaration-information 'declaration)))
+      ((class structure condition)
+       (find-class name nil))
+      ((type)
+       (sb-ext:defined-type-name-p name))
+      (t
+       (cond ((eq locative-type (dummy 'macro))
+              (ignore-errors (macro-function name)))
+             ((or (eq locative-type (dummy 'setf-function))
+                  (eq locative-type (dummy 'setf-generic-function)))
+              (ignore-errors (fdefinition name)))
+             (t
+              (assert nil () "Unexpected locative type in ~S."
+                      xref)))))))
+
+;;; We don't DEFINE-DUMMY DREF:ARGLIST and DREF:DOCSTRING because we
+;;; don't want USE-PAX to affect Texinfo output, which it would
+;;; because DREF:ARGLIST differs from the {incom,re}prehensible
+;;; LAMBDA-LIST*.
+(defun %arglist (xref)
+  (let ((name (xref-name xref))
+        (locative-type (xref-locative-type xref)))
+    (lambda-list* name locative-type)))
+
+(defun %docstring (xref)
+  (let ((sb-pcl::*normalize-sbcl-docstrings* nil))
+    (values (let ((name (xref-name xref))
+                  (locative-type (xref-locative-type xref)))
+              (case locative-type
+                ((function variable declaration)
+                 (documentation name locative-type))
+                ((generic-function)
+                 (documentation name 'function))
+                ((type class structure condition)
+                 (documentation name 'type))
+                (t
+                 (cond ((eq locative-type (dummy 'macro))
+                        (documentation (macro-function name) t))
+                       ((eq locative-type (dummy 'setf-function))
+                        (documentation (fdefinition name) t))
+                       ((eq locative-type (dummy 'setf-generic-function))
+                        (documentation (fdefinition name) t))
+                       (t
+                        (assert nil () "Unexpected locative type in ~S."
+                                xref))))))
+            ;; To be compatible with PAX::@PACKAGE-AND-READTABLE, we
+            ;; always return a non-NIL package.
+            (docstring-package xref))))
+
+(defun lambda-list* (name kind)
+  (case kind
+    ((package constant variable type structure class condition method
+              declaration nil)
+     nil)
+    (t
+     ;; KLUDGE: Eugh.
+     ;;
+     ;; believe it or not, the above comment was written before CSR
+     ;; came along and obfuscated this.  (2005-07-04)
+     (when (symbolp name)
+       (labels ((clean (x &key optional key)
+                  (typecase x
+                    (atom x)
+                    ((cons (member &optional))
+                     (cons (car x) (clean (cdr x) :optional t)))
+                    ((cons (member &key))
+                     (cons (car x) (clean (cdr x) :key t)))
+                    ((cons (member &whole &environment))
+                     ;; Skip these
+                     (clean (cdr x) :optional optional :key key))
+                    ((cons cons)
+                     (cons
+                      (cond (key (if (consp (caar x))
+                                     (caaar x)
+                                     (caar x)))
+                            (optional (caar x))
+                            (t (clean (car x))))
+                      (clean (cdr x) :key key :optional optional)))
+                    (cons
+                     (cons
+                      (cond ((or key optional) (car x))
+                            (t (clean (car x))))
+                      (clean (cdr x) :key key :optional optional))))))
+         (multiple-value-bind (ll unknown)
+             (sb-introspect:function-lambda-list name)
+           (if unknown
+               (values nil t)
+               (clean ll))))))))
+
+
+(defun locative-type-to-texinfo (locative-type)
+  (case locative-type
+    (function
+     (values "Function" "ffindex"))
+    (generic-function
+     (values "Generic function" "ffindex"))
+    (variable
+     (values "Variable" "vvindex"))
+    (class
+     (values "Class" "ttindex"))
+    (condition
+     (values "Condition" "ttindex"))
+    (structure
+     (values "Structure" "ttindex"))
+    (type
+     (values "Type" "ttindex"))
+    (declaration
+     (values "Declaration" "ddindex"))
+    (t
+     (cond
+       ((eq locative-type (dummy 'macro))
+        (values "Macro" "ffindex"))
+       ((eq locative-type (dummy 'setf-function))
+        (values "Setf function" "ffindex"))
+       ((eq locative-type (dummy 'setf-generic-function))
+        (values "Setf generic function" "ffindex"))
+       (t
+        (assert nil () "Unexpected locative type ~S." locative-type))))))
+
+(defmacro with-texinfo-to-file (file &body body)
+  `(call-maybe-with-texinfo-to-file (lambda () ,@body)
+                                   ,file))
+
+(defun call-maybe-with-texinfo-to-file (fn file)
+  (if file
+      (with-open-file (*standard-output* file :direction :output
+                                         :if-does-not-exist :create
+                                         :if-exists :supersede)
+        (format t "@c Generated by the sb-manual contrib. Do not edit.~%~%")
+        (funcall fn))
+      (funcall fn)))
+
+(defun remove-markup (string)
+  (remove #\\ string))
+
+;;; Write the Texinfo for SECTION to *STANDARD-OUTPUT*. When recursing
+;;; into child sections, if a section is in PAGES, then emit an
+;;; @include and open a new a file for output.
+(defun emit-texinfo-for-section (section &key pages (depth 0)
+                                 top-level-menus-to-file
+                                 top-level-contents-to-file)
+  (let ((title (remove-markup (section-title section)))
+        (entries (section-entries section)))
+    (format t "@node ~A~%" (texinfo-node-id section))
+    (write-concept-keys (concept-keys section) *standard-output*)
+    (format t "~A ~A~%~%"
+            (ecase depth
+              (0 "@top")
+              (1 "@chapter")
+              (2 "@section")
+              (3 "@subsection")
+              (4 "@subsubsection"))
+            title)
+    ;; Generate the @menu
+    (let ((child-sections
+            (loop for entry in entries
+                  when (and (not (stringp entry))
+                            (eq (xref-locative-type entry) (dummy 'section)))
+                    collect (symbol-value (xref-name entry)))))
+      (when child-sections
+        (unless top-level-menus-to-file
+          (format t "@menu~%"))
+        (with-texinfo-to-file top-level-menus-to-file
+          (dolist (child-section child-sections)
+            (format t "* ~A: ~A.~%"
+                    (remove-markup (section-title child-section))
+                    (texinfo-node-id child-section))))
+        (unless top-level-menus-to-file
+          (format t "@end menu~%~%"))))
+    ;; Generate the documentation
+    (let ((*package* (section-package section)))
+      (with-texinfo-to-file top-level-contents-to-file
+        (dolist (entry entries)
+          (cond ((stringp entry)
+                 ;; KLUDGE: @SBCL-MANUAL has an extra docstring that's
+                 ;; pretty much the same as @copying in
+                 ;; doc/manual/sbcl.texinfo. Skip it.
+                 (unless top-level-contents-to-file
+                   (emit-texinfo-for-docstring entry)
+                   (format t "~%")))
+                (t
+                 (if (not (eq (xref-locative-type entry) (dummy 'section)))
+                     (emit-texinfo-for-definition entry)
+                     (let ((page (find (xref-name entry) pages
+                                       :key #'first)))
+                       (when page
+                         (format t "@include ~A~%" (second page)))
+                       (with-texinfo-to-file (second page)
+                         (emit-texinfo-for-section
+                          (symbol-value (xref-name entry))
+                          :pages pages
+                          :depth (1+ depth))))))))))))
+
+(defun emit-texinfo-for-definition (xref)
+  (if (not (xref-defined-p xref))
+      (warn "~@<Not documenting ~S because it is not defined.~:@>" xref)
+      (multiple-value-bind (docstring *package*) (%docstring xref)
+        (multiple-value-bind (type index)
+            (locative-type-to-texinfo (xref-locative-type xref))
+          (let* ((name (xref-name xref))
+                 (*print-case* :downcase)
+                 ;; For e.g. #'print
+                 (*print-pretty* t)
+                 ;; The arglist must be on the @deffn line.
+                 (*print-right-margin* most-positive-fixnum))
+            (format t "@anchor{~A ~A ~A}~%" type
+                    (string-downcase (package-name (symbol-package name)))
+                    (string-downcase (symbol-name name)))
+            ;; E.g. @vvindex @sortas{save-hooks* sb-ext} *save-hooks* [sb-ext]
+            (let ((symbol-name (string-downcase (symbol-name name)))
+                  (symbol-package-name
+                    (string-downcase (package-name (symbol-package name)))))
+              (format t "@~A @sortas{~A ~A} ~A [~A]~%"
+                      index
+                      (sort-as-name symbol-name)
+                      (sort-as-name symbol-package-name)
+                      symbol-name
+                      symbol-package-name))
+            ;; Since we took indexing into our own hands, we just use
+            ;; @deffn for all definitions. We could also use @defblock and
+            ;; @defline.
+            (format t "@deffn{~A} ~A~{ ~A~}~%"
+                    ;; E.g. "Variable"
+                    type
+                    (let ((*package* (find-package :cl)))
+                      (prin1-to-string name))
+                    (%arglist xref))
+            (when docstring
+              (emit-texinfo-for-docstring docstring (%arglist xref)))
+            (format t "@end deffn~%"))))))
+
+;;; Remove leading non-alphanumeric characters. They are not important
+;;; when sorting names into indices.
+(defun sort-as-name (name)
+  (subseq name (or (position-if #'alphanumericp name) 0)))
+
+(defun emit-texinfo-for-docstring (docstring &optional arglist)
+  (markdown-to-texinfo (reindent-docstring docstring) arglist))
+
+
+;;; Currently, we have the Texinfo file under version control to keep
+;;; a closer eye on the Markdown-to-Texinfo converter, which is young.
+;;; When that's no longer the case, this is no longer needed.
+(defparameter *pages*
+  '((@support-and-bugs "support-and-bugs.texinfo")
+    (@introduction "intro.texinfo")
+    (@starting-and-stopping "start-stop.texinfo")
+    (@compiler "compiler.texinfo")
+    (@debugger "debugger.texinfo")
+    (@efficiency "efficiency.texinfo")
+    (@beyond-the-ansi-standard "beyond-ansi.texinfo")
+    (@external-formats "external-formats.texinfo")
+    (@foreign-function-interface "ffi.texinfo")
+    (@pathnames "pathnames.texinfo")
+    (@streams "streams.texinfo")
+    (@package-locks "package-locks.texinfo")
+    (@threading "threading.texinfo")
+    (@timers "timers.texinfo")
+    (@networking "../../contrib/sb-bsd-sockets/sb-bsd-sockets.texinfo")
+    (@profiling "profiling.texinfo")
+    (@statistical-profiler "../../contrib/sb-sprof/sb-sprof.texinfo")
+    (@contributed-modules "contrib-modules.texinfo")
+    (@sb-aclrepl "../../contrib/sb-aclrepl/sb-aclrepl.texinfo")
+    (@sb-concurrency "../../contrib/sb-concurrency/sb-concurrency.texinfo")
+    (@sb-cover "../../contrib/sb-cover/sb-cover.texinfo")
+    (@sb-grovel "../../contrib/sb-grovel/sb-grovel.texinfo")
+    (@sb-introspect "../../contrib/sb-introspect/sb-introspect.texinfo")
+    (@sb-manual "../../contrib/sb-manual/sb-manual.texinfo")
+    (@sb-md5 "../../contrib/sb-md5/sb-md5.texinfo")
+    (@sb-posix "../../contrib/sb-posix/sb-posix.texinfo")
+    (@sb-queue "../../contrib/sb-queue/sb-queue.texinfo")
+    (@sb-rotate-byte "../../contrib/sb-rotate-byte/sb-rotate-byte.texinfo")
+    (@sb-sb-simd "../../contrib/sb-simd/sb-simd.texinfo")
+    (@sb-simple-streams
+     "../../contrib/sb-simple-streams/sb-simple-streams.texinfo")
+    (@deprecation "deprecation.texinfo")))
+
+(defun generate-texinfo ()
+  (let ((*default-pathname-defaults*
+          (truename (merge-pathnames
+                     "../../doc/manual/"
+                     sb-sys::*sbcl-homedir-pathname*))))
+    (with-texinfo-to-file "variables.texinfo"
+      (format t "@set VERSION ~A~%~
+                 @set UPDATE-MONTH ~A~%"
+              (lisp-implementation-version)
+              (documentation-generation-date-string)))
+    ;; We redirect most lines via *PAGES*, :TOP-LEVEL-MENUS-TO-FILE,
+    ;; :TOP-LEVEL-CONTENTS-TO-FILE. Silence the rest, which are not
+    ;; needed, as sbcl.texinfo only needs the includes.
+    (let ((*standard-output* (make-broadcast-stream)))
+      (emit-texinfo-for-section
+       (symbol-value '@sbcl-manual) :pages *pages*
+       :top-level-menus-to-file "sbcl-menu.texinfo"
+       :top-level-contents-to-file "sbcl-contents.texinfo"))))
+
+#+nil
+(generate-texinfo)
+
+#+nil
+(emit-texinfo-for-section @sb-aclrepl)
+#+nil
+(emit-texinfo-for-section @starting-and-stopping)

contrib/sb-md5/manual.lisp

@@ -0,0 +1,18 @@
+(in-package :sb-manual)
+
+(defsection @sb-md5 (:title "sb-md5")
+  ;; FIXME: cite
+  "The `SB-MD5` module implements the RFC1321 MD5 Message Digest
+  Algorithm."
+  (sb-md5:md5sum-file function)
+  (sb-md5:md5sum-sequence function)
+  (sb-md5:md5sum-stream function)
+  (sb-md5:md5sum-string function)
+  "The implementation for CMUCL was largely done by Pierre Mai, with help
+  from members of the `cmucl-help` mailing list. Since CMUCL and SBCL
+  are similar in many respects, it was not too difficult to extend the
+  low-level implementation optimizations for CMUCL to SBCL. Following
+  this, SBCL's compiler was extended to implement efficient
+  compilation of modular arithmetic (@MODULAR-ARITHMETIC), which
+  enabled the implementation to be expressed in portable arithmetical
+  terms, apart from the use of @SB-ROTATE-BYTE for bitwise rotation.")

contrib/sb-md5/md5.lisp

@@ -314,8 +314,8 @@ registers A, B, C and D."
 
 (defun update-md5-block (regs block)
   "This is the core part of the MD5 algorithm.  It takes a complete 16
-word block of input, and updates the working state in A, B, C, and D
-accordingly."
+word block of input, and updates the working state in registers A,
+B, C, and D accordingly."
   (declare (type md5-regs regs)
            (type md5-block block)
            (optimize (speed 3) (safety 0) (space 0) (debug 0) #+lw-int32 (float 0)))
@@ -358,7 +358,7 @@ accordingly."
 (declaim (inline fill-block fill-block-ub8 fill-block-char))
 (defun fill-block-ub8 (block buffer offset)
   "Convert a complete 64 (unsigned-byte 8) input vector segment
-starting from `offset' into the given 16 word MD5 block."
+starting from OFFSET into the given 16 word MD5 block."
   (declare (type (integer 0 #.(- most-positive-fixnum 64)) offset)
            (type md5-block block)
            (type (simple-array (unsigned-byte 8) (*)) buffer)
@@ -384,7 +384,7 @@ starting from `offset' into the given 16 word MD5 block."
 
 (defun fill-block-char (block buffer offset)
   "DEPRECATED: Convert a complete 64 character input string segment
-starting from `offset' into the given 16 word MD5 block."
+starting from OFFSET into the given 16 word MD5 block."
   (declare (type (integer 0 #.(- most-positive-fixnum 64)) offset)
            (type md5-block block)
            (type simple-string buffer)
@@ -412,9 +412,9 @@ starting from `offset' into the given 16 word MD5 block."
 (defun fill-block (block buffer offset)
   "Convert a complete 64 byte input vector segment into the given 16
 word MD5 block.  This currently works on (unsigned-byte 8) and
-character simple-arrays, via the functions `fill-block-ub8' and
-`fill-block-char' respectively.  Note that it will not work correctly
-on character simple-arrays if `char-code-limit' is greater than 256."
+character simple-arrays, via the functions FILL-BLOCK-UB8 and
+FILL-BLOCK-CHAR respectively.  Note that it will not work correctly
+on character simple-arrays if CHAR-CODE-LIMIT is greater than 256."
   (declare (type (integer 0 #.(- most-positive-fixnum 64)) offset)
            (type md5-block block)
            (type (simple-array * (*)) buffer)
@@ -431,7 +431,7 @@ on character simple-arrays if `char-code-limit' is greater than 256."
 (declaim (inline md5regs-digest))
 (defun md5regs-digest (regs)
   "Create the final 16 byte message-digest from the MD5 working state
-in `regs'.  Returns a (simple-array (unsigned-byte 8) (16))."
+in REGS.  Returns a (simple-array (unsigned-byte 8) (16))."
   (declare (optimize (speed 3) (safety 0) (space 0) (debug 0)
                      #+lw-int32 (float 0) #+lw-int32 (hcl:fixnum-safety 0))
            (type md5-regs regs))
@@ -474,9 +474,9 @@ in `regs'.  Returns a (simple-array (unsigned-byte 8) (16))."
 
 (declaim (inline copy-to-buffer))
 (defun copy-to-buffer (from from-offset count buffer buffer-offset)
-  "Copy a partial segment from input vector `from' starting at
-`from-offset' and copying `count' elements into the 64 byte buffer
-starting at `buffer-offset'."
+  "Copy a partial segment from input vector FROM starting at
+FROM-OFFSET and copying COUNT elements into the 64 byte buffer
+starting at BUFFER-OFFSET."
   (declare (optimize (speed 3) (safety 0) (space 0) (debug 0)
                      #+lw-int32 (float 0) #+lw-int32 (hcl:fixnum-safety 0))
            (type sb-int:index from-offset)
@@ -510,12 +510,12 @@ starting at `buffer-offset'."
                        from-index))))))
 
 (defun update-md5-state (state sequence &key (start 0) (end (length sequence)))
-  "Update the given md5-state from `sequence', which is either a
+  "Update the given md5-state from SEQUENCE, which is either a
 simple-string or a simple-array with element-type (unsigned-byte 8),
-bounded by `start' and `end', which must be numeric bounding-indices.
+bounded by START and END, which must be numeric bounding-indices.
 Note that usage on simple-strings is DEPRECATED, since this will not
-work correctly if `char-code-limit' is more than 256.  String input
-should be converted to (unsigned-byte 8) simple-arrays with
+work correctly if CHAR-CODE-LIMIT is more than 256.  String input
+should be converted to (UNSIGNED-BYTE 8) simple-arrays with
 external-format conversion routines beforehand."
   (declare (type md5-state state)
            (type (simple-array * (*)) sequence)
@@ -591,8 +591,8 @@ by processing any remaining input in its buffer, with suitable padding
 and appended bit-length, as specified by the MD5 standard.
 
 The resulting MD5 message-digest is returned as an array of sixteen
-(unsigned-byte 8) values.  Calling `update-md5-state' after a call to
-`finalize-md5-state' results in unspecified behaviour."
+(unsigned-byte 8) values.  Calling UPDATE-MD5-STATE after a call to
+FINALIZE-MD5-STATE results in unspecified behaviour."
   (declare (type md5-state state)
            (optimize (speed 3) (safety 1) (space 0) (debug 1) #+lw-int32 (float 0)))
   (locally
@@ -633,8 +633,8 @@ The resulting MD5 message-digest is returned as an array of sixteen
 ;;; High-Level Drivers
 
 (defun md5sum-sequence (sequence &key (start 0) end)
-  "Calculate the MD5 message-digest of data in `sequence', which should
-be a 1d simple-array with element type (unsigned-byte 8).  On CMU CL
+  "Calculate the MD5 message-digest of data in SEQUENCE, which should
+be a 1d SIMPLE-ARRAY with element type (UNSIGNED-BYTE 8). On CMU CL
 and SBCL non-simple and non-1d arrays with this element-type are also
 supported."
   (declare (optimize (speed 3) (safety 3) (space 0) (debug 1))
@@ -666,8 +666,8 @@ supported."
 
 (defun md5sum-string (string &key (external-format :default) (start 0) end)
   "Calculate the MD5 message-digest of the binary representation of
-`string' (as octets) in the external format specified by
-`external-format'. The boundaries `start' and `end' refer to character
+STRING (as octets) in the external format specified by
+EXTERNAL-FORMAT. The boundaries START and END refer to character
 positions in the string, not to octets in the resulting binary
 representation.  The permissible external format specifiers are
 determined by the underlying implementation."
@@ -716,16 +716,16 @@ determined by the underlying implementation."
 
 (eval-when (:compile-toplevel :load-toplevel :execute)
   (defconstant +buffer-size+ (* 128 1024)
-    "Size of internal buffer to use for `md5sum-stream' and `md5sum-file'
+    "Size of internal buffer to use for MD5SUM-STREAM and MD5SUM-FILE
 operations.  This should be a multiple of 64, the MD5 block size."))
 
 (deftype buffer-index () `(integer 0 ,+buffer-size+))
 
 (defun md5sum-stream (stream)
-  "Calculate an MD5 message-digest of the contents of `stream'.  Its
-element-type has to be (unsigned-byte 8). Use on character streams is
+  "Calculate an MD5 message-digest of the contents of STREAM.  Its
+element-type has to be (UNSIGNED-BYTE 8). Use on character streams is
 DEPRECATED, as this will not work correctly on implementations with
-`char-code-limit' > 256 and ignores character coding issues."
+CHAR-CODE-LIMIT > 256 and ignores character coding issues."
   (declare (optimize (speed 3) (safety 3) (space 0) (debug 1)))
   (locally
       (declare (optimize (safety 1) (debug 0)))
@@ -755,7 +755,7 @@ DEPRECATED, as this will not work correctly on implementations with
                 (stream-element-type stream) stream))))))
 
 (defun md5sum-file (pathname)
-  "Calculate the MD5 message-digest of the file specified by `pathname'."
+  "Calculate the MD5 message-digest of the file specified by PATHNAME."
   (declare (optimize (speed 3) (safety 3) (space 0) (debug 1)))
   (with-open-file (stream pathname :element-type '(unsigned-byte 8))
     (md5sum-stream stream)))

contrib/sb-md5/sb-md5.texinfo

@@ -1,27 +1,48 @@
-@node sb-md5
+@c Generated by the sb-manual contrib. Do not edit.
+
+@node sb md5
 @section sb-md5
-@cindex Hashing, cryptographic
 
 The @code{sb-md5} module implements the RFC1321 MD5 Message Digest
-Algorithm. [FIXME cite]
-
-@include fun-sb-md5-md5sum-file.texinfo
-
-@include fun-sb-md5-md5sum-sequence.texinfo
-
-@include fun-sb-md5-md5sum-stream.texinfo
-
-@include fun-sb-md5-md5sum-string.texinfo
-
-@subsection Credits
+Algorithm.
 
+@anchor{Function sb-md5 md5sum-file}
+@ffindex @sortas{md5sum-file sb-md5} md5sum-file [sb-md5]
+@deffn{Function} sb-md5:md5sum-file pathname
+Calculate the MD5 message-digest of the file specified by @code{pathname}.
+@end deffn
+@anchor{Function sb-md5 md5sum-sequence}
+@ffindex @sortas{md5sum-sequence sb-md5} md5sum-sequence [sb-md5]
+@deffn{Function} sb-md5:md5sum-sequence sequence &key start end
+Calculate the MD5 message-digest of data in @code{sequence}, which should
+be a 1d @code{simple-array} with element type (@code{unsigned-byte} 8). On CMU CL
+and SBCL non-simple and non-1d arrays with this element-type are also
+supported.
+@end deffn
+@anchor{Function sb-md5 md5sum-stream}
+@ffindex @sortas{md5sum-stream sb-md5} md5sum-stream [sb-md5]
+@deffn{Function} sb-md5:md5sum-stream stream
+Calculate an MD5 message-digest of the contents of @code{stream}.  Its
+element-type has to be (@code{unsigned-byte} 8). Use on character streams is
+DEPRECATED, as this will not work correctly on implementations with
+@code{char-code-limit} > 256 and ignores character coding issues.
+@end deffn
+@anchor{Function sb-md5 md5sum-string}
+@ffindex @sortas{md5sum-string sb-md5} md5sum-string [sb-md5]
+@deffn{Function} sb-md5:md5sum-string string &key external-format start end
+Calculate the MD5 message-digest of the binary representation of
+@code{string} (as octets) in the external format specified by
+@code{external-format}. The boundaries @code{start} and @code{end} refer to character
+positions in the string, not to octets in the resulting binary
+representation.  The permissible external format specifiers are
+determined by the underlying implementation.
+@end deffn
 The implementation for CMUCL was largely done by Pierre Mai, with help
-from members of the @code{cmucl-help} mailing list.  Since CMUCL and
-SBCL are similar in many respects, it was not too difficult to extend
-the low-level implementation optimizations for CMUCL to SBCL.
-Following this, SBCL's compiler was extended to implement efficient
-compilation of modular arithmetic (@pxref{Modular arithmetic}), which
+from members of the @code{cmucl-help} mailing list. Since CMUCL and SBCL
+are similar in many respects, it was not too difficult to extend the
+low-level implementation optimizations for CMUCL to SBCL. Following
+this, SBCL's compiler was extended to implement efficient
+compilation of modular arithmetic (@ref{modular arithmetic}), which
 enabled the implementation to be expressed in portable arithmetical
-terms, apart from the use of @code{rotate-byte} for bitwise rotation.
-@findex @sbrotatebyte{rotate-byte}
+terms, apart from the use of @ref{sb rotate byte} for bitwise rotation.
 

contrib/sb-posix/constants.lisp

@@ -438,8 +438,7 @@
  ;; utime(), utimes()
  #-win32
  (:type suseconds-t ; OAOOM warning: similar kludge in tools-for-build
-        #+(and os-provides-suseconds-t 64-bit-time) "__suseconds64_t"
-        #+(and os-provides-suseconds-t (not 64-bit-time)) "suseconds_t"
+        #+os-provides-suseconds-t "suseconds_t"
         #-os-provides-suseconds-t "long")
 
  #-win32
@@ -451,9 +450,7 @@
  (:structure alien-timeval
              ("struct timeval"
               (time-t sec "time_t" "tv_sec")
-              (suseconds-t usec #+64-bit-time "__suseconds64_t"
-                                #-64-bit-time "suseconds_t"
-                                "tv_usec")))
+              (signed usec nil "tv_usec")))
 
  (:integer veof "VEOF" nil t)
  (:integer veol "VEOL" nil t)

contrib/sb-posix/defpackage.lisp

@@ -20,7 +20,8 @@
            #:stat-gid #:stat-size #:stat-atime #:stat-mtime #:stat-ctime
            #:stat-rdev
            #:termios-iflag #:termios-oflag #:termios-cflag
-           #:termios-lflag #:termios-cc #:timeval-sec #:timeval-usec
+           #:termios-lflag #:termios-cc
+           #:timeval-sec #:timeval-usec
            #:flock-type #:flock-whence #:flock-start #:flock-len
            #:flock-pid
 

contrib/sb-posix/designator.lisp

@@ -4,10 +4,8 @@
 (defmacro define-designator (name (lisp-type alien-type) doc
                              &body conversions)
   (let ((designator-type `(or ,@(mapcar #'car conversions)))
-        (designator-name (intern (format nil "~A-~A"
-                                         (symbol-name name)
-                                  (symbol-name :designator))
-                                 #.*package*))
+        (designator-name
+         (sb-int:package-symbolicate #.*package* name "-DESIGNATOR"))
         (name (intern (symbol-name name) :sb-posix)))
     `(progn
        (deftype ,name ()

contrib/sb-posix/interface.lisp

@@ -2,18 +2,42 @@
 
 (defmacro define-protocol-class
     (name alien-type superclasses slots &rest options)
-  (let ((to-protocol (intern (format nil "ALIEN-TO-~A" name)))
-        (to-alien (intern (format nil "~A-TO-ALIEN" name))))
+  (let* ((to-protocol (intern (format nil "ALIEN-TO-~A" name)))
+         (to-alien (intern (format nil "~A-TO-ALIEN" name)))
+         (remaining-field-size sb-vm:n-word-bytes)
+         (remaining-fields (loop for field in (sb-alien::alien-record-type-fields
+                                               (sb-alien-internals:parse-alien-type alien-type nil))
+                                 for name = (sb-alien-internals:alien-record-field-name field)
+                                 for offset = (/ (sb-alien-internals:alien-record-field-offset field)
+                                                       8)
+                                 for size = (/ (sb-alien-internals:alien-type-bits (sb-alien-internals:alien-record-field-type field))
+                                                     8)
+                                 when (eql (search "PADDING-" (string name)) 0)
+                                 collect (cons offset size)
+                                 and do (cond ((not (and (zerop (rem offset 4))
+                                                         (zerop (rem size 4))))
+                                               (setf remaining-field-size 1))
+                                              ((not (and (zerop (rem offset sb-vm:n-word-bytes))
+                                                         (zerop (rem size sb-vm:n-word-bytes))))
+                                               (setf remaining-field-size
+                                                     (min remaining-field-size 4))))))
+         (remaining-fields-length (/ (reduce #'+ remaining-fields :key #'cdr)
+                                     remaining-field-size))
+         (remaining-fields-type `(simple-array (unsigned-byte ,(* remaining-field-size 8))
+                                               (,remaining-fields-length))))
     `(progn
       (export ',name :sb-posix)
       (defclass ,name ,superclasses
          ;; KLUDGE: Splice out some slot options (they're
          ;; for the conversion functions, not for DEFCLASS).
-        ,(loop for slotd in slots
-               collect
-               (let ((slotd (copy-list slotd)))
-                 (dolist (keyword '(:array-length :from-alien) slotd)
-                   (remf (cdr slotd) keyword))))
+        (,@(loop for slotd in slots
+                collect
+                (let ((slotd (copy-list slotd)))
+                  (dolist (keyword '(:array-length :from-alien) slotd)
+                    (remf (cdr slotd) keyword))))
+         ,@(when remaining-fields
+             `((remaining-fields :initform (make-sequence ',remaining-fields-type ,remaining-fields-length
+                                                          :initial-element 0)))))
         ,@options)
       ;; TODO (maybe): there's no reason to define to-alien routines
       ;; struct stat, passwd, or group: OS interfaces only ever write
@@ -50,6 +74,18 @@
                                      (,from-alien (sb-alien:slot alien ',(car slotd))))
                               `(setf (slot-value instance ',(car slotd))
                                      (sb-alien:slot alien ',(car slotd)))))
+        ,@(when remaining-fields
+            `((let ((vector (the ,remaining-fields-type
+                                 (slot-value instance 'remaining-fields))))
+                ,@(loop with array-offset = 0
+                        for (offset . size) in remaining-fields
+                        append (loop repeat (/ size remaining-field-size)
+                                     for i from (/ offset remaining-field-size)
+                                     collect `(setf (aref vector ,array-offset)
+                                                    (deref (sb-alien:cast alien (* (unsigned ,(* remaining-field-size 8))))
+                                                           ,i))
+                                     do
+                                     (incf array-offset))))))
         instance)
       (defun ,to-alien (instance &optional alien)
         (declare (type (or null (sb-alien:alien (* ,alien-type))) alien)
@@ -70,7 +106,19 @@
                 ;; counterpart of :FROM-ALIEN so far.
                 else
                   collect `(setf (sb-alien:slot alien ',(car slotd))
-                                 (slot-value instance ',(car slotd)))))
+                                 (slot-value instance ',(car slotd))))
+        ,@(when remaining-fields
+            `((let ((vector (the ,remaining-fields-type
+                                 (slot-value instance 'remaining-fields))))
+                ,@(loop with array-offset = 0
+                        for (offset . size) in remaining-fields
+                        append (loop repeat (/ size remaining-field-size)
+                                     for i from (/ offset remaining-field-size)
+                                     collect `(setf (deref (sb-alien:cast alien (* (unsigned ,(* remaining-field-size 8))))
+                                                           ,i)
+                                                    (aref vector ,array-offset))
+                                     do
+                                     (incf array-offset)))))))
       (find-class ',name))))
 
 (define-condition sb-posix:syscall-error (error)
@@ -274,7 +322,8 @@
                           (arg alien-pointer-to-anything-or-nil))
   (define-protocol-class flock alien-flock ()
    ((type :initarg :type :accessor flock-type
-          :documentation "Type of lock; F_RDLCK, F_WRLCK, F_UNLCK.")
+          :documentation "Type of lock; `\\\\F_RDLCK`, `\\\\F_WRLCK`,
+`\\\\F_UNLCK`.")
     (whence :initarg :whence :accessor flock-whence
             :documentation "Flag for starting offset.")
     (start :initarg :start :accessor flock-start
@@ -286,8 +335,9 @@
     ;; so we initialize it to 0.
     (pid :initform 0 :reader flock-pid
          :documentation
-         "Process ID of the process holding the lock; returned with F_GETLK."))
-   (:documentation "Class representing locks used in fcntl(2)."))
+         "Process ID of the process holding the lock;
+         returned with `\\\\F_GETLK`."))
+   (:documentation "Class representing locks used in `\\fcntl(2)`."))
   (define-entry-point "fcntl" (fd cmd &optional (arg nil argp))
     (if argp
         (etypecase arg
@@ -509,6 +559,8 @@ not supported."
 
  (define-call "munmap" int minusp
    (start sb-sys:system-area-pointer) (length unsigned))
+ (define-call "mprotect" int minusp
+   (addr system-area-pointer) (length size-t) (prot unsigned))
 
  (define-call "msync" int minusp
    (addr sb-sys:system-area-pointer) (length unsigned) (flags int)))
@@ -642,8 +694,10 @@ not supported."
                       ,name (function ,result-type ,@(when arg-type `(,arg-type))))
                    ,@(when arg `(,arg))))))
           (if (null-alien r)
-              (when (plusp (get-errno))
-                (syscall-error ',lisp-name))
+              (let ((errno (get-errno)))
+                (when (and (plusp errno)
+                           (/= errno ENOENT))
+                  (syscall-error ',lisp-name)))
               (,conv r)))))))
   (define-enumerator-call (name assertion)
       (let ((lisp-name (intern (string-upcase name) :sb-posix)))
@@ -811,15 +865,15 @@ not supported."
 
 #-win32
 (define-protocol-class termios alien-termios ()
-  ((iflag :initarg :iflag :accessor sb-posix:termios-iflag
+  ((iflag :initarg :iflag :accessor termios-iflag
           :documentation "Input modes.")
-   (oflag :initarg :oflag :accessor sb-posix:termios-oflag
+   (oflag :initarg :oflag :accessor termios-oflag
           :documentation "Output modes.")
-   (cflag :initarg :cflag :accessor sb-posix:termios-cflag
+   (cflag :initarg :cflag :accessor termios-cflag
           :documentation "Control modes.")
-   (lflag :initarg :lflag :accessor sb-posix:termios-lflag
+   (lflag :initarg :lflag :accessor termios-lflag
           :documentation "Local modes.")
-   (cc :initarg :cc :accessor sb-posix:termios-cc :array-length nccs
+   (cc :initarg :cc :accessor termios-cc :array-length nccs
        :documentation "Control characters."))
   (:documentation
    "Instances of this class represent I/O characteristics of the terminal."))
@@ -1020,7 +1074,7 @@ not supported."
   (defun syslog (priority format &rest args)
     "Send a message to the syslog facility, with severity level
 PRIORITY.  The message will be formatted as by CL:FORMAT (rather
-than C's printf) with format string FORMAT and arguments ARGS."
+than C's `printf`) with format string FORMAT and arguments ARGS."
     (flet ((syslog1 (priority message)
              (alien-funcall (extern-alien
                              "syslog" (function void int

contrib/sb-posix/macros.lisp

@@ -5,7 +5,7 @@
 
 Note that native namestring syntax is distinct from Lisp namestring syntax:
 
-  \(pathname \"/foo*/bar\")
+    (pathname \"/foo*/bar\")
 
 is a wild pathname with a pattern-matching directory component.
 SB-EXT:PARSE-NATIVE-NAMESTRING may be used to construct Lisp pathnames that
@@ -15,16 +15,16 @@ namestring syntax.
 
 Note also that POSIX filename syntax does not distinguish the names of files
 from the names of directories: in order to parse the name of a directory in
-POSIX filename syntax into a pathname MY-DEFAULTS for which
+POSIX filename syntax into a pathname `MY-DEFAULTS` for which
 
-  \(merge-pathnames (make-pathname :name \"FOO\" :case :common)
-                    my-defaults)
+    (merge-pathnames (make-pathname :name \"FOO\" :case :common)
+                      my-defaults)
 
 returns a pathname that denotes a file in the directory, supply a true
 :AS-DIRECTORY argument to SB-EXT:PARSE-NATIVE-NAMESTRING. Likewise, to supply
 the name of a directory to a POSIX function in non-directory syntax, supply a
 true :AS-FILE argument to SB-EXT:NATIVE-NAMESTRING."
-   "Designator for a FILENAME: a STRING designating itself, or a
+   "Designator for a [FILENAME][type]: a STRING designating itself, or a
 designator for a PATHNAME designating the corresponding native namestring."
    "Converts FILENAME-DESIGNATOR into a FILENAME.")
   (pathname
@@ -43,8 +43,8 @@ native file descriptor.
 
 Note that mixing I/O operations on a FILE-STREAM with operations directly on its
 descriptor may produce unexpected results if the stream is buffered."
-     "Designator for a FILE-DESCRIPTOR: either a fixnum designating itself, or
-a FILE-STREAM designating the underlying file-descriptor."
+     "Designator for a [FILE-DESCRIPTOR][type]: either a fixnum designating
+itself, or a FILE-STREAM designating the underlying file-descriptor."
      "Converts FILE-DESCRIPTOR-DESIGNATOR into a FILE-DESCRIPTOR.")
   (file-stream
    (sb-sys:fd-stream-fd file-descriptor))