From e20983b2adcdae60b5f7401372edcd1f09dac477 Mon Sep 17 00:00:00 2001 From: Gabor Melis Date: Sun, 7 Jun 2026 19:16:35 +0200 Subject: [PATCH] doc: add new lisp manual files This is in preparation for the "PAXlike docs" commit. --- contrib/sb-aclrepl/manual.lisp | 51 + contrib/sb-bsd-sockets/manual.lisp | 125 +++ contrib/sb-concurrency/manual.lisp | 72 ++ contrib/sb-cover/manual.lisp | 42 + contrib/sb-grovel/manual.lisp | 208 ++++ contrib/sb-introspect/manual.lisp | 45 + contrib/sb-manual/doc/beyond-ansi.lisp | 1049 ++++++++++++++++++ contrib/sb-manual/doc/compiler.lisp | 888 +++++++++++++++ contrib/sb-manual/doc/contrib-modules.lisp | 20 + contrib/sb-manual/doc/debugger.lisp | 865 +++++++++++++++ contrib/sb-manual/doc/deprecation.lisp | 434 ++++++++ contrib/sb-manual/doc/efficiency.lisp | 399 +++++++ contrib/sb-manual/doc/external-formats.lisp | 136 +++ contrib/sb-manual/doc/ffi.lisp | 781 +++++++++++++ contrib/sb-manual/doc/intro.lisp | 500 +++++++++ contrib/sb-manual/doc/networking.lisp | 1 + contrib/sb-manual/doc/package-locks.lisp | 276 +++++ contrib/sb-manual/doc/pathnames.lisp | 152 +++ contrib/sb-manual/doc/profiling.lisp | 158 +++ contrib/sb-manual/doc/sb-aclrepl.lisp | 1 + contrib/sb-manual/doc/sb-concurrency.lisp | 1 + contrib/sb-manual/doc/sb-cover.lisp | 1 + contrib/sb-manual/doc/sb-grovel.lisp | 1 + contrib/sb-manual/doc/sb-introspect.lisp | 1 + contrib/sb-manual/doc/sb-md5.lisp | 1 + contrib/sb-manual/doc/sb-posix.lisp | 1 + contrib/sb-manual/doc/sb-queue.lisp | 1 + contrib/sb-manual/doc/sb-rotate-byte.lisp | 1 + contrib/sb-manual/doc/sb-simd.lisp | 1 + contrib/sb-manual/doc/sb-simple-streams.lisp | 1 + contrib/sb-manual/doc/sbcl.lisp | 21 + contrib/sb-manual/doc/start-stop.lisp | 339 ++++++ contrib/sb-manual/doc/streams.lisp | 322 ++++++ contrib/sb-manual/doc/support-and-bugs.lisp | 123 ++ contrib/sb-manual/doc/threading.lisp | 333 ++++++ contrib/sb-manual/doc/timers.lisp | 41 + contrib/sb-md5/manual.lisp | 18 + contrib/sb-posix/manual.lisp | 174 +++ contrib/sb-queue/manual.lisp | 5 + contrib/sb-rotate-byte/manual.lisp | 13 + contrib/sb-simd/manual.lisp | 233 ++++ contrib/sb-simple-streams/manual.lisp | 22 + 42 files changed, 7857 insertions(+) create mode 100644 contrib/sb-aclrepl/manual.lisp create mode 100644 contrib/sb-bsd-sockets/manual.lisp create mode 100644 contrib/sb-concurrency/manual.lisp create mode 100644 contrib/sb-cover/manual.lisp create mode 100644 contrib/sb-grovel/manual.lisp create mode 100644 contrib/sb-introspect/manual.lisp create mode 100644 contrib/sb-manual/doc/beyond-ansi.lisp create mode 100644 contrib/sb-manual/doc/compiler.lisp create mode 100644 contrib/sb-manual/doc/contrib-modules.lisp create mode 100644 contrib/sb-manual/doc/debugger.lisp create mode 100644 contrib/sb-manual/doc/deprecation.lisp create mode 100644 contrib/sb-manual/doc/efficiency.lisp create mode 100644 contrib/sb-manual/doc/external-formats.lisp create mode 100644 contrib/sb-manual/doc/ffi.lisp create mode 100644 contrib/sb-manual/doc/intro.lisp create mode 120000 contrib/sb-manual/doc/networking.lisp create mode 100644 contrib/sb-manual/doc/package-locks.lisp create mode 100644 contrib/sb-manual/doc/pathnames.lisp create mode 100644 contrib/sb-manual/doc/profiling.lisp create mode 120000 contrib/sb-manual/doc/sb-aclrepl.lisp create mode 120000 contrib/sb-manual/doc/sb-concurrency.lisp create mode 120000 contrib/sb-manual/doc/sb-cover.lisp create mode 120000 contrib/sb-manual/doc/sb-grovel.lisp create mode 120000 contrib/sb-manual/doc/sb-introspect.lisp create mode 120000 contrib/sb-manual/doc/sb-md5.lisp create mode 120000 contrib/sb-manual/doc/sb-posix.lisp create mode 120000 contrib/sb-manual/doc/sb-queue.lisp create mode 120000 contrib/sb-manual/doc/sb-rotate-byte.lisp create mode 120000 contrib/sb-manual/doc/sb-simd.lisp create mode 120000 contrib/sb-manual/doc/sb-simple-streams.lisp create mode 100644 contrib/sb-manual/doc/sbcl.lisp create mode 100644 contrib/sb-manual/doc/start-stop.lisp create mode 100644 contrib/sb-manual/doc/streams.lisp create mode 100644 contrib/sb-manual/doc/support-and-bugs.lisp create mode 100644 contrib/sb-manual/doc/threading.lisp create mode 100644 contrib/sb-manual/doc/timers.lisp create mode 100644 contrib/sb-md5/manual.lisp create mode 100644 contrib/sb-posix/manual.lisp create mode 100644 contrib/sb-queue/manual.lisp create mode 100644 contrib/sb-rotate-byte/manual.lisp create mode 100644 contrib/sb-simd/manual.lisp create mode 100644 contrib/sb-simple-streams/manual.lisp diff --git a/contrib/sb-aclrepl/manual.lisp b/contrib/sb-aclrepl/manual.lisp new file mode 100644 index 000000000..e7ac49126 --- /dev/null +++ b/contrib/sb-aclrepl/manual.lisp @@ -0,0 +1,51 @@ +(in-package :sb-manual) + +(defsection @sb-aclrepl (:title "sb-aclrepl") + "The `SB-ACLREPL` module offers an Allegro CL-style + Read-Eval-Print Loop for SBCL, with integrated inspector. Adding a + debugger interface is planned. + + Allegro CL is a registered trademark of Franz Inc." + (@sb-aclrepl-usage section) + (@sb-aclrepl-customization section) + (@sb-aclrepl-example-initialization section)) + +(defsection @sb-aclrepl-usage (:title "Usage") + "To start `SB-ACLREPL` as your read-eval-print loop, put the form + + (require 'sb-aclrepl) + + in your `~/.sbclrc`, one of your @INITIALIZATION-FILES.") + +(defsection @sb-aclrepl-customization (:title "Customization") + "The following customization variables are available:" + (sb-aclrepl:*command-char* variable) + (sb-aclrepl:*prompt* variable) + (sb-aclrepl:*exit-on-eof* variable) + (sb-aclrepl:*use-short-package-name* variable) + (sb-aclrepl:*max-history* variable)) + +(defsection @sb-aclrepl-example-initialization (:title "Example Initialization") + "Here's a longer example of a `~/.sbclrc` file that shows off + some of the features of sb-aclrepl: + + (ignore-errors (require 'sb-aclrepl)) + + (when (find-package 'sb-aclrepl) + (push :aclrepl cl:*features*)) + #+aclrepl + (progn + (setq sb-aclrepl:*max-history* 100) + (setf (sb-aclrepl:alias \"asdc\") + #'(lambda (sys) (asdf:operate 'asdf:compile-op sys))) + (sb-aclrepl:alias \"l\" (sys) (asdf:operate 'asdf:load-op sys)) + (sb-aclrepl:alias \"t\" (sys) (asdf:operate 'asdf:test-op sys)) + ;; The 1 below means that two characaters (\"up\") are required + (sb-aclrepl:alias (\"up\" 1 \"Use package\") (package) (use-package package)) + ;; The 0 below means only the first letter (\"r\") is required, + ;; such as \":r base64\" + (sb-aclrepl:alias (\"require\" 0 \"Require module\") (sys) (require sys)) + (setq cl:*features* (delete :aclrepl cl:*features*))) + + Questions, comments, or bug reports should be sent to Kevin Rosenberg + (kevin@rosenberg.net).") diff --git a/contrib/sb-bsd-sockets/manual.lisp b/contrib/sb-bsd-sockets/manual.lisp new file mode 100644 index 000000000..f1f27bd36 --- /dev/null +++ b/contrib/sb-bsd-sockets/manual.lisp @@ -0,0 +1,125 @@ +(in-package :sb-manual) + +(defsection @networking (:title "Networking") + "The `SB-BSD-SOCKETS` module provides a thinly disguised BSD + socket API for SBCL. Ideas have been stolen from the BSD socket API + for C and Graham Barr's `IO::Socket` classes for Perl. + + Sockets are represented as CLOS objects, and the API naming + conventions attempt to balance between the BSD names and good lisp + style." + (@sockets-overview section) + (@general-sockets section) + (@socket-options section) + (@inet-domain-sockets section) + (@local-domain-sockets section) + (@name-service section)) + +(defsection @sockets-overview (:title "Sockets Overview") + "Most of the functions are modelled on the BSD socket API. BSD sockets + are widely supported, portably (by Unix standards, at least) + available on a variety of systems, and documented. There are some + differences in approach where we have taken advantage of some of the + more useful features of Common Lisp -- briefly: + + - Where the C API would typically return -1 and set `errno`, + `SB-BSD-SOCKETS` signals an error. All the errors are subclasses + of SB-BSD-SOCKETS:SOCKET-ERROR and generally correspond one for + one with possible `errno` values. + + - We use multiple return values in many places where the C API would + use pass-by-reference values. + + - We can often avoid supplying an explicit length argument to + functions because we already know how long the argument is. + + - IP addresses and ports are represented in slightly friendlier + fashion than \"network-endian integers\".") + +(defsection @general-sockets (:title "General Sockets") + (sb-bsd-sockets:socket class) + (sb-bsd-sockets:socket-bind function) + (sb-bsd-sockets:socket-accept function) + (sb-bsd-sockets:socket-connect function) + (sb-bsd-sockets:socket-peername function) + (sb-bsd-sockets:socket-name function) + (sb-bsd-sockets:socket-receive function) + (sb-bsd-sockets:socket-send function) + (sb-bsd-sockets:socket-listen function) + (sb-bsd-sockets:socket-open-p function) + (sb-bsd-sockets:socket-close function) + (sb-bsd-sockets:socket-shutdown function) + (sb-bsd-sockets:socket-make-stream function) + (sb-bsd-sockets:socket-error function) + (sb-bsd-sockets:non-blocking-mode function)) + +(defsection @socket-options (:title "Socket Options") + "A subset of socket options are supported, using a fairly general + framework which should make it simple to add more as required -- see + `\\\\SYS:CONTRIB;SB-BSD-SOCKETS:SOCKOPT.LISP` for details. The name + mapping from C is fairly straightforward: `\\\\SO_RCVLOWAT` becomes + SB-BSD-SOCKETS:SOCKOPT-RECEIVE-LOW-WATER and `(SETF + SB-BSD-SOCKETS:SOCKOPT-RECEIVE-LOW-WATER)`." + (sb-bsd-sockets:sockopt-reuse-address function) + (sb-bsd-sockets:sockopt-keep-alive function) + (sb-bsd-sockets:sockopt-oob-inline function) + (sb-bsd-sockets:sockopt-bsd-compatible function) + (sb-bsd-sockets:sockopt-pass-credentials function) + (sb-bsd-sockets:sockopt-debug function) + (sb-bsd-sockets:sockopt-dont-route function) + (sb-bsd-sockets:sockopt-broadcast function) + (sb-bsd-sockets:sockopt-tcp-nodelay function)) + +(defsection @inet-domain-sockets (:title "INET Domain Sockets") + "The TCP and UDP sockets that you know and love. Some representation + issues: + + - IPv4 Internet addresses are represented by vectors of + `(UNSIGNED-BYTE 8)` (e.g. `#(127 0 0 1)`). Ports are just + integers. No conversion between network- and host-order data is + needed from the user of this package. + + - IPv6 Internet addresses are represented by length 16 vectors of + `(UNSIGNED-BYTE 8)` (e.g. `#(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1)`. + Ports are just integers. As for IPv4 addresses, no conversion + between network- and host-order data is needed from the user of + this package. + + - Socket addresses are represented by the two values for address and + port, so for example, `(SB-BSD-SOCKETS:SOCKET-CONNECT SOCKET #(192 + 168 1 1) 80)` for IPv4 and `(SB-BSD-SOCKETS:SOCKET-CONNECT SOCKET + #(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1) 80)` for IPv6." + (sb-bsd-sockets:inet-socket class) + (sb-bsd-sockets:inet6-socket class) + (sb-bsd-sockets:make-inet-address function) + (sb-bsd-sockets:make-inet6-address function) + (sb-bsd-sockets:get-protocol-by-name function)) + +(defsection @local-domain-sockets (:title "Local Domain Sockets") + "Local domain (`\\\\AF_LOCAL`) sockets are also known as Unix-domain + sockets but were renamed by POSIX presumably on the basis that they + may be available on other systems too. + + A local socket address is a string, which is used to create a node + in the local filesystem. This means of course that they cannot be + used across a network." + (sb-bsd-sockets:local-socket class) + "A local abstract socket address is also a string the scope of which is + the local machine. However, in contrast to a local socket address, there + is no corresponding filesystem node." + (sb-bsd-sockets:local-abstract-socket class)) + +(defsection @name-service (:title "Name Service") + "Presently name service is implemented by calling out to the + `getaddrinfo(3)` and `gethostinfo(3)`, or to `gethostbyname(3)` and + `gethostbyaddr(3)` on platforms where the preferred functions are + not available. The exact details of the name resolving process (for + example the choice of whether DNS or a hosts file is used for + lookup) are platform dependent." + ;; Direct links to the asynchronous `resolver(3)` routines would be + ;; nice to have eventually, so that we can do DNS lookups in + ;; parallel with other things. + (sb-bsd-sockets:host-ent class) + (sb-bsd-sockets:get-host-by-name function) + (sb-bsd-sockets:get-host-by-address function) + (sb-bsd-sockets:host-ent-address function)) diff --git a/contrib/sb-concurrency/manual.lisp b/contrib/sb-concurrency/manual.lisp new file mode 100644 index 000000000..5e0024497 --- /dev/null +++ b/contrib/sb-concurrency/manual.lisp @@ -0,0 +1,72 @@ +(in-package :sb-manual) + +(defsection @sb-concurrency (:title "sb-concurrency") + "Additional data structures, synchronization primitives and tools for + concurrent programming. Similiar to Java's `java.util.concurrent` + package." + (@sb-concurrency-queue section) + (@sb-concurrency-mailbox section) + (@sb-concurrency-gates section) + (@sb-concurrency-frlocks section)) + +(defsection @sb-concurrency-queue (:title "Queue") + "SB-CONCURRENCY:QUEUE is a lock-free, thread-safe FIFO queue + datatype. + + The implementation is based on _An Optimistic Approach to Lock-Free + FIFO Queues_ by Edya Ladan-Mozes and Nir Shavit. + + Before SBCL 1.0.38, this implementation resided in its own contrib + (see @SB-QUEUE), which is still provided for + backwards-compatibility, but which has since been deprecated." + (sb-concurrency:queue structure) + (sb-concurrency:dequeue function) + (sb-concurrency:enqueue function) + (sb-concurrency:list-queue-contents function) + (sb-concurrency:make-queue function) + (sb-concurrency:queue-count function) + (sb-concurrency:queue-empty-p function) + (sb-concurrency:queue-name function) + (sb-concurrency:queuep function)) + +(defsection @sb-concurrency-mailbox (:title "Mailbox (lock-free)") + "SB-CONCURRENCY:MAILBOX is a lock-free message queue where one or + multiple ends can send messages to one or multiple receivers. The + difference to @SB-CONCURRENCY-QUEUE is that the receiving end may + block until a message arrives. + + Built on top of the @SB-CONCURRENCY-QUEUE implementation." + (sb-concurrency:mailbox structure) + (sb-concurrency:list-mailbox-messages function) + (sb-concurrency:mailbox-count function) + (sb-concurrency:mailbox-empty-p function) + (sb-concurrency:mailbox-name function) + (sb-concurrency:mailboxp function) + (sb-concurrency:make-mailbox function) + (sb-concurrency:receive-message function) + (sb-concurrency:receive-message-no-hang function) + (sb-concurrency:receive-pending-messages function) + (sb-concurrency:send-message function)) + +(defsection @sb-concurrency-gates (:title "Gates") + "SB-CONCURRENCY:GATE is a synchronization object suitable for when + multiple threads must wait for a single event before proceeding." + (sb-concurrency:gate structure) + (sb-concurrency:close-gate function) + (sb-concurrency:gate-name function) + (sb-concurrency:gate-open-p function) + (sb-concurrency:gatep function) + (sb-concurrency:make-gate function) + (sb-concurrency:open-gate function) + (sb-concurrency:wait-on-gate function)) + +(defsection @sb-concurrency-frlocks (:title "Frlocks, aka Fast Read Locks") + (sb-concurrency:frlock structure) + (sb-concurrency:frlock-read macro) + (sb-concurrency:frlock-write macro) + (sb-concurrency:make-frlock function) + (sb-concurrency:frlock-name function) + (sb-concurrency:frlock-read-begin function) + (sb-concurrency:frlock-read-end function) + (sb-concurrency:grab-frlock-write-lock function) + (sb-concurrency:release-frlock-write-lock function)) diff --git a/contrib/sb-cover/manual.lisp b/contrib/sb-cover/manual.lisp new file mode 100644 index 000000000..8ba711411 --- /dev/null +++ b/contrib/sb-cover/manual.lisp @@ -0,0 +1,42 @@ +(in-package :sb-manual) + +;;; FIXME: Write some documentation about how to interpret the results. +(defsection @sb-cover (:title "sb-cover") + "The `SB-COVER` module provides a code coverage tool for SBCL. The + tool has support for expression coverage, and for some branch + coverage. Coverage reports are only generated for code compiled + using COMPILE-FILE with the value of the + SB-COVER:STORE-COVERAGE-DATA optimization quality set to 3. + + As of SBCL 1.0.6, `SB-COVER` is still experimental, and the + interfaces documented here might change in later versions. + + How to use it: + + ;;; Load SB-COVER + (require :sb-cover) + + ;;; Turn on generation of code coverage instrumentation in the compiler + (declaim (optimize sb-cover:store-coverage-data)) + + ;;; Load some code, ensuring that it's recompiled with the new optimization + ;;; policy. + (asdf:oos 'asdf:load-op :cl-ppcre-test :force t) + + ;;; Run the test suite. + (cl-ppcre-test:test) + + ;;; Produce a coverage report + (sb-cover:report \"/tmp/report/\") + + ;;; Turn off instrumentation + (declaim (optimize (sb-cover:store-coverage-data 0)))" + (sb-cover:report function) + (sb-cover:reset-coverage function) + (sb-cover:clear-coverage function) + (sb-cover:save-coverage function) + (sb-cover:save-coverage-in-file function) + (sb-cover:restore-coverage function) + (sb-cover:restore-coverage-from-file function) + (sb-cover:merge-coverage function) + (sb-cover:merge-coverage-from-file function)) diff --git a/contrib/sb-grovel/manual.lisp b/contrib/sb-grovel/manual.lisp new file mode 100644 index 000000000..31d9a89d9 --- /dev/null +++ b/contrib/sb-grovel/manual.lisp @@ -0,0 +1,208 @@ +(in-package :sb-manual) + +(defsection @sb-grovel (:title "sb-grovel") + "The `SB-GROVEL` module helps in generation of foreign function + interfaces. It aids in extracting constants' values from the C + compiler and in generating sb-alien structure and union types, + @DEFINING-FOREIGN-TYPES. + + The ASDF () component type + GROVEL-CONSTANTS-FILE has its ASDF:PERFORM operation defined to + write out a C source file, compile it, and run it. The output from + this program is Lisp, which is then itself compiled and loaded. + + `SB-GROVEL` is used in a few contributed modules, and it is + currently compatible only to SBCL. However, if you want to use it, + here are a few directions." + (@using-sb-grovel section) + (@sb-grovel-constants-file section) + (@sb-grovel-structures section) + (@sb-grovel-traps section)) + +(defsection @using-sb-grovel (:title "Using sb-grovel in your own ASDF System") + "- Create a Lisp package for the foreign constants/functions to go + into. + + - Make your system depend on the `SB-GROVEL` system. + + - Create a grovel-constants data file -- for an example, see + `example-constants.lisp` in the `contrib/sb-grovel/` directory in + the SBCL source distribution. + + - Add it as a component in your system. For example: + + (eval-when (:compile-toplevel :load-toplevel :execute) + (require :sb-grovel)) + + (defpackage :example-package.system + (:use :cl :asdf :sb-grovel :sb-alien)) + + (in-package :example-package.system) + + (defsystem example-system + :depends-on (sb-grovel) + :components + ((:module \"sbcl\" + :components + ((:file \"defpackage\") + (grovel-constants-file \"example-constants\" + :package :example-package))))) + + Make sure to specify the package you chose in step 1. + + - Build stuff.") + +(defsection @sb-grovel-constants-file + (:title "Contents of a grovel-constants-file") + "The grovel-constants-file, typically named `constants.lisp`, + comprises lisp expressions describing the foreign things that you + want to grovel for. A `constants.lisp` file contains two sections: + + - a list of headers to include in the C program, for example: + + (\"sys/types.h\" \"sys/socket.h\" \"sys/stat.h\" \"unistd.h\" \"sys/un.h\" + \"netinet/in.h\" \"netinet/in_systm.h\" \"netinet/ip.h\" \"net/if.h\" + \"netdb.h\" \"errno.h\" \"netinet/tcp.h\" \"fcntl.h\" \"signal.h\") + + - A list of sb-grovel clauses describing the things you want to + grovel from the C compiler, for example: + + ((:integer af-local + #+(or sunos solaris) \"AF_UNIX\" + #-(or sunos solaris) \"AF_LOCAL\" + \"Local to host (pipes and file-domain).\") + (:structure stat (\"struct stat\" + (integer dev \"dev_t\" \"st_dev\") + (integer atime \"time_t\" \"st_atime\"))) + (:function getpid (\"getpid\" int ))) + + There are two types of things that sb-grovel can sensibly extract + from the C compiler: constant integers and structure layouts. It is + also possible to define foreign functions in the constants.lisp + file, but these definitions don't use any information from the C + program; they expand directly to SB-ALIEN:DEFINE-ALIEN-ROUTINE + forms. + + Here's how to use the grovel clauses: + + - :INTEGER: constant expressions in C. Used in this form: + + (:integer lisp-variable-name \"C expression\" &optional doc export) + + `\"C expression\"` will be typically be the name of a constant, + but other forms are possible. + + - :ENUM: + + (:enum lisp-type-name ((lisp-enumerated-name c-enumerated-name) ...))) + + An SB-ALIEN:ENUM type with name `LISP-TYPE-NAME` will be + defined. The symbols are the `LISP-ENUMERATED-NAME`s, and the + values are grovelled from the `C-ENUMERATED-NAME`s. + + - :STRUCTURE: alien structure definitions look like this: + + (:structure lisp-struct-name (\"struct c_structure\" + (type-designator lisp-element-name + \"c_element_type\" \"c_element_name\" + :distrust-length nil) + ; ... + )) + + `TYPE-DESIGNATOR` is a reference to a type whose size (and type + constraints) will be groveled for. sb-grovel accepts a form of + type designator that doesn't quite conform to either lisp nor + sb-alien's type specifiers. Here's a list of type designators + that sb-grovel currently accepts: + + - `\\INTEGER`: a C integral type; sb-grovel will infer the exact + type from size information extracted from the C program. All + common C integer types can be grovelled for with this type + designator, but it is not possible to grovel for bit fields + yet. + + - `(UNSIGNED N)`: an unsigned integer variable that is `N` bytes + long. No size information from the C program will be used. + + - `(SIGNED N)`: an signed integer variable that is `N` bytes + long. No size information from the C program will be used. + + - `\\C-STRING`: an array of `\\char` in the structure. sb-grovel + will use the array's length from the C program, unless you + pass it the :DISTRUST-LENGTH keyword argument with non-`NIL` + value (this might be required for structures such as solaris's + `struct dirent`). + + - SB-GROVEL::C-STRING-POINTER: a pointer to a C string, + corresponding to the SB-ALIEN:C-STRING type (see + @FOREIGN-TYPE-SPECIFIERS). + + - `(ARRAY ALIEN-TYPE)`: an array of the previously-declared + `ALIEN-TYPE`. The array's size will be determined from the + output of the C program and the alien type's size. + + - `(ARRAY ALIEN-TYPE N):` an array of the previously-declared + `ALIEN-TYPE`. The array's size will be assumed as being `N`. + + Note that `\\C-STRING` and SB-GROVEL::C-STRING-POINTER do not have + the same meaning. If you declare that an element is of type + C-STRING, it will be treated as if the string is a part of the + structure, whereas if you declare that the element is of type + SB-GROVEL::C-STRING-POINTER, a _pointer to a string_ will be the + structure member. + + - :FUNCTION: alien function definitions are similar to + DEFINE-ALIEN-ROUTINE definitions, because they expand to such + forms when the lisp program is loaded. See + @FOREIGN-FUNCTION-CALLS. + + (:function lisp-function-name + (\"alien_function_name\" alien-return-type + (argument alien-type) + (argument2 alien-type)))") + +(defsection @sb-grovel-structures + (:title "Programming with sb-grovel's structure types") + "Let us assume that you have a grovelled structure definition: + + (:structure mystruct (\"struct my_structure\" + (integer myint \"int\" \"st_int\") + (c-string mystring \"char[]\" \"st_str\"))) + + What can you do with it? Here's a short interface document: + + - Creating and destroying objects: + + - Function `(ALLOCATE-MYSTRUCT)` allocates an object of type + `mystruct` and returns a system area pointer to it. + + - Macro `(WITH-MYSTRUCT VAR ((MEMBER INIT) [...]) &BODY BODY)` + allocates an object of type `MYSTRUCT` that is valid in + `BODY`. If `BODY` terminates or performs an non-local exit, + the object pointed to by `VAR` will be deallocated. + + - Accessing structure members: + + - `(MYSTRUCT-MYINT VAR)` and `(MYSTRUCT-MYSTRING VAR)` return + the value of the respective fields in `MYSTRUCT`. + + - `(SETF (MYSTRUCT-MYINT VAR) NEW-VAL)` and + `(SETF (MYSTRUCT-MYSTRING VAR) NEW-VAL)` sets the value of the + respective structure member to the value of `NEW-VAL`. Notice + that in `(SETF (MYSTRUCT-MYSTRING VAR) NEW-VAL)`'s case, + `NEW-VAL` is a lisp string.") + +(defsection @sb-grovel-traps (:title "Traps and Pitfalls") + "Basically, you can treat functions and data structure definitions that + sb-grovel spits out as if they were alien routines and types. This has + a few implications that might not be immediately obvious (especially + if you have programmed in a previous version of sb-grovel that didn't + use alien types): + + - You must take care of grovel-allocated structures yourself. They + are alien types, so the garbage collector will not collect them + when you drop the last reference. + + - If you use the `WITH-MYSTRUCT` macro, be sure that no references + to the variable thus allocated leaks out. It will be deallocated + when the block exits.") diff --git a/contrib/sb-introspect/manual.lisp b/contrib/sb-introspect/manual.lisp new file mode 100644 index 000000000..b525be64f --- /dev/null +++ b/contrib/sb-introspect/manual.lisp @@ -0,0 +1,45 @@ +(in-package :sb-manual) + +(defsection @sb-introspect (:title "sb-introspect") + "The `SB-INTROSPECT` module is about finding definitions, as well + as querying their properties and relationships in the running image." + (@finding-definitions section) + (@sb-introspect-variables section) + (@sb-introspect-functions section) + (@sb-introspect-types section) + (@sb-introspect-allocation section)) + +(defsection @finding-definitions (:title "Finding Definitions") + (sb-introspect:definition-source structure) + (sb-introspect:definition-source-pathname function) + (sb-introspect:definition-source-form-path function) + (sb-introspect:definition-source-form-number function) + (sb-introspect:definition-source-character-offset function) + (sb-introspect:definition-source-file-write-date function) + (sb-introspect:definition-source-plist function) + (sb-introspect:find-definition-source function) + (sb-introspect:find-definition-sources-by-name function)) + +(defsection @sb-introspect-variables (:title "Special Variables") + (sb-introspect:who-binds function) + (sb-introspect:who-references function) + (sb-introspect:who-sets function)) + +(defsection @sb-introspect-functions (:title "Functions") + (sb-introspect:function-lambda-list function) + (sb-introspect:function-type function) + (sb-introspect:method-combination-lambda-list function) + (sb-introspect:valid-function-name-p function) + (sb-introspect:find-function-callers function) + (sb-introspect:find-function-callees function) + (sb-introspect:who-calls function) + (sb-introspect:who-macroexpands function)) + +(defsection @sb-introspect-types (:title "Types and Classes") + (sb-introspect:deftype-lambda-list function) + (sb-introspect:who-specializes-directly function) + (sb-introspect:who-specializes-generally function)) + +(defsection @sb-introspect-allocation (:title "Allocation") + (sb-introspect:allocation-information function) + (sb-introspect:map-root function)) diff --git a/contrib/sb-manual/doc/beyond-ansi.lisp b/contrib/sb-manual/doc/beyond-ansi.lisp new file mode 100644 index 000000000..f4e39cd58 --- /dev/null +++ b/contrib/sb-manual/doc/beyond-ansi.lisp @@ -0,0 +1,1049 @@ +(in-package :sb-manual) + +(defsection @beyond-the-ansi-standard (:title "Beyond the ANSI Standard") + "SBCL is derived from CMUCL, which implements many extensions to the + ANSI standard. SBCL doesn't support as many extensions as CMUCL, but + it still has quite a few. See @CONTRIBUTED-MODULES." + (@reader-extensions section) + (@package-local-nicknames section) + (@package-variance section) + (@garbage-collection section) + (@generic-function-dispatch section) + (@extended-slot-access section) + (@metaobject-protocol section) + (@extensible-sequences section) + (@support-for-unix section) + (@unicode-support section) + (@customization-hooks-for-users section) + (@tools-to-help-developers section) + (@resolution-of-name-conflicts section) + (@hash-table-extensions section) + (@random-number-generation section) + (@timeouts-and-deadlines section) + (@miscellaneous-extensions section) + (@stale-extensions section) + (@efficiency-hacks section)) + +(defsection @reader-extensions (:title "Reader Extensions") + (@extended-package-prefix-syntax section) + (@symbol-name-normalization section) + (@decimal-syntax-for-rationals section)) + +(defsection @extended-package-prefix-syntax + (:title "Extended Package Prefix Syntax") + "SBCL supports extended package prefix syntax, which allows specifying + an alternate package instead of *PACKAGE* for the reader to use as + the default package for interning symbols: + + :: + + Example: + + 'foo::(bar quux zot) == '(foo::bar foo::quux foo::zot) + + *PACKAGE* is not rebound during the course of reading a form with + extended package prefix syntax; if `FOO::BAR` would cause a + read-time package lock violation, so does `FOO::(BAR)`.") + +(defsection @symbol-name-normalization (:title "Symbol Name Normalization") + "SBCL also extends the reader to normalize all symbols to _Normalization + Form KC_ in builds with Unicode enabled. Whether symbols are + normalized is controlled by" + (sb-ext:readtable-normalization function) + "Symbols created by INTERN and similar functions are not affected by + this setting. If SB-EXT:READTABLE-NORMALIZATION is T, symbols that + are not normalized are escaped during printing.") + +(defsection @decimal-syntax-for-rationals + (:title "Decimal Syntax for Rationals") + "SBCL supports a decimal syntax for rationals, modelled after the + standard syntax for floating-point numbers. If a number with + floating-point syntax has an exponent marker of `r` or `R` + (rather than one of the standard exponent markers), it is read as + the rational with the exact value of the decimal number expressed as + a float. + + In addition, setting or binding the value of + *READ-DEFAULT-FLOAT-FORMAT* to RATIONAL around a call to READ or + READ-FROM-STRING has the effect that floating-point numbers without + exponent markers are read as rational numbers, as if there had been + an explicit `r` or `R` marker. + + Floating point numbers of all types are printed with an exponent + marker while the value of *READ-DEFAULT-FLOAT-FORMAT* is RATIONAL; + however, rational numbers are printed in their standard syntax, + irrespective of the value of *READ-DEFAULT-FLOAT-FORMAT*.") + +(defsection @package-local-nicknames (:title "Package-Local Nicknames") + "SBCL allows giving packages local nicknames: they allow short and + easy-to-use names to be used without fear of name conflict associated + with normal nicknames. + + A local nickname is valid only when inside the package for which it + has been specified. Different packages can use same local nickname + for different global names, or different local nickname for same + global name. + + The symbol :PACKAGE-LOCAL-NICKNAMES in *FEATURES* denotes the + support for this feature. + + DEFPACKAGE options are extended to include + + :local-nicknames ( )* + + with the semantics of adding the package package-local nicknames + ``s for the corresponding ``s. + + Example: + + (defpackage :bar (:intern \"X\")) + (defpackage :foo (:intern \"X\")) + (defpackage :quux (:use :cl) (:local-nicknames (:bar :foo) (:foo :bar))) + (find-symbol \"X\" :foo) ; => FOO::X + (find-symbol \"X\" :bar) ; => BAR::X + (let ((*package* (find-package :quux))) + (find-symbol \"X\" :foo)) ; => BAR::X + (let ((*package* (find-package :quux))) + (find-symbol \"X\" :bar)) ; => FOO::X" + (sb-ext:package-local-nicknames function) + (sb-ext:package-locally-nicknamed-by-list function) + (sb-ext:add-package-local-nickname function) + (sb-ext:remove-package-local-nickname function)) + +(defsection @package-variance (:title "Package Variance") + "DEFPACKAGE CLHS specifies that _if the new definition is at + variance with the current state of that package, the consequences + are undefined_. SBCL by default signals a full warning and retains + as much of the package state as possible. This can be adjusted with + the following variable." + (sb-ext:*on-package-variance* variable)) + +(defsection @garbage-collection (:title "Garbage Collection") + "SBCL provides additional garbage collection functionality not + specified by ANSI." + (sb-ext:gc function) + (sb-ext:*after-gc-hooks* variable) + (@finalization section) + (@weak-pointers section) + (@introspection-and-tuning section) + (@tracing-live-objects-back-to-roots section)) + +(defsection @finalization (:title "Finalization") + "Finalization allows code to be executed after an object has been + garbage collected. This is useful for example for releasing foreign + memory associated with a Lisp object." + (sb-ext:finalize function) + (sb-ext:cancel-finalization function)) + +(defsection @weak-pointers (:title "Weak Pointers") + "Weak pointers allow references to objects to be maintained without + keeping them from being garbage collected: useful for building caches + among other things. + + Hash tables can also have weak keys and values. See + @HASH-TABLE-EXTENSIONS." + (sb-ext:make-weak-pointer function) + (sb-ext:weak-pointer-value function)) + +(defsection @introspection-and-tuning (:title "Introspection and Tuning") + (sb-ext:*gc-run-time* (variable 0)) + (sb-ext:*gc-real-time* (variable 0)) + (sb-ext:bytes-consed-between-gcs function) + (sb-ext:dynamic-space-size function) + (sb-ext:get-bytes-consed function) + (sb-ext:gc-logfile function) + (sb-ext:generation-average-age function) + (sb-ext:generation-bytes-allocated function) + (sb-ext:generation-bytes-consed-between-gcs function) + (sb-ext:generation-minimum-age-before-gc function) + (sb-ext:generation-number-of-gcs-before-promotion function) + (sb-ext:generation-number-of-gcs function)) + +(defsection @tracing-live-objects-back-to-roots + (:title "Tracing Live Objects Back to Roots") + "This feature is intended to help expert users diagnose rare low-level + issues and should not be needed during normal usage. On top of that, + the interface and implementation are experimental and may change at + any time without further notice. + + It is sometimes important to understand why a given object is + retained in the Lisp image instead of being garbage collected. To + help with this problem, SBCL provides a mechanism that searches + through the different memory spaces, builds a path of references + from a root to the object in question and finally reports this + paths:" + (sb-ext:search-roots function) + "An example of using this could look like this: + + * (defvar *my-string* (list 1 2 \"my string\")) + *MY-STRING* + + * (sb-ext:search-roots (sb-ext:make-weak-pointer (third *my-string*))) + -> ((SIMPLE-VECTOR 3)) #x10004E9EAF[2] -> (SYMBOL) #x5044100F[1] -> (CONS) #x100181FAE7[1] -> (CONS) #x100181FAF7[1] -> (CONS) #x100181FB07[0] -> #x100181F9AF + + The single line of output on *STANDARD-OUTPUT* shows the path from a + root to `\"my string\"`: the path starts with SBCL's internal + package system data structures followed by the symbol + (`CL-USER:*MY-STRING*`) followed the three cons cells of the list. + + The `:PRINT :VERBOSE` argument produces similar behavior but + describes the path elements in more detail: + + * (sb-ext:search-roots (sb-ext:make-weak-pointer (third *my-string*)) + :print :verbose) + Path to \"my string\": + 6 10004E9EAF [ 2] a (simple-vector 3) + 0 5044100F [ 1] COMMON-LISP-USER::*MY-STRING* + 0 100181FAE7 [ 1] a cons + 0 100181FAF7 [ 1] a cons + 0 100181FB07 [ 0] a cons + + The `:PRINT NIL` argument is a bit different: + + * (sb-ext:search-roots (sb-ext:make-weak-pointer (third *my-string*)) + :print nil) + ((\"my string\" :STATIC (#(*MY-STRING* 0 0) . 2) (*MY-STRING* . 1) + ((1 2 \"my string\") . 1) ((2 \"my string\") . 1) ((\"my string\") . 0))) + + + There is no output on *STANDARD-OUTPUT*, and the return value is a + single path for the target object `\"my string\"`. As before, the + path shows the symbol and the three cons cells.") + +(defsection @generic-function-dispatch (:title "Generic Function Dispatch") + "If a generic function with standard or short method combination is + called, and the set of applicable methods does not include any + primary methods, then the generic function SB-PCL:NO-PRIMARY-METHOD + will be invoked with the arguments being the invoked generic + function and its arguments, similar to the standard function + NO-APPLICABLE-METHOD. As with NO-APPLICABLE-METHOD, the default + method on SB-PCL:NO-PRIMARY-METHOD signals an error; programmers may + define methods on it.") + +(defsection @extended-slot-access (:title "Extended Slot Access") + "The slot access functions SLOT-VALUE, `(SETF SLOT-VALUE)`, + SLOT-BOUNDP and SLOT-MAKUNBOUND are defined to function as expected + on conditions (of metaclass SB-PCL::CONDITION-CLASS) and, with some + limitations, on structures (of metaclass STRUCTURE-CLASS). + + For structures: + + - The name of a slot for the purposes of the slot access functions + is the symbol used as the slot-name in the slot-description in the + DEFSTRUCT form; + + - SLOT-VALUE and SLOT-BOUNDP function as expected, including (for + SLOT-VALUE) calling and respecting the return value of + SLOT-UNBOUND if the slot is unbound; + + - `(SETF SLOT-VALUE)` functions as expected, including performing + type checks to verify that the new value is of an appropriate type + for the slot; + + - SLOT-MAKUNBOUND makes the slot unbound only when the slot + corresponds to an &AUX argument with no default in a + by-order-of-arguments (BOA) constructor. In all other cases + calling SLOT-MAKUNBOUND on a structure signals an error. + + - If any of the slot access functions is called with a structure + instance which does not have a slot of the given name, + SLOT-MISSING is called and the return value of the effective + method, if any, is respected.") + +(defsection @metaobject-protocol (:title "Metaobject Protocol") + (@amop-compatibility-of-metaobject-protocol section) + (@metaobject-protocol-extensions section)) + +(defsection @amop-compatibility-of-metaobject-protocol + (:title "AMOP Compatibility of Metaobject Protocol") + "SBCL supports a metaobject protocol which is intended to be compatible + with AMOP; present exceptions to this (as distinct from current bugs) + are: + + - SB-MOP:COMPUTE-EFFECTIVE-METHOD only returns one value, not two. + There is no record of what the second return value was meant to + indicate, and apparently no clients for it. + + - The direct superclasses of SB-MOP:FUNCALLABLE-STANDARD-OBJECT are + (FUNCTION STANDARD-OBJECT) instead of the correct (STANDARD-OBJECT + FUNCTION). + + This is to ensure that the STANDARD-OBJECT class is the last of + the standardized classes before class T appearing in the + precedence list of GENERIC-FUNCTION and + STANDARD-GENERIC-FUNCTION, as required by CLHS `1.4.4.5`. + + - The arguments :DECLARE and :DECLARATIONS are both accepted by + ENSURE-GENERIC-FUNCTION, with the leftmost argument defining the + declarations to be stored and returned by + SB-MOP:GENERIC-FUNCTION-DECLARATIONS. + + Where AMOP specifies :DECLARATIONS as the keyword argument to + ENSURE-GENERIC-FUNCTION, the Common Lisp standard specifies + :DECLARE. Portable code should use :DECLARE. + + - Although SBCL obeys the requirement in AMOP that + SB-MOP:VALIDATE-SUPERCLASS should treat STANDARD-CLASS and + SB-MOP:FUNCALLABLE-STANDARD-CLASS as compatible metaclasses, we + impose an additional requirement at class finalization time: a + class of metaclass SB-MOP:FUNCALLABLE-STANDARD-CLASS must have + FUNCTION in its superclasses, and a class of metaclass + STANDARD-CLASS must not. + + After a class has been finalized, it is associated with a class + prototype which is accessible by a standard MOP function + SB-MOP:CLASS-PROTOTYPE. The user can then ask whether this + object is a FUNCTION or not in several different ways: whether + it is a function according to TYPEP; whether its CLASS-OF is + SUBTYPEP FUNCTION, or whether FUNCTION appears in the + superclasses of the class. The additional consistency + requirement comes from the desire to make all of these answers + the same. + + The following class definitions are bad, and will lead to errors + either immediately or if an instance is created: + + (defclass bad-object (funcallable-standard-object) + () + (:metaclass standard-class)) + (defclass bad-funcallable-object (standard-object) + () + (:metaclass funcallable-standard-class)) + + The following definition is acceptable: + + (defclass mixin () + ((slot :initarg slot))) + (defclass funcallable-object (funcallable-standard-object mixin) + () + (:metaclass funcallable-standard-class)) + + and leads to a class whose instances are funcallable and have one slot. + + Note that this requirement also applies to the class + SB-MOP:FUNCALLABLE-STANDARD-OBJECT, which has metaclass + SB-MOP:FUNCALLABLE-STANDARD-CLASS rather than STANDARD-CLASS as + AMOP specifies. + + - The requirement that _no portable class may inherit, by virtue of + being a direct or indirect subclass of a specified class, any slot + for which the name is a symbol accessible in the + `COMMON-LISP-USER` package or exported by any package defined in + the ANSI Common Lisp standard_. is interpreted to mean that the + standardized classes themselves should not have slots named by + external symbols of public packages. + + The rationale behind the restriction is likely to be similar to + the ANSI Common Lisp restriction on defining functions, + variables and types named by symbols in the Common Lisp package: + preventing two independent pieces of software from colliding + with each other. + + - Specializations of the `NEW-VALUE` argument to (SETF + SB-MOP:SLOT-VALUE-USING-CLASS) are not allowed: all user-defined + methods must have a specializer of the class T. + + This prohibition is motivated by a separation of layers: the + SB-MOP:SLOT-VALUE-USING-CLASS family of functions is intended + for use in implementing different and new slot allocation + strategies, rather than in performing application-level + dispatching. Additionally, with this requirement, there is a + one-to-one mapping between metaclass, class and + slot-definition-class tuples and effective methods of (SETF + SB-MOP:SLOT-VALUE-USING-CLASS), which permits optimization + of (SETF SB-MOP:SLOT-VALUE-USING-CLASS)'s discriminating + function in the same manner as for SB-MOP:SLOT-VALUE-USING-CLASS + and SB-MOP:SLOT-BOUNDP-USING-CLASS. + + Note that application code may specialize on the `NEW-VALUE` + argument of slot accessors. + + - The class named by the `NAME` argument to SB-MOP:ENSURE-CLASS, if any, is + only redefined if it is the proper name of that class; otherwise, + a new class is created. + + This is consistent with the description SB-MOP:ENSURE-CLASS in + AMOP as the functional version of DEFCLASS, which has this + behaviour; however, it is not consistent with the weaker + requirement in AMOP, which states that any class found by + FIND-CLASS, no matter what its [CLASS-NAME][function], is + redefined. + + - An error is not signaled in the case of the :NAME initialization + argument for SB-MOP:SLOT-DEFINITION objects being a constant, when + the slot definition is of type SB-PCL::STRUCTURE-SLOT-DEFINITION + (i.e. it is associated with a class of type STRUCTURE-CLASS). + + This allows code which uses constant names for structure slots + to continue working as specified in ANSI, while enforcing the + constraint for all other types of slot. + + - The class T is not an instance of the BUILT-IN-CLASS metaclass. + + AMOP specifies, in the _Inheritance Structure of Metaobject + Classes_ section, that the class T should be an instance of + BUILT-IN-CLASS. However, it also specifies that + SB-MOP:VALIDATE-SUPERCLASS should return true (indicating that a + direct superclass relationship is permissible) if the second + argument is the class T. Also, ANSI specifies that classes with + metaclass BUILT-IN-CLASS may not be subclassed using DEFCLASS, + and also that the class T is the universal superclass, + inconsistent with it being a BUILT-IN-CLASS. + + - Uses of CHANGE-CLASS and redefinitions of classes with + DEFCLASS (or the functional interfaces SB-MOP:ENSURE-CLASS or + SB-MOP:ENSURE-CLASS-USING-CLASS) must ensure that for each slot + with allocation :INSTANCE or :CLASS, the set of applicable methods + on the SB-MOP:SLOT-VALUE-USING-CLASS family of generic functions + is the same before and after the change. + + This is required for correct operation of the protocol to update + instances for the new or redefined class, and can be seen as + part of the contract of the :INSTANCE or :CLASS allocations. + + - Metaobject protocol users may wish to override + SB-MOP:COMPUTE-DISCRIMINATING-FUNCTION for their own generic + function classes. Overriding implementations of + SB-MOP:COMPUTE-DISCRIMINATING-FUNCTION must, in order to + participate in the NO-APPLICABLE-METHOD and + SB-PCL:NO-PRIMARY-METHOD protocols, perform appropriate checks on + the return value of COMPUTE-APPLICABLE-METHODS before processing + the effective method; the standard effective method contains + error-invoking forms, but those forms have no access to the + generic function invocation's arguments.") + +(defsection @metaobject-protocol-extensions + (:title "Metaobject Protocol Extensions") + "In addition, SBCL supports extensions to the Metaobject protocol from + AMOP; at present, they are: + + - Compile-time support for generating specializer metaobjects from + specializer names in DEFMETHOD forms is provided by the + SB-PCL:MAKE-METHOD-SPECIALIZERS-FORM function, which returns a + form which, when evaluated in the lexical environment of the + DEFMETHOD, returns a list of specializer metaobjects. This + operator suffers from similar restrictions to those affecting + SB-MOP:MAKE-METHOD-LAMBDA, namely that the generic function must + be defined when the DEFMETHOD form is expanded, so that the + correct method of SB-PCL:MAKE-METHOD-SPECIALIZERS-FORM is invoked. + The system-provided method on SB-PCL:MAKE-METHOD-SPECIALIZERS-FORM + generates a call to FIND-CLASS for each symbol specializer name, + and a call to SB-MOP:INTERN-EQL-SPECIALIZER for each + `(EQL )` specializer name. + + - Run-time support for converting between specializer names and + specializer metaobjects, mostly for the purposes of FIND-METHOD, + is provided by SB-PCL:PARSE-SPECIALIZER-USING-CLASS and + SB-PCL:UNPARSE-SPECIALIZER-USING-CLASS, which dispatch on their + first argument, the generic function associated with a method with + the given specializer. The system-provided methods on those + methods convert between classes and proper names and between lists + of the form `(EQL )` and interned eql specializer objects. + + - Distinguishing unbound instance allocated slots from bound ones + when using SB-MOP:STANDARD-INSTANCE-ACCESS and + SB-MOP:FUNCALLABLE-STANDARD-INSTANCE-ACCESS is possible by + comparison to the symbol-macro SB-PCL:+SLOT-UNBOUND+.") + +(defsection @extensible-sequences (:title "Extensible Sequences") + "ANSI Common Lisp has a class SEQUENCE with subclasses LIST and + VECTOR, on which the sequence functions like FIND, SUBSEQ, etc. + operate. As an extension to the ANSI specification, SBCL allows + additional subclasses of SEQUENCE to be defined. + + > A motivation, rationale and additional examples for the design of + > this extension can be found in the paper _Rhodes, + > Christophe (2007): User-extensible sequences in Common Lisp_ + > available for download at + > . + + Users of this extension just make instances of SEQUENCE subclasses + and transparently operate on them using sequence functions: + + (coerce (subseq (make-instance 'my-sequence) 5 10) 'list) + + From this perspective, no distinction between builtin and user-defined + SEQUENCE subclasses should be necessary. + + Providers of the extension, that is of user-defined SEQUENCE + subclasses, have to adhere to a _sequence protocol_ which consists + of a set of generic functions in the SEQUENCE package. + + A minimal SEQUENCE subclass has to specify STANDARD-OBJECT and + SEQUENCE as its superclasses and has to be the specializer of the + SEQUENCE parameter of methods on at least the following generic + functions:" + (sb-sequence:length generic-function) + (sb-sequence:elt generic-function) + (sb-sequence:elt setf-generic-function) + (sb-sequence:adjust-sequence generic-function) + (sb-sequence:make-sequence-like generic-function) + "`MAKE-SEQUENCE-LIKE` is needed for functions returning + freshly-allocated sequences such as SUBSEQ or COPY-SEQ. + `ADJUST-SEQUENCE` is needed for functions which destructively modify + their arguments such as DELETE. In fact, all other sequence + functions can be implemented in terms of the above functions and + actually are, if no additional methods are defined. However, relying + on these generic implementations, in particular not implementing the + @EXSEQ-ITERATOR-PROTOCOL can incur a high performance penalty. + + When the sequence protocol is only partially implemented for a given + SEQUENCE subclass, an attempt to apply one of the missing operations + to instances of that class signals the following condition:" + (sb-sequence:protocol-unimplemented condition) + "In addition to the mandatory functions above, methods on the sequence + functions listed below can be defined. + + There are some noteworthy irregularities: + + - The function SB-SEQUENCE:EMPTYP does not have a counterpart in the + `CL` package. It is intended to be used instead of + SB-SEQUENCE:LENGTH when working with lazy or infinite sequences. + + - SB-SEQUENCE:DOSEQUENCE does not have a direct counterpart either. + It is like DOLIST in spirit but traverses generic sequences. + + - The functions MAP, CONCATENATE and MERGE receive a type designator + specifying the type of the constructed sequence as their first + argument. However, the corresponding generic functions + SB-SEQUENCE:MAP, SB-SEQUENCE:CONCATENATE and SB-SEQUENCE:MERGE + receive a prototype instance of the requested SEQUENCE subclass + instead. + + - CL:MAP-INTO has no generic sequence counterpart, as its lambda + list does not provide reasonable specialization opportunities, but + it supports extensible sequences directly." + (sb-sequence:emptyp generic-function) + (sb-sequence:dosequence macro) + "The remaining list parallels the _Sequence Dictionary_, `17.3` CLHS." + (sb-sequence:copy-seq generic-function) + (sb-sequence:fill generic-function) + (sb-sequence:subseq generic-function) + (sb-sequence:map function) + (sb-sequence:reduce generic-function) + (sb-sequence:search generic-function) + (sb-sequence:mismatch generic-function) + (sb-sequence:replace generic-function) + (sb-sequence:concatenate function) + (sb-sequence:merge function) + "Counting:" + (sb-sequence:count generic-function) + (sb-sequence:count-if generic-function) + (sb-sequence:count-if-not generic-function) + "Reversing:" + (sb-sequence:reverse generic-function) + (sb-sequence:nreverse generic-function) + "Sorting:" + (sb-sequence:sort generic-function) + (sb-sequence:stable-sort generic-function) + "Finding an element:" + (sb-sequence:find generic-function) + (sb-sequence:find-if generic-function) + (sb-sequence:find-if-not generic-function) + "Finding a position:" + (sb-sequence:position generic-function) + (sb-sequence:position-if generic-function) + (sb-sequence:position-if-not generic-function) + "Substituting elements:" + (sb-sequence:substitute generic-function) + (sb-sequence:substitute-if generic-function) + (sb-sequence:substitute-if-not generic-function) + (sb-sequence:nsubstitute generic-function) + (sb-sequence:nsubstitute-if generic-function) + (sb-sequence:nsubstitute-if-not generic-function) + "Removing elements:" + (sb-sequence:remove generic-function) + (sb-sequence:remove-if generic-function) + (sb-sequence:remove-if-not generic-function) + (sb-sequence:delete generic-function) + (sb-sequence:delete-if generic-function) + (sb-sequence:delete-if-not generic-function) + "Removing duplicates:" + (sb-sequence:remove-duplicates generic-function) + (sb-sequence:delete-duplicates generic-function) + (@exseq-iterator-protocol section) + (@exseq-simple-iterator-protocol section)) + +(defsection @exseq-iterator-protocol (:title "Iterator Protocol") + "The general iterator protocol allows subsequently accessing some or + all elements of a sequence in forward or reverse direction. Users + first call SB-SEQUENCE:MAKE-SEQUENCE-ITERATOR to create an iteration + state and receive functions to query and mutate it. These functions + allow, among other things, moving to, retrieving or modifying + elements of the sequence. The iteration state consists of a state + object, a limit object, a from-end indicator and six functions to + query or mutate this state. + + An iterator is created by calling:" + (sb-sequence:make-sequence-iterator function) + "The following convenience macros simplify traversing sequences using + iterators:" + (sb-sequence:with-sequence-iterator macro) + (sb-sequence:with-sequence-iterator-functions macro)) + +(defsection @exseq-simple-iterator-protocol (:title "Simple Iterator Protocol") + "For cases in which the full flexibility and performance of the general + sequence iterator protocol is not required, there is a simplified + sequence iterator protocol consisting of a few generic functions which + can be specialized for iterator classes:" + (sb-sequence:iterator-step generic-function) + (sb-sequence:iterator-endp generic-function) + (sb-sequence:iterator-element generic-function) + (sb-sequence:iterator-element setf-generic-function) + (sb-sequence:iterator-index generic-function) + (sb-sequence:iterator-copy generic-function) + "Iterator objects implementing the above simple iteration protocol are + created by calling the following generic function:" + (sb-sequence:make-simple-sequence-iterator generic-function)) + +(defsection @support-for-unix (:title "Support For Unix") + (sb-ext:*posix-argv* (variable "")) + (sb-ext:posix-getenv function) + (sb-ext:posix-environ function) + (@running-external-programs section)) + +(defsection @running-external-programs (:title "Running external programs") + "External programs can be run with SB-EXT:RUN-PROGRAM. + + > _Note_: In SBCL versions prior to 1.0.13, SB-EXT:RUN-PROGRAM + > searched for executables in a manner somewhat incompatible with + > other languages. As of this version, SBCL uses the system library + > routine `execvp(3)`, and no longer contains the function + > `FIND-EXECUTABLE-IN-SEARCH-PATH`, which implemented the old + > search. Users who need this function may find it in + > `run-program.lisp` versions 1.67 and earlier in SBCL's CVS + > repository here + > . + > However, we caution such users that this search routine finds + > executables that system library routines do not." + (sb-ext:run-program function) + "When SB-EXT:RUN-PROGRAM is called with :WAIT NIL, an process object + is returned. The following functions are available for use with + processes:" + (sb-ext:process-p function) + (sb-ext:process-input function) + (sb-ext:process-output function) + (sb-ext:process-error function) + (sb-ext:process-alive-p function) + (sb-ext:process-status function) + (sb-ext:process-wait function) + (sb-ext:process-exit-code function) + (sb-ext:process-core-dumped function) + (sb-ext:process-close function) + (sb-ext:process-kill function)) + +(defsection @unicode-support (:title "Unicode Support") + "SBCL provides support for working with Unicode text and querying the + standard Unicode database for information about individual codepoints. + Unicode-related functions are located in the `SB-UNICODE` package. + + SBCL also extends ANSI character literal syntax to support Unicode + codepoints. You can either specify a character by its Unicode name, + with spaces replaced by underscores if a unique name exists or by + giving its hexadecimal codepoint preceded by a `\\U`, an optional + `\\+`, and an arbitrary number of leading zeros. You may also input + the character directly into your source code if it can be encoded in + your file. If a character had an assigned name in Unicode 1.0 that + was distinct from its current name, you may also use that name (with + spaces replaced by underscores) to specify the character, unless the + name is already associated with a codepoint in the latest Unicode + standard (such as `\\BELL`). + + > _Note_: Please note that the codepoint `U+1F5CF` (Page) introduced + > in Unicode 7.0 is named `UNICODE_PAGE`, since the name _Page_ is + > required to be assigned to form-feed (`U+0C`) by the ANSI + > standard. + + For example, you can specify the codepoint `U+00E1` ( _Latin Small + Letter A With Acute_) as + + - `#\\LATIN_SMALL_LETTER_A_WITH_ACUTE` + - `#\\LATIN_SMALL_LETTER_A_ACUTE` + - `#\\รก` (assuming a Unicode source file) + - `#\\U00E1` + - `#\\UE1` + - `#\\U+00E1`" + (@unicode-property-access section) + (@string-operations section) + (@breaking-strings section)) + +(defsection @unicode-property-access (:title "Unicode property access") + "The following functions can be used to find information about a + Unicode codepoint." + (sb-unicode:general-category function) + (sb-unicode:bidi-class function) + (sb-unicode:combining-class function) + (sb-unicode:decimal-value function) + (sb-unicode:digit-value function) + (sb-unicode:numeric-value function) + (sb-unicode:mirrored-p function) + (sb-unicode:bidi-mirroring-glyph function) + (sb-unicode:age function) + (sb-unicode:hangul-syllable-type function) + (sb-unicode:east-asian-width function) + (sb-unicode:script function) + (sb-unicode:char-block function) + (sb-unicode:unicode-1-name function) + (sb-unicode:proplist-p function) + (sb-unicode:uppercase-p function) + (sb-unicode:lowercase-p function) + (sb-unicode:cased-p function) + (sb-unicode:case-ignorable-p function) + (sb-unicode:alphabetic-p function) + (sb-unicode:ideographic-p function) + (sb-unicode:math-p function) + (sb-unicode:whitespace-p function) + (sb-unicode:soft-dotted-p function) + (sb-unicode:hex-digit-p function) + (sb-unicode:default-ignorable-p function) + (sb-unicode:grapheme-break-class function) + (sb-unicode:word-break-class function) + (sb-unicode:sentence-break-class function) + (sb-unicode:line-break-class function)) + +(defsection @string-operations (:title "String operations") + "SBCL can normalize strings using:" + (sb-unicode:normalize-string function) + (sb-unicode:normalized-p function) + "SBCL implements the full range of Unicode case operations with the + functions" + (sb-unicode:uppercase function) + (sb-unicode:lowercase function) + (sb-unicode:titlecase function) + (sb-unicode:casefold function) + "It also extends standard Common Lisp case functions such as + STRING-UPCASE and STRING-DOWNCASE to support a subset of Unicode's + casing behavior. Specifically, a character is BOTH-CASE-P if its + case mapping in Unicode is one-to-one and invertable. + + The `SB-UNICODE` package also provides functions for + collating/sorting strings according to the Unicode Collation + Algorithm." + (sb-unicode:unicode< function) + (sb-unicode:unicode= function) + (sb-unicode:unicode-equal function) + (sb-unicode:unicode<= function) + (sb-unicode:unicode> function) + (sb-unicode:unicode>= function) + "The following functions are provided for detecting visually + confusable strings:" + (sb-unicode:confusable-p function)) + +(defsection @breaking-strings (:title "Breaking strings") + "The `SB-UNICODE` package includes several functions for breaking a + Unicode string into useful parts." + (sb-unicode:graphemes function) + (sb-unicode:words function) + (sb-unicode:sentences function) + (sb-unicode:lines function)) + +(defsection @customization-hooks-for-users + (:title "Customization Hooks for Users") + "The toplevel repl prompt may be customized, and the function + that reads user input may be replaced completely. See the :TOPLEVEL + argument of SB-EXT:SAVE-LISP-AND-DIE. + + The behaviour of REQUIRE when called with only one argument is + implementation-defined. In SBCL, REQUIRE behaves in the following + way:" + (require function) + (sb-ext:*module-provider-functions* variable) + "Although SBCL does not provide a resident editor, the ED + function can be customized to hook into user-provided editing + mechanisms as follows:" + (ed function) + (sb-ext:*ed-functions* variable) + "Conditions of type WARNING and STYLE-WARNING are sometimes signaled at + runtime, especially during execution of Common Lisp defining forms + such as DEFUN, DEFMETHOD, etc. To muffle these warnings at runtime, + SBCL provides a variable SB-EXT:*MUFFLED-WARNINGS*:" + (sb-ext:*muffled-warnings* variable)) + +(defsection @tools-to-help-developers (:title "Tools To Help Developers") + "SBCL provides a profiler and other extensions to the TRACE + facility. + + The debugger supports a number of options. Its documentation is + accessed by typing `\\help` at the debugger prompt. See @DEBUGGER. + + Documentation for the command `\\inspect` is accessed by typing + `\\help` at the `\\inspect` prompt.") + +(defsection @resolution-of-name-conflicts + (:title "Resolution of Name Conflicts") + "CLHS `11.1.1.2.5` requires that name conflicts in packages be + resolvable in favour of any of the conflicting symbols. In the + interactive debugger, this is achieved by prompting for the symbol + in whose favour the conflict should be resolved; for programmatic + use, the SB-EXT:RESOLVE-CONFLICT restart should be invoked + with one argument, which should be a member of the list returned by + the condition accessor SB-EXT:NAME-CONFLICT-SYMBOLS.") + +(defsection @hash-table-extensions (:title "Hash Table Extensions") + "Hash table extensions supported by SBCL are all controlled by keyword + arguments to MAKE-HASH-TABLE." + (make-hash-table function) + (sb-ext:define-hash-table-test macro) + (sb-ext:with-locked-hash-table macro) + (sb-ext:hash-table-synchronized-p function) + (sb-ext:hash-table-weakness function)) + +(defsection @random-number-generation (:title "Random Number Generation") + "The initial value of *RANDOM-STATE* is the same each time SBCL + is started. This makes it possible for user code to obtain + repeatable pseudo random numbers using only standard-provided + functionality. See SB-EXT:SEED-RANDOM-STATE below for an SBCL + extension that allows to seed the random number generator from given + data for an additional possibility to achieve this. Non-repeatable + random numbers can always be obtained using (MAKE-RANDOM-STATE T). + + The sequence of numbers produced by repeated calls to RANDOM + starting with the same random state and using the same sequence of + `LIMIT` arguments is guaranteed to be reproducible only in the same + version of SBCL on the same platform, using the same code under the + same evaluator mode and compiler optimization qualities. Just two + examples of differences that may occur otherwise: calls to RANDOM + can be compiled differently depending on how much is known about the + `LIMIT` argument at compile time, yielding different results even if + called with the same argument at run time, and the results can + differ depending on the machine's word size, for example for limits + that are fixnums under 64-bit word size but bignums under 32-bit + word size." + (sb-ext:seed-random-state function) + "Some notes on random floats: The standard doesn't prescribe a specific + method of generating random floats. The following paragraph + describes SBCL's current implementation and should be taken as + purely informational, that is, user code should not depend on any of + its specific properties. The method used has been chosen because it + is common, conceptually simple and fast. + + To generate random floats, SBCL evaluates code that has an equivalent + effect as + + (* limit + (float (/ (random (expt 2 23)) (expt 2 23)) 1.0f0)) + + (for SINGLE-FLOATs) and correspondingly (with `52` and `1.0d0` + instead of `23` and `1.0f0`) for DOUBLE-FLOATs. Note especially that + this means that zero is a possible return value occurring with + probability `(EXPT 2 -23)` and `(EXPT 2 -52)`, respectively. Also + note that there exist twice as many equidistant floats between 0 and + 1 as are generated. For example, the largest number that + `(RANDOM 1.0F0)` ever returns is `(FLOAT (/ (1- (EXPT 2 23)) (EXPT 2 + 23)) 1.0F0)` while `(FLOAT (/ (1- (EXPT 2 24)) (EXPT 2 24)) 1.0F0)` + is the largest SINGLE-FLOAT less than 1. This is a side effect of + the fact that the implementation uses the fastest possible + conversion from bits to floats. + + SBCL currently uses the Mersenne Twister as its random number + generator, specifically the 32-bit version under both 32- and 64-bit + word size. The seeding algorithm has been improved several times by + the authors of the Mersenne Twister; SBCL uses the third version + (from 2002), which is still the most recent as of June 2012. The + implementation has been tested to provide output identical to the + recommended \\C implementation. + + While the Mersenne Twister generates random numbers of much better + statistical quality than other widely used generators, it uses only + linear operations modulo 2 and thus fails some statistical + tests. + + (See chapter 7 _Testing widely used RNGs_ in _TestU01: A C Library + for Empirical Testing of Random Number Generators_ by Pierre + L'Ecuyer and Richard Simard, ACM Transactions on Mathematical + Software, Vol. 33, article 22, 2007.) + + For example, the distribution of ranks of (sufficiently large) + random binary matrices is much distorted compared to the + theoretically expected one when the matrices are generated by the + Mersenne Twister. Thus, applications that are sensitive to this + aspect should use a different type of generator.") + +(defsection @timeouts-and-deadlines (:title "Timeouts and Deadlines") + "SBCL supports three different ways of restricting the execution time + available to individual operations or parts of computations: + + - _Timeout Parameters_: Some operations such as thread + synchronization primitives accept a :TIMEOUT parameter. See + @TIMEOUT-PARAMETERS. + + - _Synchronous Timeouts (Deadlines)_: Certain operations that may + suspend execution for extended periods of time such as CL:SLEEP, + thread synchronization primitives, IO and waiting for external + processes respect deadlines established for a part of a + computation. See @SYNCHRONOUS-TIMEOUTS. + + - _Asynchronous Timeouts_: Asynchronous timeouts can interrupt most + computations at (almost) any point. Thus, this kind of timeouts is + the most versatile but it is also somewhat unsafe. See + @ASYNCHRONOUS-TIMEOUTS." + (@timeout-parameters section) + (@synchronous-timeouts section) + (@asynchronous-timeouts section) + (@operations-supporting-timeouts-and-deadlines section)) + +(defsection @timeout-parameters (:title "Timeout Parameters") + "Certain operations accept :TIMEOUT keyword arguments. These only + affect the specific operation and must be specified at each call + site by passing a :TIMEOUT keyword argument and a corresponding + timeout value to the respective operation. Expiration of the timeout + before the operation completes results in either a normal return + with a return value indicating the timeout or in the signaling of a + specialized condition such as SB-THREAD:JOIN-THREAD-ERROR. + + Example: + + (defun join-thread-within-5-seconds (thread) + (multiple-value-bind (value result) + (sb-thread:join-thread thread :default nil :timeout 5) + (when (eq result :timeout) + (error \"Could not join ~A within 5 seconds\" thread)) + value)) + + The above code attempts to join the specified thread for up to five + seconds, returning its value in case of success. If the thread is + still running after the five seconds have elapsed, + SB-THREAD:JOIN-THREAD indicates the timeout in its second return + value. If a :DEFAULT value was not provided, SB-THREAD:JOIN-THREAD + would signal a SB-THREAD:JOIN-THREAD-ERROR instead. + + To wait for an arbitrary condition, optionally with a timeout, the + SB-EXT:WAIT-FOR macro can be used:" + (sb-ext:wait-for macro) + ;; SB-SYS:MAKE-FD-STREAM also takes a :TIMEOUT} argument resulting + ;; in @code{sb-sys:io-timeout}, but that seems too niche to document + ;; here. + ) + +(defsection @synchronous-timeouts (:title "Synchronous Timeouts") + "Deadlines, in contrast to timeout parameters, are established for a + dynamic scope using the SB-SYS:WITH-DEADLINE macro and indirectly + affect operations within that scope. In case of nested uses, the + effective deadline is the one that expires first unless an inner use + explicitly overrides outer deadlines." + (sb-sys:with-deadline macro) + "Expiration of deadlines set up this way only has an effect when it + happens before or during the execution of a deadline-aware operation + (@OPERATIONS-SUPPORTING-TIMEOUTS-AND-DEADLINES). In this case, a + SB-SYS:DEADLINE-TIMEOUT is signaled. A handler for this condition + type may use the SB-SYS:DEFER-DEADLINE or SB-SYS:CANCEL-DEADLINE + restarts to defer or cancel the deadline respectively and resume + execution of the interrupted operation." + (sb-sys:deadline-timeout condition) + (sb-sys:defer-deadline function) + (sb-sys:cancel-deadline function) + "When a thread is executing the debugger, signaling of + SB-SYS:DEADLINE-TIMEOUT conditions for that thread is deferred until + it exits the debugger. + + Example: + + (defun read-input () + (list (read-line) (read-line))) + + (defun do-it () + (sb-sys:with-deadline (:seconds 5)) + (read-input) + (sleep 2) + (sb-ext:run-program \"my-program\")) + + The above code establishes a deadline of five seconds within which + the body of the `DO-IT` function should execute. All calls of + deadline-aware functions in the dynamic scope, in this case two + READ-LINE calls, a SLEEP call and a SB-EXT:RUN-PROGRAM call, are + affected by the deadline. If, for example, the first READ-LINE call + completes in one second and the second READ-LINE call completes in + three seconds, a SB-SYS:DEADLINE-TIMEOUT condition will be signaled + after the SLEEP call has been executing for one second.") + +(defsection @asynchronous-timeouts (:title "Asynchronous Timeouts") + "Asynchronous timeouts are established for a dynamic scope using the + SB-EXT:WITH-TIMEOUT macro:" + (sb-ext:with-timeout macro) + "Expiration of the timeout will cause the operation being executed at + that moment to be interrupted by an asynchronously signaled + SB-EXT:TIMEOUT condition, (almost) irregardless of the operation + and its context." + (sb-ext:timeout condition)) + +(defsection @operations-supporting-timeouts-and-deadlines + (:title "Operations Supporting Timeouts and Deadlines") + ;; FIXME: make this conditional on texinfo output + #+nil + "@multitable @columnfractions .5 .25 .25 + @headitem Operation @tab Timeout Parameter @tab Affected by Deadlines + - @code{cl:sleep} @tab - @tab since SBCL 1.4.3 + - @code{cl:read-line}, etc. @tab no @tab yes + - @ref{Macro sb-ext wait-for,,@code{wait-for}}@: @tab yes @tab yes + - @ref{Function sb-ext process-wait,,@code{process-wait}}@: @tab no @tab yes + - @ref{Function sb-thread grab-mutex,,@code{grab-mutex}}@: @tab yes @tab yes + - @ref{Function sb-thread condition-wait,,@code{condition-wait}}@: @tab yes @tab yes + - @ref{Function sb-thread wait-on-semaphore,,@code{wait-on-semaphore}}@: @tab yes @tab yes + - @ref{Function sb-thread join-thread,,@code{join-thread}}@: @tab yes @tab yes + - @ref{Function sb-concurrency receive-message,,@code{receive-message}}@: @tab yes @tab yes? + - @ref{Function sb-concurrency wait-on-gate,,@code{wait-on-gate}}@: @tab yes @tab yes? + - @ref{Macro sb-concurrency frlock-write,,@code{frlock-write}}@: @tab yes @tab yes? + - @ref{Function sb-concurrency grab-frlock-write-lock,,@code{grab-frlock-write-lock}}@: @tab yes @tab yes? + @end multitable" + "``` + | Operation | Timeout parameter | Affected by deadlines | + |------------------------+-------------------+-----------------------| + | cl:sleep | - | since SBCL 1.4.3 | + | cl:read-line, etc. | no | yes | + | wait-for | yes | yes | + | process-wait | no | yes | + | grab-mutex | yes | yes | + | condition-wait | yes | yes | + | wait-on-semaphore | yes | yes | + | join-thread | yes | yes | + | receive-message | yes | yes? | + | wait-on-gate | yes | yes? | + | frlock-write | yes | yes? | + | grab-frlock-write-lock | yes | yes? | + ```") + +(defsection @miscellaneous-extensions (:title "Miscellaneous Extensions") + (sb-ext:array-storage-vector function) + (sb-ext:delete-directory function) + (sb-ext:get-time-of-day function) + (sb-ext:assert-version->= function) + (sb-ext:unencapsulated-function function)) + +(defsection @stale-extensions (:title "Stale Extensions") + "SBCL has inherited from CMUCL various hooks to allow the user to + tweak and monitor the garbage collection process. These are somewhat + stale code, and their interface might need to be cleaned up. If you + have urgent need of them, look at the code in `src/code/gc.lisp` and + bring it up on the developers' mailing list. + + SBCL has various hooks inherited from CMUCL, like + SB-EXT:FLOAT-DENORMALIZED-P, to allow a program to take advantage of + IEEE floating point arithmetic properties which aren't conveniently + or efficiently expressible using the ANSI standard. These look good, + and their interface looks good, but IEEE support is slightly broken + due to a stupid decision to remove some support for infinities + (because it wasn't in the ANSI spec and it didn't occur to me that + it was in the IEEE spec). If you need this stuff, take a look at the + code and bring it up on the developers' mailing list.") + +(defsection @efficiency-hacks (:title "Efficiency Hacks") + "The SB-EXT:PURIFY function (available when `#+cheneygc`) causes + SBCL first to collect all garbage, then to mark all uncollected + objects as permanent, never again attempting to collect them as + garbage. This can cause a large increase in efficiency when using a + primitive garbage collector, or a more moderate increase in + efficiency when using a more sophisticated garbage collector which + is well suited to the program's memory usage pattern. It also allows + permanent code to be frozen at fixed addresses, a precondition for + using copy-on-write to share code between multiple Lisp processes. + This is less important with modern generational garbage collectors, + but not all SBCL platforms use such a garbage collector. + + The SB-EXT:TRULY-THE special form declares the type of the result of + the operations, producing its argument; the declaration is not + checked. In short: don't use it." + ;; FIXME: It's not a macro. + (sb-ext:truly-the macro) + "The SB-EXT:FREEZE-TYPE declaration declares that a type will never + change, which can make type testing (e.g. with TYPEP) more efficient + for structure types.") diff --git a/contrib/sb-manual/doc/compiler.lisp b/contrib/sb-manual/doc/compiler.lisp new file mode 100644 index 000000000..f88382826 --- /dev/null +++ b/contrib/sb-manual/doc/compiler.lisp @@ -0,0 +1,888 @@ +(in-package :sb-manual) + +(defsection @compiler (:title "Compiler") + "This chapter will discuss most compiler issues other than efficiency, + including compiler error messages, the SBCL compiler's unusual + approach to type safety in the presence of type declarations, the + effects of various compiler optimization policies, and the way that + inlining and open coding may cause optimized code to differ from a + naive translation. Efficiency issues are sufficiently varied and + separate that they have their own chapter, @EFFICIENCY." + (@diagnostic-messages section) + (@handling-of-types section) + (@compiler-policy section) + (@compiler-errors section) + (@open-coding-and-inline-expansion section) + (@interpreter section) + (@advanced-compiler-use-and-efficiency-hints section)) + +(defsection @diagnostic-messages (:title "Diagnostic Messages") + (@controlling-verbosity section) + (@diagnostic-severity section) + (@understanding-compiler-diagnostics section)) + +(defsection @controlling-verbosity (:title "Controlling Verbosity") + "The compiler can be quite verbose in its diagnostic reporting, rather + more then some users would prefer -- the amount of noise emitted can + be controlled, however. + + To control emission of compiler diagnostics (of any severity other + than ERROR: @DIAGNOSTIC-SEVERITY) use the SB-EXT:MUFFLE-CONDITIONS + and SB-EXT:UNMUFFLE-CONDITIONS declarations, specifying the type of + condition that is to be muffled (the muffling is done using an + associated MUFFLE-WARNING restart). + + Global control: + + ;;; Muffle compiler-notes globally + (declaim (sb-ext:muffle-conditions sb-ext:compiler-note)) + + Local control: + + ;;; Muffle compiler-notes based on lexical scope + (defun foo (x) + (declare (optimize speed) (fixnum x) + (sb-ext:muffle-conditions sb-ext:compiler-note)) + (values (* x 5) ; no compiler note from this + (locally + (declare (sb-ext:unmuffle-conditions sb-ext:compiler-note)) + ;; this one gives a compiler note + (* x -5)))) + + - [__declaration__] SB-EXT:MUFFLE-CONDITIONS + + Syntax: `(SB-EXT:MUFFLE-CONDITIONS &REST TYPES)`. + + Muffle the diagnostic messages that would be caused by + compile-time signals of TYPES. + + - [__declaration__] SB-EXT:UNMUFFLE-CONDITIONS + + Syntax: `(SB-EXT:MUFFLE-CONDITIONS &REST TYPES)`. + + Cancel the effect of a previous SB-EXT:MUFFLE-CONDITIONS + declaration. + + Various details of _how_ the compiler messages are printed can be + controlled via the alist SB-EXT:*COMPILER-PRINT-VARIABLE-ALIST*." + (sb-ext:*compiler-print-variable-alist* variable) + "For information about muffling warnings signaled outside of the + compiler, see @CUSTOMIZATION-HOOKS-FOR-USERS.") + +;; FIXME: How much control over error messages is in SBCL? How much +;; should be? How much of this documentation should we save or adapt? +;; +;; %%\node Error Message Parameterization, , Read Errors, Interpreting Error Messages +;; \subsection{Error Message Parameterization} +;; \cpsubindex{error messages}{verbosity} +;; \cpsubindex{verbosity}{of error messages} +;; +;; There is some control over the verbosity of error messages. See also +;; \varref{undefined-warning-limit}, \code{*efficiency-note-limit*} and +;; \varref{efficiency-note-cost-threshold}. +;; +;; \begin{defvar}{}{enclosing-source-cutoff} +;; +;; This variable specifies the number of enclosing actual source forms +;; that are printed in full, rather than in the abbreviated processing +;; path format. Increasing the value from its default of \code{1} +;; allows you to see more of the guts of the macroexpanded source, +;; which is useful when debugging macros. +;; \end{defvar} +;; +;; \begin{defmac}{extensions:}{define-source-context}{% +;; \args{\var{name} \var{lambda-list} \mstar{form}}} +;; +;; This macro defines how to extract an abbreviated source context from +;; the \var{name}d form when it appears in the compiler input. +;; \var{lambda-list} is a \code{defmacro} style lambda-list used to +;; parse the arguments. The \var{body} should return a list of +;; subforms that can be printed on about one line. There are +;; predefined methods for \code{defstruct}, \code{defmethod}, etc. If +;; no method is defined, then the first two subforms are returned. +;; Note that this facility implicitly determines the string name +;; associated with anonymous functions. +;; \end{defmac} + +(defsection @diagnostic-severity (:title "Diagnostic Severity") + "There are four levels of compiler diagnostic severity: + + - error + - warning + - style warning + - note + + The first three levels correspond to condition classes which are + defined in the ANSI standard for Common Lisp and which have special + significance to the COMPILE and COMPILE-FILE functions. These levels + of compiler error severity occur when the compiler handles + conditions of these classes. + + The fourth level of compiler error severity, _note_, corresponds to + the SB-EXT:COMPILER-NOTE, and is used for problems which are too + mild for the standard condition classes, typically hints about how + efficiency might be improved. The SB-EXT:CODE-DELETION-NOTE, a + subtype of SB-EXT:COMPILER-NOTE, is signalled when the compiler + deletes user-supplied code after proving that the code in question + is unreachable. + + Future work for SBCL includes expanding this hierarchy of types to + allow more fine-grained control over emission of diagnostic + messages." + (sb-ext:compiler-note condition) + (sb-ext:code-deletion-note condition)) + +(defsection @understanding-compiler-diagnostics + (:title "Understanding Compiler Diagnostics") + "The messages emitted by the compiler contain a lot of detail in a + terse format, so they may be confusing at first. The messages will be + illustrated using this example program: + + (defmacro zoq (x) + `(roq (ploq (+ ,x 3)))) + + (defun foo (y) + (declare (symbol y)) + (zoq y)) + + The main problem with this program is that it is trying to add `3` + to a symbol. Note also that the functions `ROQ` and `PLOQ` aren't + defined anywhere." + (@parts-of-a-compiler-diagnostic section) + (@original-and-actual-source section) + (@processing-path section)) + +(defsection @parts-of-a-compiler-diagnostic + (:title "Parts of a Compiler Diagnostic") + "When processing this program, the compiler will produce this warning: + + ; file: /tmp/foo.lisp + ; in: DEFUN FOO + ; (ZOQ Y) + ; --> ROQ PLOQ + ; ==> + ; (+ Y 3) + ; + ; caught WARNING: + ; Asserted type NUMBER conflicts with derived type (VALUES SYMBOL &OPTIONAL). + + In this example we see each of the six possible parts of a compiler + diagnostic: + + - `file: /tmp/foo.lisp` is the name of the file that the compiler + read the relevant code from. The file name is displayed because it + may not be immediately obvious when there is an error during + compilation of a large system, especially when + WITH-COMPILATION-UNIT is used to delay undefined warnings. + + - `in: DEFUN FOO` is the definition top level form responsible for + the diagnostic. It is obtained by taking the first two elements of + the enclosing form whose first element is a symbol beginning with + `DEF`. If there is no such enclosing `DEF` form, then the + outermost form is used. If there are multiple `DEF` forms, then + they are all printed from the outside in, separated by `=>`s. In + this example, the problem was in the DEFUN for `FOO`. + + - `(ZOQ Y)` is the _original source_ form responsible for the + diagnostic. Original source means that the form directly appeared + in the original input to the compiler, i.e. in the lambda passed + to COMPILE or in the top level form read from the source file. In + this example, the expansion of the `ZOQ` macro was responsible for + the message. + + - `--> ROQ PLOQ` This is the _processing path_ that the compiler + used to produce the code that caused the message to be emitted. + The processing path is a representation of the evaluated forms + enclosing the actual source that the compiler encountered when + processing the original source. The path is the first element of + each form, or the form itself if the form is not a list. These + forms result from the expansion of macros or source-to-source + transformation done by the compiler. In this example, the + enclosing evaluated forms are the calls to `ROQ` and `PLOQ`. These + calls resulted from the expansion of the `ZOQ` macro. + + - `==> (+ Y 3)` is the _actual source_ responsible for the + diagnostic. If the actual source appears in the explanation, then + we print the next enclosing evaluated form, instead of printing + the actual source twice. (This is the form that would otherwise + have been the last form of the processing path.) In this example, + the problem is with the evaluation of the reference to the + variable `Y`. + + - `caught WARNING: Asserted type NUMBER conflicts with derived type + (VALUES SYMBOL &OPTIONAL).` is the _explanation_ of the problem. + In this example, the problem is that, while the call to `+` + requires that its arguments are all of type NUMBER, the compiler + has derived that Y will evaluate to a SYMBOL. Note that + `(VALUES SYMBOL &OPTIONAL)` expresses that `Y` evaluates to + precisely one value. + + Note that each part of the message is distinctively marked: + + - `file:` and `in:` mark the file and definition, respectively. + + - The original source is an indented form with no prefix. + + - Each line of the processing path is prefixed with `-->`. + + - The actual source form is indented like the original source, but + is marked by a preceding `==>` line. (FIXME: no it isn't.) + + - The explanation is prefixed with the diagnostic severity, which + can be `caught ERROR:`, `caught WARNING:`, `caught + STYLE-WARNING:`, or `note:`. + + Each part of the message is more specific than the preceding one. If + consecutive messages are for nearby locations, then the front part + of the messages would be the same. In this case, the compiler omits + as much of the second message as in common with the first. For + example: + + ; file: /tmp/foo.lisp + ; in: DEFUN FOO + ; (ZOQ Y) + ; --> ROQ + ; ==> + ; (PLOQ (+ Y 3)) + ; + ; caught STYLE-WARNING: + ; undefined function: PLOQ + + ; ==> + ; (ROQ (PLOQ (+ Y 3))) + ; + ; caught STYLE-WARNING: + ; undefined function: ROQ + + In this example, the file, definition and original source are + identical for the two messages, so the compiler omits them in the + second message. If consecutive messages are entirely identical, then + the compiler prints only the first message, followed by: `[Last + message occurs times]` where `` is the number of + times the message was given. + + If the source was not from a file, then no file line is printed. If + the actual source is the same as the original source, then the + processing path and actual source will be omitted. If no forms + intervene between the original source and the actual source, then + the processing path will also be omitted.") + +(defsection @original-and-actual-source (:title "Original and Actual Source") + "The _original source_ displayed will almost always be a list. If + the actual source for an message is a symbol, the original source will + be the immediately enclosing evaluated list form. So even if the + offending symbol does appear in the original source, the compiler will + print the enclosing list and then print the symbol as the actual + source (as though the symbol were introduced by a macro.) + + When the _actual source_ is displayed (and is not a symbol), it will + always be code that resulted from the expansion of a macro or a + source-to-source compiler optimization. This is code that did not + appear in the original source program; it was introduced by the + compiler. + + Keep in mind that when the compiler displays a source form in an + diagnostic message, it always displays the most specific (innermost) + responsible form. For example, compiling this function + + (defun bar (x) + (let (a) + (declare (fixnum a)) + (setq a (foo x)) + a)) + + gives this error message + + ; file: /tmp/foo.lisp + ; in: DEFUN BAR + ; (LET (A) + ; (DECLARE (FIXNUM A)) + ; (SETQ A (FOO X)) + ; A) + ; + ; caught WARNING: + ; Asserted type FIXNUM conflicts with derived type (VALUES NULL &OPTIONAL). + + This message is not saying that there is a problem somewhere in this + LET -- it is saying that there is a problem with the LET itself. In + this example, the problem is that `A`'s NIL initial value is not a + FIXNUM.") + +(defsection @processing-path (:title "Processing Path") + "The processing path is mainly useful for debugging macros, so if you + don't write macros, you can probably ignore it. Consider this example: + + (defun foo (n) + (dotimes (i n *undefined*))) + + Compiling results in this error message: + + ; in: DEFUN FOO + ; (DOTIMES (I N *UNDEFINED*)) + ; --> DO BLOCK LET TAGBODY RETURN-FROM + ; ==> + ; (PROGN *UNDEFINED*) + ; + ; caught WARNING: + ; undefined variable: *UNDEFINED* + + Note that DO appears in the processing path. This is because + DOTIMES expands into: + + (do ((i 0 (1+ i)) (#:g1 n)) + ((>= i #:g1) *undefined*) + (declare (type unsigned-byte i))) + + The rest of the processing path results from the expansion of DO: + + (block nil + (let ((i 0) (#:g1 n)) + (declare (type unsigned-byte i)) + (tagbody (go #:g3) + #:g2 (psetq i (1+ i)) + #:g3 (unless (>= i #:g1) (go #:g2)) + (return-from nil (progn *undefined*))))) + + In this example, the compiler descended into the BLOCK, LET, TAGBODY + and RETURN-FROM to reach the PROGN printed as the actual source. + This is a place where the \"actual source appears in explanation\" + rule was applied. The innermost actual source form was the symbol + _undefined_ itself, but that also appeared in the explanation, so + the compiler backed out one level.") + +(defsection @handling-of-types (:title "Handling of Types") + "One of the most important features of the SBCL compiler (similar to + the original CMUCL compiler, also known as _Python_) is its fairly + sophisticated understanding of the Common Lisp type system and its + conservative approach to the implementation of type declarations. + + These two features reward the use of type declarations throughout + development, even when high performance is not a concern. Also, as + discussed in the chapter on performance (see @EFFICIENCY), the use + of appropriate type declarations can be very important for + performance as well. + + The SBCL compiler also has a greater knowledge of the Common Lisp + type system than other compilers. Support is incomplete only for + types involving the SATISFIES type specifier." + (@declarations-as-assertions section) + (@precise-type-checking section) + (@getting-existing-programs-to-run section) + (@implementation-limitations section)) + +;; FIXME: See also sections \ref{advanced-type-stuff} and +;; \ref{type-inference}, once we snarf them from the CMU CL manual. +;; +;; Also see my paper on improving Baker, when I get round to it. +;; +;; Whose paper? + +(defsection @declarations-as-assertions (:title "Declarations as Assertions") + "The SBCL compiler treats type declarations differently from most other + Lisp compilers. Under default compilation policy the compiler doesn't + blindly believe type declarations, but considers them assertions about + the program that should be checked: all type declarations that have + not been proven to always hold are asserted at runtime. + + _Remaining bugs in the compiler's handling of types unfortunately + provide some exceptions to this rule, see + @IMPLEMENTATION-LIMITATIONS._ + + CLOS slot types form a notable exception. Types declared using the + :TYPE slot option in DEFCLASS are asserted if and only if the class + was defined in _safe code_ and the slot access location is in _safe + code_ as well. This laxness does not pose any internal consistency + issues, as the CLOS slot types are not available for the type + inferencer, nor do CLOS slot types provide any efficiency benefits. + + There are three type checking policies available in SBCL, selectable + via OPTIMIZE declarations." + ;; FIXME: This should be properly integrated with general policy + ;; stuff, once that gets cleaned up. + "- __Full Type Checks__ + + All declarations are considered assertions to be checked at + runtime, and all type checks are precise. The default + compilation policy provides full type checks. + + Used when `(OR (>= SAFETY 2) (>= SAFETY SPEED 1))`. + + - __Weak Type Checks__ + + Declared types may be simplified into faster to check + supertypes: for example, `(OR (INTEGER -17 -7) (INTEGER 7 17))` + is simplified into `(INTEGER -17 17)`. + + > __Warning__: It is relatively easy to corrupt the heap when + > weak type checks are used if the program contains type-errors. + + Used when `(AND (< SAFETY 2) (< SAFETY SPEED))`. + + - __No Type Checks__ + + All declarations are believed without assertions. Also disables + argument count and array bounds checking. + + > __Warning__: Any type errors in code where type checks are not + > performed are liable to corrupt the heap. + + Used when `(= SAFETY 0)`.") + +(defsection @precise-type-checking (:title "Precise Type Checking") + "Precise checking means that the check is done as though TYPEP + had been called with the exact type specifier that appeared in the + declaration. + + If a variable is declared to be `(INTEGER 3 17)`, then its value + must always be an integer between `3` and `17`. If multiple type + declarations apply to a single variable, then all the declarations + must be correct; it is as though all the types were intersected + producing a single AND type specifier. + + To gain maximum benefit from the compiler's type checking, you + should always declare the types of function arguments and structure + slots as precisely as possible. This often involves the use of OR, + MEMBER, and other list-style type specifiers.") + +(defsection @getting-existing-programs-to-run + (:title "Getting Existing Programs to Run") + "Since SBCL's compiler does much more comprehensive type checking than + most Lisp compilers, SBCL may detect type errors in programs that have + been debugged using other compilers. These errors are mostly incorrect + declarations, although compile-time type errors can find actual bugs + if parts of the program have never been tested. + + Some incorrect declarations can only be detected by run-time type + checking. It is very important to initially compile a program with + full type checks (high SAFETY optimization) and then test this safe + version. After the checking version has been tested, then you can + consider weakening or eliminating type checks. _This applies even to + previously debugged programs_ because the SBCL compiler does much + more type inference than other Common Lisp compilers, so an + incorrect declaration can do more damage. + + The most common problem is with variables whose constant initial + value doesn't match the type declaration. Incorrect constant initial + values will always be flagged by a compile-time type error, and they + are simple to fix once located. Consider this code fragment: + + (prog (foo) + (declare (fixnum foo)) + (setq foo ...) + ...) + + Here `FOO` is given an initial value of NIL but is declared to be a + FIXNUM. Even if it is never read, the initial value of a variable + must match the declared type. There are two ways to fix this + problem. Change the declaration + + (prog (foo) + (declare (type (or fixnum null) foo)) + (setq foo ...) + ...) + + or change the initial value + + (prog ((foo 0)) + (declare (fixnum foo)) + (setq foo ...) + ...) + + It is generally preferable to change to a legal initial value rather + than to weaken the declaration, but sometimes it is simpler to + weaken the declaration than to try to make an initial value of the + appropriate type. + + Another declaration problem occasionally encountered is incorrect + declarations on DEFMACRO arguments. This can happen when a function + is converted into a macro. Consider this macro: + + (defmacro my-1+ (x) + (declare (fixnum x)) + `(the fixnum (1+ ,x))) + + Although legal and well-defined Common Lisp code, this meaning of + this definition is almost certainly not what the writer intended. + For example, this call is illegal: + + (my-1+ (+ 4 5)) + + This call is illegal because the argument to the macro is `(+ 4 5)`, + which is a LIST, not a FIXNUM. Because of macro semantics, it is + hardly ever useful to declare the types of macro arguments. If you + really want to assert something about the type of the result of + evaluating a macro argument, then put a THE in the expansion: + + (defmacro my-1+ (x) + `(the fixnum (1+ (the fixnum ,x)))) + + + In this case, it would be stylistically preferable to change this + macro back to a function and declare it inline." + ;; FIXME: inline-expansion, once we crib the relevant text + ;; from the CMU CL manual. + "Some more subtle problems are caused by incorrect declarations that + can't be detected at compile time. Consider this code: + + (do ((pos 0 (position #\a string :start (1+ pos)))) + ((null pos)) + (declare (fixnum pos)) + ...) + + Although `POS` is almost always a FIXNUM, it is NIL at the end of + the loop. If this example is compiled with full type checks (the + default), then running it will signal a type error at the end of the + loop. If compiled without type checks, the program will go into an + infinite loop (or perhaps POSITION will complain because `(1+ NIL)` + isn't a sensible start.) Why? Because if you compile without type + checks, the compiler just quietly believes the type declaration. + Since the compiler believes that `POS` is always a FIXNUM, it + believes that `POS` is never NIL, so `(NULL POS)` is never true, and + the loop exit test is optimized away. Such errors are sometimes + flagged by unreachable code notes, but it is still important to + initially compile and test any system with full type checks, even if + the system works fine when compiled using other compilers. + + In this case, the fix is to weaken the type declaration to `(OR + FIXNUM NULL)`. (Actually, this declaration is unnecessary in SBCL, + since it already knows that POSITION returns a non-negative FIXNUM + or NIL.) + + Note that there is usually little performance penalty for weakening + a declaration in this way. Any numeric operations in the body can + still assume that the variable is a FIXNUM, since NIL is not a legal + numeric argument. Another possible fix would be to say: + + (do ((pos 0 (position #\a string :start (1+ pos)))) + ((null pos)) + (let ((pos pos)) + (declare (fixnum pos)) + ...)) + + This would be preferable in some circumstances, since it would allow + a non-standard representation to be used for the local `POS` + variable in the loop body." + ;; FIXME: ND-variables, once we crib the text from the CMU CL + ;; manual. + ) + +(defsection @implementation-limitations (:title "Implementation Limitations") + "If an FTYPE is placed after the function definition the function won't + perform any type checks, and the calls to the function will blindly + trust the declared types. + (OPTIMIZE (DEBUG 3)) will not trust any FTYPE declarations.") + +(defsection @compiler-policy (:title "Compiler Policy") + "Compiler policy is controlled by the OPTIMIZE declaration, + supporting all ANSI optimization qualities (DEBUG, safety, space, + and speed). (A deprecated extension SB-EXT:INHIBIT-WARNINGS is still + supported but liable to go away at any time.) + + For effects of various optimization qualities on type-safety and + debuggability see @DECLARATIONS-AS-ASSERTIONS and + @DEBUGGER-POLICY-CONTROL. + + Ordinarily, when the speed quality is high, the compiler emits notes + to notify the programmer about its inability to apply various + optimizations. For selective muffling of these notes, see + @CONTROLLING-VERBOSITY. + + The value of space mostly influences the compiler's decision whether + to inline operations, which tend to increase the size of programs. + Use the value `0` with caution, since it can cause the compiler to + inline operations so indiscriminately that the net effect is to slow + the program by causing cache misses or even swapping." + (sb-ext:describe-compiler-policy function) + (sb-ext:restrict-compiler-policy function) + (with-compilation-unit macro)) + +;; FIXME: old CMU CL compiler policy, should perhaps be adapted for +;; SBCL. (Unfortunately, the CMU CL docs are out of sync with the CMU +;; CL code, so adapting this requires not only reformatting the +;; documentation, but rooting out code rot.) +;; +;; Compiler Policy</1000 +;; INDEX {policy}{compiler} +;; INDEX compiler policy +;; +;; <para>The policy is what tells the compiler <emphasis>how</emphasis> to +;; compile a program. This is logically (and often textually) distinct +;; from the program itself. Broad control of policy is provided by the +;; <parameter>optimize</parameter> declaration; other declarations and variables +;; control more specific aspects of compilation. +;; +;; \begin{comment} +;; * The Optimize Declaration:: +;; * The Optimize-Interface Declaration:: +;; \end{comment} +;; +;; %%\node The Optimize Declaration, The Optimize-Interface Declaration, Compiler Policy, Compiler Policy +;; \subsection{The Optimize Declaration} +;; \label{optimize-declaration} +;; \cindex{optimize declaration} +;; \cpsubindex{declarations}{\code{optimize}} +;; +;; The \code{optimize} declaration recognizes six different +;; \var{qualities}. The qualities are conceptually independent aspects +;; of program performance. In reality, increasing one quality tends to +;; have adverse effects on other qualities. The compiler compares the +;; relative values of qualities when it needs to make a trade-off; i.e., +;; if \code{speed} is greater than \code{safety}, then improve speed at +;; the cost of safety. +;; +;; The default for all qualities (except \code{debug}) is \code{1}. +;; Whenever qualities are equal, ties are broken according to a broad +;; idea of what a good default environment is supposed to be. Generally +;; this downplays \code{speed}, \code{compile-speed} and \code{space} in +;; favor of \code{safety} and \code{debug}. Novice and casual users +;; should stick to the default policy. Advanced users often want to +;; improve speed and memory usage at the cost of safety and +;; debuggability. +;; +;; If the value for a quality is \code{0} or \code{3}, then it may have a +;; special interpretation. A value of \code{0} means ``totally +;; unimportant'', and a \code{3} means ``ultimately important.'' These +;; extreme optimization values enable ``heroic'' compilation strategies +;; that are not always desirable and sometimes self-defeating. +;; Specifying more than one quality as \code{3} is not desirable, since +;; it doesn't tell the compiler which quality is most important. +;; +;; +;; These are the optimization qualities: +;; \begin{Lentry} +;; +;; \item[\code{speed}] \cindex{speed optimization quality}How fast the +;; program should is run. \code{speed 3} enables some optimizations +;; that hurt debuggability. +;; +;; \item[\code{compilation-speed}] \cindex{compilation-speed optimization +;; quality}How fast the compiler should run. Note that increasing +;; this above \code{safety} weakens type checking. +;; +;; \item[\code{space}] \cindex{space optimization quality}How much space +;; the compiled code should take up. Inline expansion is mostly +;; inhibited when \code{space} is greater than \code{speed}. A value +;; of \code{0} enables indiscriminate inline expansion. Wide use of a +;; \code{0} value is not recommended, as it may waste so much space +;; that run time is slowed. \xlref{inline-expansion} for a discussion +;; of inline expansion. +;; +;; \item[\code{debug}] \cindex{debug optimization quality}How debuggable +;; the program should be. The quality is treated differently from the +;; other qualities: each value indicates a particular level of debugger +;; information; it is not compared with the other qualities. +;; \xlref{debugger-policy} for more details. +;; +;; \item[\code{safety}] \cindex{safety optimization quality}How much +;; error checking should be done. If \code{speed}, \code{space} or +;; \code{compilation-speed} is more important than \code{safety}, then +;; type checking is weakened (\pxlref{weakened-type-checks}). If +;; \code{safety} if \code{0}, then no run time error checking is done. +;; In addition to suppressing type checks, \code{0} also suppresses +;; argument count checking, unbound-symbol checking and array bounds +;; checks. +;; ... and checking of tag existence in RETURN-FROM and GO. +;; +;; \item[\code{extensions:inhibit-warnings}] \cindex{inhibit-warnings +;; optimization quality}This is a CMU extension that determines how +;; little (or how much) diagnostic output should be printed during +;; compilation. This quality is compared to other qualities to +;; determine whether to print style notes and warnings concerning those +;; qualities. If \code{speed} is greater than \code{inhibit-warnings}, +;; then notes about how to improve speed will be printed, etc. The +;; default value is \code{1}, so raising the value for any standard +;; quality above its default enables notes for that quality. If +;; \code{inhibit-warnings} is \code{3}, then all notes and most +;; non-serious warnings are inhibited. This is useful with +;; \code{declare} to suppress warnings about unavoidable problems. +;; \end{Lentry} +;; +;; %%\node The Optimize-Interface Declaration, , The Optimize Declaration, Compiler Policy +;; \subsection{The Optimize-Interface Declaration} +;; \label{optimize-interface-declaration} +;; \cindex{optimize-interface declaration} +;; \cpsubindex{declarations}{\code{optimize-interface}} +;; +;; The \code{extensions:optimize-interface} declaration is identical in +;; syntax to the \code{optimize} declaration, but it specifies the policy +;; used during compilation of code the compiler automatically generates +;; to check the number and type of arguments supplied to a function. It +;; is useful to specify this policy separately, since even thoroughly +;; debugged functions are vulnerable to being passed the wrong arguments. +;; The \code{optimize-interface} declaration can specify that arguments +;; should be checked even when the general \code{optimize} policy is +;; unsafe. +;; +;; Note that this argument checking is the checking of user-supplied +;; arguments to any functions defined within the scope of the +;; declaration, \code{not} the checking of arguments to \llisp{} +;; primitives that appear in those definitions. +;; +;; The idea behind this declaration is that it allows the definition of +;; functions that appear fully safe to other callers, but that do no +;; internal error checking. Of course, it is possible that arguments may +;; be invalid in ways other than having incorrect type. Functions +;; compiled unsafely must still protect themselves against things like +;; user-supplied array indices that are out of bounds and improper lists. +;; See also the \kwd{context-declarations} option to +;; \macref{with-compilation-unit}. +;; +;; (end of section on compiler policy) + +(defsection @compiler-errors (:title "Compiler Errors") + (@type-errors-at-compile-time section) + (@errors-during-macroexpansion section) + (@read-errors section)) + +(defsection @type-errors-at-compile-time (:title "Type Errors at Compile Time") + "If the compiler can prove at compile time that some portion of the + program cannot be executed without a type error, then it will give a + warning at compile time. + + It is possible that the offending code would never actually be + executed at run-time due to some higher level consistency constraint + unknown to the compiler, so a type warning doesn't always indicate an + incorrect program. + + For example, consider this code fragment: + + (defun raz (foo) + (let ((x (case foo + (:this 13) + (:that 9) + (:the-other 42)))) + (declare (fixnum x)) + (foo x))) + + Compilation produces this warning: + + ; in: DEFUN RAZ + ; (CASE FOO (:THIS 13) (:THAT 9) (:THE-OTHER 42)) + ; --> LET COND IF COND IF COND IF + ; ==> + ; (COND) + ; + ; caught WARNING: + ; This is not a FIXNUM: + ; NIL + + In this case, the warning means that if `FOO` isn't any of `:THIS`, + `:THAT` or `:THE-OTHER`, then `x` will be initialized to NIL, which + the FIXNUM declaration makes illegal. The warning will go away if + ECASE is used instead of CASE, or if `:THE-OTHER` is changed to T. + + This sort of spurious type warning happens moderately often in the + expansion of complex macros and in inline functions. In such cases, + there may be dead code that is impossible to correctly execute. The + compiler can't always prove this code is dead (could never be + executed), so it compiles the erroneous code (which will always signal + an error if it is executed) and gives a warning.") + +(defsection @errors-during-macroexpansion + (:title "Errors During Macroexpansion") + "The compiler handles errors that happen during macroexpansion, turning + them into compiler errors. If you want to debug the error (to debug + a macro), you can set *BREAK-ON-SIGNALS* to ERROR. For example, this + definition: + + (defun foo (e l) + (do ((current l (cdr current)) + ((atom current) nil)) + (when (eq (car current) e) (return current)))) + + gives this error: + + ; in: DEFUN FOO + ; (DO ((CURRENT L (CDR CURRENT)) + ; ((ATOM CURRENT) NIL)) + ; (WHEN (EQ (CAR CURRENT) E) (RETURN CURRENT))) + ; + ; caught ERROR: + ; (in macroexpansion of (DO # #)) + ; (hint: For more precise location, try *BREAK-ON-SIGNALS*.) + ; DO step variable is not a symbol: (ATOM CURRENT)") + +(defsection @read-errors (:title "Read Errors") + "SBCL's compiler does not attempt to recover from read errors when + reading a source file, but instead just reports the offending + character position and gives up on the entire source file.") + +(defsection @open-coding-and-inline-expansion + (:title "Open Coding and Inline Expansion") + "Since Common Lisp forbids the redefinition of standard functions, the + compiler can have special knowledge of these standard functions + embedded in it. This special knowledge is used in various ways (open + coding, inline expansion, source transformation), but the implications + to the user are basically the same: + + - Attempts to redefine standard functions may be frustrated, since + the function may never be called. Although it is technically + illegal to redefine standard functions, users sometimes want to + implicitly redefine these functions when they are debugging using + the TRACE macro. Special-casing of standard functions can be + inhibited using the NOTINLINE declaration, but even then some + phases of analysis such as type inferencing are applied by the + compiler. + + - The compiler can have multiple alternate implementations of + standard functions that implement different trade-offs of speed, + space and safety. This selection is based on the @COMPILER-POLICY. + + When a function call is _open coded_, inline code whose effect is + equivalent to the function call is substituted for that function + call. When a function call is _closed coded_, it is usually left as + is, although it might be turned into a call to a different function + with different arguments. As an example, if NTHCDR were to be open + coded, then + + (nthcdr 4 foobar) + + might turn into + + (cdr (cdr (cdr (cdr foobar)))) + + or even + + (do ((i 0 (1+ i)) + (list foobar (cdr foobar))) + ((= i 4) list)) + + If NTH is closed coded, then + + (nth x l) + + might stay the same, or turn into something like + + (car (nthcdr x l)) + + In general, open coding sacrifices space for speed, but some functions + (such as CAR) are so simple that they are always open-coded. Even + when not open-coded, a call to a standard function may be + transformed into a different function call (as in the last example) + or compiled as _static call_. Static function call uses a more + efficient calling convention that forbids redefinition.") + +(defsection @interpreter (:title "Interpreter") + "By default SBCL implements EVAL by calling the native code + compiler. + + SBCL also includes an interpreter for use in special cases where + using the compiler is undesirable, for example due to compilation + overhead. Unlike in some other Lisp implementations, in SBCL + interpreted code is not safer or more debuggable than compiled code." + (sb-ext:*evaluator-mode* variable)) + +(defsection @advanced-compiler-use-and-efficiency-hints + (:title "Advanced Compiler Use and Efficiency Hints") + "For more advanced usages of the compiler, please see the chapter of the + same name in the CMUCL manual. Many aspects of the compiler have stayed + exactly the same, and there is a much more detailed explanation of the + compiler's behavior and how to maximally optimize code in their + manual. In particular, while SBCL no longer supports byte-code + compilation, it does support CMUCL's block compilation facility allowing + whole program optimization and increased use of the local call + convention. + + Unlike CMUCL, SBCL is able to open-code forward-referenced type + tests while block compiling. This helps for mutually referential + DEFSTRUCTs in particular.") diff --git a/contrib/sb-manual/doc/contrib-modules.lisp b/contrib/sb-manual/doc/contrib-modules.lisp new file mode 100644 index 000000000..61b7d6317 --- /dev/null +++ b/contrib/sb-manual/doc/contrib-modules.lisp @@ -0,0 +1,20 @@ +(in-package :sb-manual) + +(defsection @contributed-modules (:title "Contributed Modules") + "SBCL comes with a number of modules that are not part of the core + system. These are loaded via `(REQUIRE :<MODULENAME>)` + (see @CUSTOMIZATION-HOOKS-FOR-USERS). This section contains + documentation (or pointers to documentation) for some of the + contributed modules." + (@sb-aclrepl section) + (@sb-concurrency section) + (@sb-cover section) + (@sb-grovel section) + (@sb-introspect section) + (@sb-manual section) + (@sb-md5 section) + (@sb-posix section) + (@sb-queue section) + (@sb-rotate-byte section) + (@sb-simd section)) + diff --git a/contrib/sb-manual/doc/debugger.lisp b/contrib/sb-manual/doc/debugger.lisp new file mode 100644 index 000000000..6c7fd85d1 --- /dev/null +++ b/contrib/sb-manual/doc/debugger.lisp @@ -0,0 +1,865 @@ +(in-package :sb-manual) + +(defsection @debugger (:title "Debugger") + "This chapter documents the debugging facilities of SBCL, including + the debugger, single-stepper and TRACE, and the effect of `(OPTIMIZE + DEBUG)` declarations." + (@debugger-entry section) + (@debugger-command-loop section) + (@stack-frames section) + (@variable-access section) + (@source-location-printing section) + (@debugger-policy-control section) + (@exiting-commands section) + (@information-commands section) + (@breakpoint-commands section) + (@function-tracing section) + (@single-stepping section) + (@enabling-and-disabling-the-debugger section)) + +(defsection @debugger-entry (:title "Debugger Entry") + (@debugger-banner section) + (@debugger-invocation section)) + +(defsection @debugger-banner (:title "Debugger Banner") + "When you enter the debugger, it looks something like this: + + debugger invoked on a TYPE-ERROR in thread 11184: + The value 3 is not of type LIST. + + You can type HELP for debugger help, or (SB-EXT:QUIT) to exit from SBCL. + + restarts (invokable by number or by possibly-abbreviated name): + 0: [ABORT ] Reduce debugger level (leaving debugger, returning to toplevel). + 1: [TOPLEVEL] Restart at toplevel READ/EVAL/PRINT loop. + (CAR 1 3) + 0] + + The first group of lines describe what the error was that put us in + the debugger. In this case CAR was called on `3`, causing a + TYPE-ERROR. + + This is followed by the \"beginner help line\", which appears only + if SB-DEBUG:*DEBUG-BEGINNER-HELP-P* is true (default). + + Next comes a listing of the active restart names, along with their + descriptions -- the ways we can restart execution after this error. + In this case, both options return to top-level. Restarts can be + selected by entering the corresponding number or name. + + The current frame appears right underneath the restarts, immediately + followed by the debugger prompt.") + +(defsection @debugger-invocation (:title "Debugger Invocation") + "The debugger is invoked when: + + - ERROR is called, and the condition it signals is not handled. + + - BREAK is called, or SIGNAL is called with a condition that matches + the current *BREAK-ON-SIGNALS*. + + - The debugger is explicitly entered with the INVOKE-DEBUGGER + function. + + When the debugger is invoked by a condition, ANSI mandates that the + value of *DEBUGGER-HOOK*, if any, be called with two arguments: the + condition that caused the debugger to be invoked and the previous + value of *DEBUGGER-HOOK*. When this happens, *DEBUGGER-HOOK* is + bound to NIL to prevent recursive errors. However, ANSI also + mandates that *DEBUGGER-HOOK* not be invoked when the debugger is to + be entered by the BREAK function. For users who wish to provide an + alternate debugger interface (and thus catch BREAK entries into the + debugger), SBCL provides SB-EXT:*INVOKE-DEBUGGER-HOOK*, which is + invoked during any entry into the debugger." + ;; When Swank is loaded, it sets this variable. + (sb-ext:*invoke-debugger-hook* (variable nil))) + +(defsection @debugger-command-loop (:title "Debugger Command Loop") + "The debugger is an interactive read-eval-print loop much like the + normal top level, but some symbols are interpreted as debugger + commands instead of being evaluated. A debugger command starts with + the symbol name of the command, possibly followed by some arguments + on the same line. Some commands prompt for additional input. + Debugger commands can be abbreviated by any unambiguous prefix: + `help` can be typed as `h`, `he`, etc. + + The package is not significant in debugger commands; any symbol with + the name of a debugger command will work. If you want to show the + value of a variable that happens also to be the name of a debugger + command you can wrap the variable in a PROGN to hide it from + the command loop. + + The debugger prompt is `<frame>]`, where `<frame>` is the number of + the current frame. Frames are numbered starting from zero at the + top (most recent call), increasing down to the bottom. The current + frame is the frame that commands refer to. + + It is possible to override the normal printing behaviour in the + debugger by using the SB-EXT:*DEBUG-PRINT-VARIABLE-ALIST*." + (sb-ext:*debug-print-variable-alist* variable)) + +(defsection @stack-frames (:title "Stack Frames") + "A _stack frame_ is the run-time representation of a call to a + function; the frame stores the state that a function needs to + remember what it is doing. Frames have: + + - _Variables_ (see @VARIABLE-ACCESS), which are the values being + operated on. + + - _Arguments_ to the call (which are really just particularly + interesting variables). + + - A current source location (@SOURCE-LOCATION-PRINTING), which is + the place in the program where the function was running when it + stopped to call another function, or because of an interrupt or + error." + (@stack-motion section) + (@how-arguments-are-printed section) + (@function-names section) + (@debug-tail-recursion section) + (@unknown-locations-and-interrupts section)) + +(defsection @stack-motion (:title "Stack Motion") + "These commands move to a new stack frame and print the name of the + function and the values of its arguments in the style of a Lisp + function call: + + - `up`: Move up to the next higher frame. More recent function calls + are considered to be higher on the stack. + + - `down`: Move down to the next lower frame. + + - `top`: Move to the highest frame, that is, the frame where the + debugger was entered. + + - `bottom`: Move to the lowest frame. + + - `frame [<n>]`: Move to the frame with the specified number. + Prompts for the number if not supplied. The frame with number 0 is + the frame where the debugger was entered.") + +(defsection @how-arguments-are-printed (:title "How Arguments are Printed") + "A frame is printed to look like a function call, but with the actual + argument values in the argument positions. So the frame for this call + in the source: + + (myfun (+ 3 4) 'a) + + would look like this: + + (MYFUN 7 A) + + All keyword and optional arguments are displayed with their actual + values; if the corresponding argument was not supplied, the value will + be the default. So this call: + + (subseq \"foo\" 1) + + would look like this: + + (SUBSEQ \"foo\" 1 3) + + And this call: + + (string-upcase \"test case\") + + would look like this: + + (STRING-UPCASE \"test case\" :START 0 :END NIL) + + The arguments to a function call are displayed by accessing the + argument variables. Although those variables are initialized to the + actual argument values, they can be set inside the function; in this + case the new value will be displayed. + + &REST arguments are handled somewhat differently. The value of the + rest argument variable is displayed as the spread-out arguments to + the call, so: + + (format t \"~A is a ~A.\" \"This\" 'test) + + would look like this: + + (FORMAT T \"~A is a ~A.\" \"This\" 'TEST) + + Rest arguments cause an exception to the normal display of keyword + arguments in functions that have both &REST and &KEY arguments. In + this case, the keyword argument variables are not displayed at all; + the rest arg is displayed instead. So for these functions, only the + keywords actually supplied will be shown, and the values displayed + will be the argument values, not values of the + (possibly modified) variables. + + If the variable for an argument is never referenced by the function, + it will be deleted. The variable value is then unavailable, so the + debugger prints `#<unused-arg>` instead of the value. Similarly, if + for any of a number of reasons the value of the variable is + unavailable or not known to be available (@VARIABLE-ACCESS), then + `#<unavailable-arg>` will be printed instead of the argument value. + + Note that inline expansion and open-coding affect what frames are + present in the debugger, see @DEBUGGER-POLICY-CONTROL." + ;; FIXME: Link here to section about open coding once it exists. + ) + +(defsection @function-names (:title "Function Names") + "If a function is defined by DEFUN it will appear in backtrace + by that name. Functions defined by LABELS and FLET will appear as + `(FLET <NAME>)` and `(LABELS <NAME>)` respectively. Anonymous + lambdas will appear as `(LAMBDA <LAMBDA-LIST>)`." + (@entry-point-details section)) + +(defsection @entry-point-details (:title "Entry Point Details") + "Sometimes the compiler introduces new functions that are used to + implement a user function, but are not directly specified in the + source. This is mostly done for argument type and count checking. + + With recursive or block compiled functions, an additional `external` + frame may appear before the frame representing the first call to the + recursive function or entry to the compiled block. This is a + consequence of the way the compiler works: there is nothing odd with + your program. You may also see `cleanup` frames during the execution + of UNWIND-PROTECT cleanup code, and `optional` for variable argument + entry points.") + +(defsection @debug-tail-recursion (:title "Debug Tail Recursion") + "The compiler is _properly tail recursive_. If a function call is + in a tail-recursive position, the stack frame will be deallocated + _at the time of the call_, rather than after the call returns. + Consider this backtrace: + + (BAR ...) + (FOO ...) + + Because of tail recursion, it is not necessarily the case that `FOO` + directly called `BAR`. It may be that `FOO` called some other + function `FOO2`, which then called `BAR` tail-recursively, as in + this example: + + (defun foo () + ... + (foo2 ...) + ...) + + (defun foo2 (...) + ... + (bar ...)) + + (defun bar (...) + ...) + + Usually the elimination of tail-recursive frames makes debugging + more pleasant, since these frames are mostly uninformative. If there + is any doubt about how one function called another, it can usually + be eliminated by finding the source location in the calling frame. + See @SOURCE-LOCATION-PRINTING. + + The elimination of tail-recursive frames can be prevented by + disabling tail-recursion optimization, which happens when the DEBUG + optimization quality is greater than 2. See + @DEBUGGER-POLICY-CONTROL." + ;; FIXME: reinstate this link once the chapter is in the manual. For + ;; a more thorough discussion of tail recursion, see @TAIL-RECURSION. + ) + +(defsection @unknown-locations-and-interrupts + (:title "Unknown Locations and Interrupts") + "The debugger operates using special debugging information attached to + the compiled code. This debug information tells the debugger what it + needs to know about the locations in the code where the debugger can + be invoked. If the debugger somehow encounters a location not + described in the debug information, then it is said to be _unknown_. + If the code location for a frame is unknown, then some variables may + be inaccessible, and the source location cannot be precisely + displayed. + + There are three reasons why a code location could be unknown: + + - There is inadequate debug information due to the value of the + DEBUG optimization quality. See @DEBUGGER-POLICY-CONTROL. + + - The debugger was entered because of an interrupt such as `C-c`. + + - A hardware error such as a bus error occurred in code that was + compiled unsafely due to the value of the SAFETY + optimization quality." + ;; FIXME: reinstate link when section on optimize qualities exists. + ;; @OPTIMIZE-DECLARATION. + "In the last two cases, the values of argument variables are + accessible, but may be incorrect. For more details on when variable + values are accessible, see @VARIABLE-VALUE-AVAILABILITY. + + It is possible for an interrupt to happen when a function call or + return is in progress. The debugger may then flame out with some + obscure error or insist that the bottom of the stack has been + reached, when the real problem is that the current stack frame can't + be located. If this happens, return from the interrupt and try + again.") + +(defsection @variable-access (:title "Variable Access") + "There are two ways to access the current frame's local variables in + the debugger: `list-locals` and SB-DEBUG:VAR. + + The debugger doesn't really understand lexical scoping; it has just + one namespace for all the variables in the current stack frame. If a + symbol is the name of multiple variables in the same function, then + the reference appears ambiguous, even though lexical scoping + specifies which value is visible at any given source location. If + the scopes of the two variables are not nested, then the debugger + can resolve the ambiguity by observing that only one variable is + accessible. + + When there are ambiguous variables, the evaluator assigns each one a + small integer identifier. The SB-DEBUG:VAR function uses this + identifier to distinguish between ambiguous variables. The + `list-locals` command prints the identifier. In the following + example, there are two variables named `X`. The first one has + identifier 0 (which is not printed), the second one has identifier + 1. + + X = 1 + X#1 = 2 + + - `list-locals [<prefix>]`: This command prints the name and value + of all variables in the current frame whose name has the specified + `<prefix>`, which may be a string or a symbol. If no `<prefix>` is + given, then all available variables are printed. If a variable has + a potentially ambiguous name, then the name is printed with a + `#<identifier>` suffix, where `<identifier>` is the small integer + used to make the name unique." + (sb-debug:var function) + (@variable-value-availability section) + (@note-on-lexical-variable-access section)) + +(defsection @variable-value-availability (:title "Variable Value Availability") + "The value of a variable may be unavailable to the debugger in portions + of the program where Lisp says that the variable is defined. If a + variable value is not available, the debugger will not let you read + or write that variable. With one exception, the debugger will never + display an incorrect value for a variable. Rather than displaying + incorrect values, the debugger tells you the value is unavailable. + + The one exception is this: if you interrupt (e.g. with `C-c`) or if + there is an unexpected hardware error such as a bus error (which + should only happen in unsafe code), then the values displayed for + arguments to the interrupted frame might be incorrect. This + exception applies only to the interrupted frame: any frame farther + down the stack will be fine. + + > _Note_: Since the location of an interrupt or hardware error will + > always be an unknown location, non-argument variable values will + > never be available in the interrupted frame. See + > @UNKNOWN-LOCATIONS-AND-INTERRUPTS.) + + The value of a variable may be unavailable for these reasons: + + - The value of the DEBUG optimization quality may have omitted debug + information needed to determine whether the variable is available. + Unless a variable is an argument, its value will only be available + when DEBUG is at least 2. + + - The compiler did lifetime analysis and determined that the value + was no longer needed, even though its scope had not been exited. + Lifetime analysis is inhibited when the DEBUG optimization + quality is 3. + + - The variable's name is an uninterned symbol (gensym). To save + space, the compiler only dumps debug information about uninterned + variables when the DEBUG optimization quality is 3. + + - The frame's location is unknown (see + @UNKNOWN-LOCATIONS-AND-INTERRUPTS) because the debugger was + entered due to an interrupt or unexpected hardware error. Under + these conditions the values of arguments will be available, but + might be incorrect. This is the exception mentioned above. + + - The variable (or the code referencing it) was optimized out of + existence. Variables with no reads are always optimized away. The + degree to which the compiler deletes variables will depend on the + value of the COMPILATION-SPEED optimization quality, but most + source-level optimizations are done under all compilation + policies. + + - The variable is never set and its definition looks like + + (LET ((var1 var2)) + ...) + + In this case, `VAR1` is substituted with `VAR2`. + + - The variable is never set and is referenced exactly once. In this + case, the reference is substituted with the variable initial + value. + + Since it is especially useful to be able to get the arguments to a + function, argument variables are treated specially when the SPEED + optimization quality is less than 3 and the DEBUG quality is at + least 1. With this compilation policy, the values of argument + variables are almost always available everywhere in the function, + even at unknown locations. For non-argument variables, DEBUG must be + at least 2 for values to be available, and even then, values are + only available at known locations.") + +(defsection @note-on-lexical-variable-access + (:title "Note On Lexical Variable Access") + "When the debugger command loop establishes variable bindings for + available variables, these variable bindings have lexical scope and + dynamic extent. You can close over them, but such closures can't be + used as upward function arguments. + + > _Note_: The variable bindings are actually created using the Lisp + > SYMBOL-MACROLET special form. + + You can also set local variables using SETQ, but if the variable was + closed over in the original source and never set, then setting the + variable in the debugger may not change the value in all the + functions the variable is defined in. Another risk of setting + variables is that you may assign a value of a type that the compiler + proved the variable could never take on. This may result in bad + things happening.") + +(defsection @source-location-printing (:title "Source Location Printing") + "One of the debugger's capabilities is source level debugging of + compiled code. These commands display the source location for the + current frame: + + - `source [<context>]`: This command displays the file that the + current frame's function was defined from (if it was defined from + a file), and then the source form responsible for generating the + code that the current frame was executing. If `<context>` is + specified, then it is an integer specifying the number of + enclosing levels of list structure to print. + + The source form for a location in the code is the innermost list + present in the original source that encloses the form responsible + for generating that code. If the actual source form is not a list, + then some enclosing list will be printed. For example, if the source + form was a reference to the variable `*SOME-RANDOM-SPECIAL*`, then + the innermost enclosing evaluated form will be printed. Here are + some possible enclosing forms: + + (let ((a *some-random-special*)) + ...) + + (+ *some-random-special* ...) + + If the code at a location was generated from the expansion of a + macro or a source-level compiler optimization, then the form in the + original source that expanded into that code will be printed. + Suppose the file `/usr/me/mystuff.lisp` looked like this: + + (defmacro mymac () + '(myfun)) + + (defun foo () + (mymac) + ...) + + If `FOO` has called `MYFUN`, and is waiting for it to return, then + the `source` command would print: + + ; File: /usr/me/mystuff.lisp + + (MYMAC) + + Note that the macro use was printed, not the actual function call form, + `(MYFUN)`. + + If enclosing source is printed by giving an argument to `source` or + `vsource`, then the actual source form is marked by wrapping it in a + list whose first element is `#:***HERE***`. In the previous example, + `source 1` would print: + + ; File: /usr/me/mystuff.lisp + + (DEFUN FOO () + (#:***HERE*** + (MYMAC)) + ...)" + (@how-the-source-is-found section) + (@source-location-availability section)) + +(defsection @how-the-source-is-found (:title "How the Source is Found") + "If the code was defined from Lisp by COMPILE or EVAL, then the source + can always be reliably located. If the code was defined from a FASL + file created by COMPILE-FILE, then the debugger gets the source + forms it prints by reading them from the original source file. This + is a potential problem, since the source file might have moved or + changed since the time it was compiled. + + The source file is opened using the TRUENAME of the source file + pathname originally given to the compiler. This is an absolute + pathname with all logical names and symbolic links expanded. If the + file can't be located using this name, then the debugger gives up + and signals an error. + + If the source file can be found, but has been modified since the time it was + compiled, the debugger prints this warning: + + ; File has been modified since compilation: + ; <filename> + ; Using form offset instead of character position. + + where `<filename>` is the name of the source file. It then proceeds + using a robust but not foolproof heuristic for locating the source. + This heuristic works if: + + - No top-level forms before the top-level form containing the source + have been added or deleted, and + + - the top-level form containing the source has not been modified + much. (More precisely, none of the list forms beginning before the + source form have been added or deleted.) + + If the heuristic doesn't work, the displayed source will be wrong, + but will probably be near the actual source. If the \"shape\" of the + top-level form in the source file is too different from the original + form, then an error will be signaled. When the heuristic is used, + the source location commands are noticeably slowed. + + Source location printing can also be confused if (after the source + was compiled) a read-macro you used in the code was redefined to + expand into something different, or if a read-macro ever returns the + same EQ list twice. If you don't define read macros and don't use + `##` in perverted ways, you don't need to worry about this.") + +(defsection @source-location-availability + (:title "Source Location Availability") + "Source location information is only available when the DEBUG + optimization quality is at least 2. If source location information + is unavailable, the source commands will give an error message. + + If source location information is available, but the source location + is unknown because of an interrupt or unexpected hardware error + (see @UNKNOWN-LOCATIONS-AND-INTERRUPTS), then the command will + print + + Unknown location: using block start. + + and then proceed to print the source location for the start of the + _basic block_ enclosing the code location. It's a bit complicated to + explain exactly what a basic block is, but here are some properties + of the block start location: + + - The block start location may be the same as the true location. + + - The block start location will never be later in the program's flow + of control than the true location. + + - No conditional control structures (such as IF, COND, OR) will + intervene between the block start and the true location (but note + that some conditionals present in the original source could be + optimized away.) Function calls _do not_ end basic blocks. + + - The head of a loop will be the start of a block. + + - The programming language concept of block structure and the Lisp + BLOCK special form are totally unrelated to the compiler's basic + block. + + In other words, the true location lies between the printed location + and the next conditional (but watch out because the compiler may + have changed the program on you.)") + +(defsection @debugger-policy-control (:title "Debugger Policy Control") + "The compilation policy specified by OPTIMIZE declarations + affects the behavior seen in the debugger. The DEBUG quality + directly affects the debugger by controlling the amount of debugger + information dumped. Other optimization qualities have indirect but + observable effects due to changes in the way compilation is done. + + Unlike the other optimization qualities (which are compared in + relative value to evaluate tradeoffs), the DEBUG optimization + quality is directly translated to a level of debug information. This + absolute interpretation allows the user to count on a particular + amount of debug information being available even when the values of + the other qualities are changed during compilation. These are the + levels of debug information that correspond to the values of the + DEBUG quality: + + - `0`: Only the function name and enough information to allow the + stack to be parsed. + + - `> 0`: Any level greater than 0 gives level 0 plus all argument + variables. Values will only be accessible if the argument variable + is never set and SPEED is not 3. SBCL allows any real value for + optimization qualities. It may be useful to specify 0.5 to get + backtrace argument display without argument documentation. + + - `1`: Level 1 provides argument documentation (printed argument + lists) and derived argument/result type information. This makes + DESCRIBE more informative, and allows the compiler to do + compile-time argument count and type checking for any calls + compiled at run-time. This is the default. + + - `2`: Level 1 plus all interned local variables, source location + information, and lifetime information that tells the debugger when + arguments are available (even when SPEED is 3 or the argument is + set). + + - `> 2`: Any level greater than 2 gives level 2 and in addition + disables tail-call optimization, so that the backtrace will + contain frames for all invoked functions, even those in tail + positions. + + - `3`: Level 2 plus all uninterned variables. In addition, lifetime + analysis is disabled (even when SPEED is 3), ensuring that all + variable values are available at any known location within the + scope of the binding. This has a speed penalty in addition to the + obvious space penalty. + + Inlining of local functions is inhibited so that they may be TRACEd. + + - `> (MAX SPEED SPACE)`: If DEBUG is greater than both SPEED and + SPACE, the command `return` can be used to continue execution by + returning a value from the current stack frame. + + - `> (MAX SPEED SPACE COMPILATION-SPEED)`: If DEBUG is greater than + all of SPEED, SPACE and COMPILATION-SPEED the code will be + steppable (see @SINGLE-STEPPING). + + As you can see, if the SPEED quality is 3, debugger performance is + degraded. This effect comes from the elimination of argument + variable special-casing (see @VARIABLE-VALUE-AVAILABILITY). Some + degree of speed/debuggability tradeoff is unavoidable, but the + effect is not too drastic when DEBUG is at least 2. + + In addition to INLINE and NOTINLINE declarations, the relative + values of the SPEED and SPACE qualities also change whether + functions are inline expanded. If a function is inline expanded, + then there will be no frame to represent the call, and the arguments + will be treated like any other local variable. Functions may also be + _semi-inline_, in which case there is a frame to represent the call, + but the call is to an optimized local version of the function, not + to the original function." + ;; FIXME: link to section about inline expansion when it exists + ;; (@INLINE-EXPANSION). + ) + +(defsection @exiting-commands (:title "Exiting Commands") + "These commands get you out of the debugger. + + - `toplevel`: Throw to top level. + + - `restart [<n>]`: Invoke the `<n>`th restart case as displayed by + the `error` command. If `<n>` is not specified, the available + restart cases are reported. + + - `\\continue`: Call CONTINUE on the condition given to DEBUG. If + there is no restart case named CONTINUE, then an error is + signaled. + + - `\\abort`: Call ABORT on the condition given to DEBUG. This is + useful for popping debug command loop levels or aborting to top + level, as the case may be. + + - `return <value>`: Return `VALUE` from the current stack frame. + This command is available when the DEBUG optimization quality is + greater than both SPEED and SPACE. Care must be taken that the + value is of the same type as SBCL expects the stack frame to + return. + + - `restart-frame`: Restart execution of the current stack frame. + This command is available when the DEBUG optimization quality is + greater than both SPEED and SPACE and when the frame is for a + global function. If the function is redefined in the debugger + before the frame is restarted, the new function will be used.") + +(defsection @information-commands (:title "Information Commands") + "Most of these commands print information about the current frame or + function, but a few show general information. + + - `help` or `?`: Display a synopsis of debugger commands. + + - `\\describe`: Call DESCRIBE on the current function and displays the + number of local variables. + + - `\\print`: Display the current function call as it would be + displayed by moving to this frame. + + - `\\error`: Print the condition given to INVOKE-DEBUGGER and the + active proceed cases. + + - `backtrace [<n>]`: Display all the frames from the current to the + bottom. Only shows `<n>` frames if specified. The printing is + controlled by SB-DEBUG:*DEBUG-PRINT-VARIABLE-ALIST*.") + +(defsection @breakpoint-commands (:title "Breakpoint Commands") + "SBCL supports setting of breakpoints inside compiled functions and + stepping of compiled code. Breakpoints can only be set at known + locations (see @UNKNOWN-LOCATIONS-AND-INTERRUPTS), so these commands + are largely useless unless the DEBUG optimize quality is at least + 2 (see @DEBUGGER-POLICY-CONTROL). These commands manipulate + breakpoints: + + - `breakpoint <location> [<option> <value>]*`: Set a breakpoint in + some function. `<location>` may be an integer code location + number (as displayed by `list-locations`) or a keyword. The + keyword can be used to indicate setting a breakpoint at the + function start (:START, `:S`) or function end (:END, `:E`). The + `breakpoint` command has :CONDITION, :BREAK, :PRINT and :FUNCTION + options which work similarly to the TRACE options. + + - `list-locations [<function>]` or `ll [<function>]`: List all the + code locations in the current frame's function, or in `<function>` + if it is supplied. The display format is the code location number, + a colon and then the source form for that location: + + 3: (1- N) + + If consecutive locations have the same source, then a numeric + range like `3-5:` will be printed. For example, a default + function call has a known location both immediately before and + after the call, which would result in two code locations with + the same source. The listed function becomes the new default + function for breakpoint setting (via the `breakpoint`) command. + + - `list-breakpoints` or `lb`: List all currently active breakpoints + with their breakpoint number. + + - `delete-breakpoint [<number>]` or `db [<number>]`: Delete a + breakpoint specified by its breakpoint number. If no number is + specified, delete all breakpoints. + + - `step*`: Step to the next possible breakpoint location in the + current function. This always steps over function calls, instead + of stepping into them." + (@breakpoint-example section)) + +(defsection @breakpoint-example (:title "Breakpoint Example") + "Consider this definition of the factorial function: + + (defun ! (n) + (if (zerop n) + 1 + (* n (! (1- n))))) + + This debugger session demonstrates the use of breakpoints: + + * (break) ; invoke debugger + + debugger invoked on a SIMPLE-CONDITION in thread 11184: break + + restarts (invokable by number or by possibly-abbreviated name): + 0: [CONTINUE] Return from BREAK. + 1: [ABORT ] Reduce debugger level (leaving debugger, returning to toplevel). + 2: [TOPLEVEL] Restart at toplevel READ/EVAL/PRINT loop. + (\"varargs entry for top level local call BREAK\" \"break\") + 0] ll #'! + + 0-1: (SB-INT:NAMED-LAMBDA ! (N) (BLOCK ! (IF (ZEROP N) 1 (* N (! #))))) + 2: (BLOCK ! (IF (ZEROP N) 1 (* N (! (1- N))))) + 3: (ZEROP N) + 4: (* N (! (1- N))) + 5: (1- N) + 6: (! (1- N)) + 7-8: (* N (! (1- N))) + 9-10: (IF (ZEROP N) 1 (* N (! (1- N)))) + 0] br 4 + + (* N (! (1- N))) + 1: 4 in ! + added + 0] toplevel + + > (! 10) ; Call the function + + *Breakpoint hit* + + Restarts: + 0: [CONTINUE] Return from BREAK. + 1: [ABORT ] Return to Top-Level. + + Debug (type H for help) + + (! 10) ; We are now in first call (arg 10) before the multiply + Source: (* N (! (1- N))) + 3] step* + + *Step* + + (! 10) ; We have finished evaluation of (1- n) + Source: (1- N) + 3] step* + + *Breakpoint hit* + + Restarts: + 0: [CONTINUE] Return from BREAK. + 1: [ABORT ] Return to Top-Level. + + Debug (type H for help) + + (! 9) ; We hit the breakpoint in the recursive call + Source: (* N (! (1- N))) + 3] + + > _Note_: The `step*` command differs from the single stepping + > commands in that it also functions in compiled code which has not + > been compiled with stepping instrumentation. It simply steps to + > the next compiled code location. In the future, this form of + > stepping may be improved enough to subsume the instrumentation + > based stepping commands, which have much higher overhead.") + +(defsection @function-tracing (:title "Function Tracing") + "The tracer causes selected functions to print their arguments and + their results whenever they are called. Options allow conditional + printing of the trace information and conditional breakpoints on + function entry or exit. + + In SBCL, tracing can be done either by temporarily redefining the + function name (encapsulation), or using breakpoints. When + breakpoints are used, the function object itself is destructively + modified to cause the tracing action. The advantage of using + breakpoints is that tracing works even when the function is + anonymously called via FUNCALL, that function object identity is + preserved, and that anonymous and local functions can also be + traced." + (trace macro) + "In the case of functions where the known return convention is used + to optimize, encapsulation may be necessary in order to make tracing + work at all. The symptom of this occurring is an error stating + + Error in function FOO: :FUNCTION-END breakpoints are + currently unsupported for the known return convention. + + in such cases we recommend using `(TRACE FOO :ENCAPSULATE t)`." + (untrace macro) + (sb-debug:*trace-indentation-step* variable) + (sb-debug:*max-trace-indentation* variable) + (sb-debug:*trace-encapsulate-default* variable) + (sb-debug:*trace-report-default* variable)) + +(defsection @single-stepping (:title "Single Stepping") + "SBCL includes an instrumentation based single-stepper for compiled + code, that can be invoked via the STEP macro, or from within the + debugger. See @DEBUGGER-POLICY-CONTROL, for details on enabling + stepping for compiled code. + + The following debugger commands are used for controlling single stepping. + + - `start`: Select the CONTINUE restart if one exists and starts + single stepping. None of the other single stepping commands can be + used before stepping has been started either by using `start` or + by using the standard STEP macro. + + - `step`: Step into the current form. Stepping will be resumed when + the next form that has been compiled with stepper instrumentation + is evaluated. + + - `next`: Step over the current form. Stepping will be disabled + until evaluation of the form is complete. + + - `out`: Step out of the current frame. Stepping will be disabled + until the topmost stack frame that had been stepped into returns. + + - `stop`: Stop the single stepper and resumes normal execution." + (step macro)) + +(defsection @enabling-and-disabling-the-debugger + (:title "Enabling and Disabling the Debugger") + "In certain contexts (e.g. non-interactive applications), it may be + desirable to turn off the SBCL debugger (and possibly re-enable it). + The functions here control the debugger." + (sb-ext:disable-debugger function) + (sb-ext:enable-debugger function)) diff --git a/contrib/sb-manual/doc/deprecation.lisp b/contrib/sb-manual/doc/deprecation.lisp new file mode 100644 index 000000000..9e79ecc52 --- /dev/null +++ b/contrib/sb-manual/doc/deprecation.lisp @@ -0,0 +1,434 @@ +(in-package :sb-manual) + +(defsection @deprecation (:title "Deprecation") + "In order to support evolution of interfaces in SBCL as well as in user + code, SBCL allows declaring functions, variables and types as + deprecated. Users of deprecated things are notified by means of + warnings while the deprecated thing in question is still available. + + This chapter documents the interfaces for being notified when using + deprecated thing and declaring things as deprecated, the deprecation + process used for SBCL interfaces, and lists legacy interfaces in + various stages of deprecation. + + _Deprecation_ in this context should not be confused with those + things the ANSI Common Lisp standard calls _deprecated_: the + entirety of ANSI CL is supported by SBCL, and none of those + interfaces are subject to censure." + (@why-deprecate? section) + (@the-deprecation-pipeline section) + (@deprecation-conditions section) + (@introspecting-deprecation-information section) + (@deprecation-declaration section) + (@deprecation-examples section) + (@deprecated-interfaces-in-sbcl section)) + +(defsection @why-deprecate? (:title "Why Deprecate?") + "While generally speaking we try to keep SBCL changes as backwards + compatible as feasible, there are situations when existing interfaces + are deprecated: + + - __Broken Interfaces__ + + Sometimes it turns out that an interface is sufficiently + misdesigned that fixing it would be worse than deprecating it + and replacing it with another. + + This is typically the case when fixing the interface would + change its semantics in ways that could break user code subtly: + in such cases we may end up considering the obvious breakage + caused by deprecation to be preferable. + + Another example are functions or macros whose current signature + makes them hard or impossible to extend in the future: backwards + compatible extensions would either make the interface + intolerably hairy, or are sometimes outright impossible. + + - __Internal Interfaces__ + + SBCL has several internal interfaces that were never meant to be + used in user code -- or at least never meant to be used in user + code unwilling to track changes to SBCL internals. + + Ideally, we'd like to be free to refactor our own internals as + we please, without even going through the hassle of deprecating + things. Sometimes, however, it turns out that our internal + interfaces have several external users who aren't using them + advisedly, but due to misunderstandings regarding their status + or stability. + + Consider a deprecated internal interface a reminder for SBCL + maintainers not to delete the thing just yet, even though it is + seems unused -- because it has external users. + + When internal interfaces are deprecated we try our best to + provide supported alternatives. + + - __Aesthetics & Ease of Maintenance__ + + Sometimes an interface isn't broken or internal but just + inconsistent somehow. + + This mostly happens only with historical interfaces inherited + from CMUCL which often haven't been officially supported in SBCL + before, or with new extensions to SBCL that haven't been around + for very long in the first place. + + The alternative would be to keep the suboptimal version around + forever, possibly alongside an improved version. Sometimes we + may do just that, but because every line of code comes with a + maintenance cost, sometimes we opt to deprecate the suboptimal + version instead: SBCL doesn't have infinite developer resources. + + We also believe that sometimes cleaning out legacy interfaces + helps keep the whole system more comprehensible to users, and + makes introspective tools such as APROPOS more useful.") + +(defsection @the-deprecation-pipeline (:title "The Deprecation Pipeline") + "SBCL uses a _deprecation pipeline_ with multiplestages: as + time time goes by, deprecated things move from earlier stages of + deprecation to later stages before finally being removed. The + intention is making users aware of necessary changes early but + allowing a migration to new interfaces at a reasonable pace. + + Deprecation proceeds in three stages, each lasting approximately a + year. In some cases it might move slower or faster, but one year per + stage is what we aim at in general. During each stage warnings (and + errors) of increasing severity are signaled, which note that the + interface is deprecated, and point users towards any replacements + when applicable. + + - __Early Deprecation__ + + During early deprecation the interface is kept in working + condition. However, when a thing in this deprecation stage is + used, an SB-EXT:EARLY-DEPRECATION-WARNING, which is a + STYLE-WARNING, is signaled at compile-time. + + The internals may change at this stage: typically because the + interface is re-implemented on top of its successor. While we + try to keep things as backwards-compatible as feasible (taking + maintenance costs into account), sometimes semantics change + slightly. + + For example, when the spinlock API was deprecated, spinlock + objects ceased to exist, and the whole spinlock API became a + synonym for the mutex API -- so code using the spinlock API + continued working but silently switched to mutexes instead. + However, if someone relied on + + (typep lock 'spinlock) + + returning NIL for a mutexes, trouble could ensue. + + - __Late Deprecation__ + + During late deprecation the interface remains as it was during + early deprecation, but the compile-time warning is upgraded: + when a thing in this deprecation stage is used, a + SB-EXT:LATE-DEPRECATION-WARNING, which is a full WARNING, is + signaled at compile-time. + + - __Final Deprecation__ + + During final deprecation the symbols still exist. However, when + a thing in this deprecation stage is used, a + SB-EXT:FINAL-DEPRECATION-WARNING, which is a full WARNING, is + signaled at compile-time and an ERROR is signaled at run-time. + + - __After Final Deprecation__ + + The interface is deleted entirely.") + +(defsection @deprecation-conditions (:title "Deprecation Conditions") + "SB-EXT:DEPRECATION-CONDITION is the superclass of all + deprecation-related warning and error conditions. All common slots and + readers are defined in this condition class." + (sb-ext:deprecation-condition condition) + (sb-ext:early-deprecation-warning condition) + (sb-ext:late-deprecation-warning condition) + (sb-ext:final-deprecation-warning condition) + (sb-ext:deprecation-error condition)) + +(defsection @introspecting-deprecation-information + (:title "Introspecting Deprecation Information") + "The deprecation status of functions and variables can be inspected + using the SB-CLTL2:FUNCTION-INFORMATION and + SB-CLTL2:VARIABLE-INFORMATION functions provided by the `SB-CLTL2` + contributed module.") + +(defsection @deprecation-declaration (:title "Deprecation Declaration") + "The SB-EXT:DEPRECATED declaration can be used to declare objects + in various namespaces as deprecated. + + > _Note_: See the `namespace` CLHS glossary entry in the glossary of + > the Common Lisp Hyperspec.) + + - [__declaration__] SB-EXT:DEPRECATED + + Syntax: `(SB-EXT:DEPRECATED STAGE SINCE &REST OBJECT-CLAUSES)` + + stage ::= {:EARLY | :LATE | :FINAL} + + since ::= {`<version>` | (`<software>` `<version>`)} + + object-clause ::= (namespace `<name>` [:REPLACEMENT `<replacement>`]) + + namespace ::= {CL:VARIABLE | CL:FUNCTION | CL:TYPE} + + where the terminal `<name>` is the name of the deprecated thing, + `<version>` and `<software>` are strings describing the version + in which the thing has been deprecated and `<replacement>` is a + name or a list of names designating things that should be used + instead of the deprecated thing. + + Currently the following namespaces are supported: + + - CL:FUNCTION: Declare functions, compiler-macros or macros as + deprecated. + + When declaring a function to be in :FINAL deprecation, there + should be no actual definition of the function as the + declaration emits a stub function that signals a + SB-EXT:DEPRECATION-ERROR at run-time when called. + + - CL:VARIABLE: Declare special and global variables, constants + and symbol-macros as deprecated. + + When declaring a variable to be in :FINAL deprecation, there + should be no actual definition of the variable as the + declaration emits a symbol-macro that signals a + SB-EXT:DEPRECATION-ERROR at run-time when accessed. + + - CL:TYPE: Declare named types (i.e. defined via DEFTYPE), + standard classes, structure classes and condition classes as + deprecated.") + +(defsection @deprecation-examples (:title "Deprecation Examples") + "Marking functions as deprecated: + + (defun foo ()) + (defun bar ()) + (declaim (deprecated :early (\"my-system\" \"1.2.3\") + (function foo :replacement bar))) + + ;; Remember: do not define the actual function or variable in case of + ;; :final deprecation: + (declaim (deprecated :final (\"my-system\" \"1.2.3\") + (function fez :replacement whoop))) + + Attempting to use the deprecated functions: + + (defun baz () + (foo)) + | STYLE-WARNING: The function CL-USER::FOO has been deprecated... + => BAZ + (baz) + => NIL ; no error + + (defun danger () + (fez)) + | WARNING: The function CL-USER::FEZ has been deprecated... + => DANGER + (danger) + |- ERROR: The function CL-USER::FEZ has been deprecated...") + + +(defsection @deprecated-interfaces-in-sbcl + (:title "Deprecated Interfaces in SBCL") + "This sections lists legacy interfaces in various stages of deprecation." + (@list-of-deprecated-interfaces section) + (@historical-interfaces section)) + +(defsection @list-of-deprecated-interfaces + (:title "List of Deprecated Interfaces") + (@early-deprecation section) + (@late-deprecation section) + (@final-deprecation section)) + +(defsection @early-deprecation (:title "Early Deprecation") + "- `SOCKINT::WIN32-*` + + Deprecated in favor of the corresponding prefix-less functions + (e.g. `SOCKINT::BIND` replaces `SOCKINT::WIN32-BIND`) as of + 1.2.10 in March 2015. Expected to move into late deprecation in + August 2015. + + - SB-UNIX:UNIX-EXIT + + Deprecated as of 1.0.56.55 in May 2012. Expected to move into + late deprecation in May 2013. + + When the SBCL process termination was refactored, + SB-UNIX:UNIX-EXIT ceased to be used internally. Since `SB-UNIX` + is an internal package not intended for user code to use, and + since we're slowly in the process of refactoring things to be + less Unix-oriented, SB-UNIX:UNIX-EXIT was initially deleted as + it was no longer used. Unfortunately it became apparent that it + was used by several external users, so it was re-instated in + deprecated form. + + While the cost of keeping SB-UNIX:UNIX-EXIT indefinitely is + trivial, the ability to refactor our internals is important, so + its deprecation was taken as an opportunity to highlight that + `SB-UNIX` is an internal package and `SB-POSIX` should be used + by user-programs instead -- or alternatively calling the foreign + function directly if the desired interface doesn't for some + reason exist in `SB-POSIX`. + + __Remedy__ + + For code needing to work with legacy SBCLs, use e.g. + `SYSTEM-EXIT`. In modern SBCLs, simply call either SB-POSIX:EXIT + or SB-EXT:EXIT with appropriate arguments. + + - `SB-C::MERGE-TAIL-CALLS` compiler policy + + Deprecated as of 1.0.53.74 in November 2011. Expected to move + into late deprecation in November 2012. + + This compiler policy was never functional: SBCL has always + merged tail calls when it could, regardless of this policy + setting. (It was also never officially supported, but several + code-bases have historically used it.) + + __Remedy__ + + Simply remove the policy declarations. They were never necessary: SBCL + always merged tail-calls when possible. To disable tail merging, + structure the code to avoid the tail position instead. + + - The Spinlock API + + Deprecated as of 1.0.53.11 in August 2011. Expected to move into + late deprecation in August 2012. + + Spinlocks were an internal interface but had a number of + external users and were hence deprecated instead of being simply + deleted. + + Affected symbols: SB-THREAD::SPINLOCK, SB-THREAD::MAKE-SPINLOCK, + SB-THREAD::WITH-SPINLOCK, SB-THREAD::WITH-RECURSIVE-SPINLOCK, + SB-THREAD::GET-SPINLOCK, SB-THREAD::RELEASE-SPINLOCK, + SB-THREAD::SPINLOCK-VALUE, and SB-THREAD::SPINLOCK-NAME. + + __Remedy__ + + Use the mutex API instead, or implement spinlocks suiting your + needs on top of SB-EXT:COMPARE-AND-SWAP, SB-EXT:SPIN-LOOP-HINT, + etc. + + - `SOCKINT::HANDLE->FD`, `SOCKINT::FD->HANDLE` + + Internally deprecated in 2012. Declared deprecated as of 1.2.10 + in March 2015. Expected to move into final deprecation in August + 2015.") + +(defsection @late-deprecation (:title "Late Deprecation") + "- SB-THREAD:JOIN-THREAD-ERROR-THREAD and + SB-THREAD:INTERRUPT-THREAD-ERROR-THREAD + + Deprecated in favor of SB-THREAD:THREAD-ERROR-THREAD as of + 1.0.29.17 in June 2009. Expected to move into final deprecation + in June 2012. + + __Remedy__ + + For code that needs to support legacy SBCLs, use e.g.: + + (defun get-thread-error-thread (condition) + #+#.(cl:if (cl:find-symbol \"THREAD-ERROR-THREAD\" :sb-thread) + '(and) '(or)) + (sb-thread:thread-error-thread condition) + #-#.(cl:if (cl:find-symbol \"THREAD-ERROR-THREAD\" :sb-thread) + '(and) '(or)) + (etypecase condition + (sb-thread:join-thread-error + (sb-thread:join-thread-error-thread condition)) + (sb-thread:interrupt-thread-error + (sb-thread:interrupt-thread-error-thread condition)))) + + - SB-INTROSPECT:FUNCTION-ARGLIST + + Deprecated in favor of SB-INTROSPECT:FUNCTION-LAMBDA-LIST as of + 1.0.24.5 in January 2009. Expected to move into final + deprecation in January 2012. + + Renamed for consistency and aesthetics. Functions have + lambda-lists, not arglists. + + __Remedy__ + + For code that needs to support legacy SBCLs, use e.g.: + + (defun get-function-lambda-list (function) + #+#.(cl:if (cl:find-symbol \"FUNCTION-LAMBDA-LIST\" :sb-introspect) + '(and) '(or)) + (sb-introspect:function-lambda-list function) + #-#.(cl:if (cl:find-symbol \"FUNCTION-LAMBDA-LIST\" :sb-introspect) + '(and) '(or)) + (sb-introspect:function-arglist function)) + + - Stack Allocation Policies + + Deprecated in favor of SB-EXT:*STACK-ALLOCATE-DYNAMIC-EXTENT* as + of 1.0.19.7 in August 2008, and are expected to be removed in + August 2012. + + Affected symbols: `SB-C::STACK-ALLOCATE-DYNAMIC-EXTENT`, + `SB-C::STACK-ALLOCATE-VECTOR`, and + `SB-C::STACK-ALLOCATE-VALUE-CELLS`. + + These compiler policies were never officially supported, and + turned out the be a flawed design. + + __Remedy__ + + For code that needs stack-allocation in legacy SBCLs, + conditionalize using: + + #-#.(cl:if (cl:find-symbol \"*STACK-ALLOCATE-DYNAMIC-EXTENT*\" :sb-ext) + '(and) '(or)) + (declare (optimize sb-c::stack-allocate-dynamic-extent)) + + However, unless stack allocation is essential, we recommend + simply removing these declarations. Refer to documentation on + `SB-EXT:*STACK-ALLOCATE-DYNAMIC*` for details on stack + allocation control in modern SBCLs. + + - `SB-SYS:OUTPUT-RAW-BYTES` + + Deprecated as of 1.0.8.16 in June 2007. Expected to move into final + deprecation in June 2012. + + Internal interface with some external users. Never officially + supported, deemed unnecessary in presence of WRITE-SEQUENCE and + bivalent streams. + + __Remedy__ + + Use streams with element-type (UNSIGNED-BYTE 8) or + :DEFAULT -- the latter allowing both binary and character IO -- + in conjunction with WRITE-SEQUENCE.") + +(defsection @final-deprecation (:title "Final Deprecation") + "No interfaces are currently in final deprecation.") + +(defsection @historical-interfaces (:title "Historical Interfaces") + "The following is a partial list of interfaces present in historical + versions of SBCL, which have since then been deleted. + + - `SB-KERNEL:INSTANCE-LAMBDA` + + Historically needed for CLOS code. Deprecated as of 0.9.3.32 in + August 2005. Deleted as of 1.0.47.8 in April 2011. Plain LAMBDA + can be used where SB-KERNEL:INSTANCE-LAMBDA used to be needed. + + - `SB-ALIEN:DEF-ALIEN-ROUTINE`, `SB-ALIEN:DEF-ALIEN-VARIABLE`, + `SB-ALIEN:DEF-ALIEN-TYPE` + + Inherited from CMUCL, naming convention not consistent with + preferred SBCL style. Deprecated as of 0.pre7.90 in December + 2001. Deleted as of 1.0.9.17 in September 2007. Replaced by + SB-ALIEN:DEFINE-ALIEN-ROUTINE, SB-ALIEN:DEFINE-ALIEN-VARIABLE, + and SB-ALIEN:DEFINE-ALIEN-TYPE.") diff --git a/contrib/sb-manual/doc/efficiency.lisp b/contrib/sb-manual/doc/efficiency.lisp new file mode 100644 index 000000000..99853687c --- /dev/null +++ b/contrib/sb-manual/doc/efficiency.lisp @@ -0,0 +1,399 @@ +(in-package :sb-manual) + +(defsection @efficiency (:title "Efficiency") + (@slot-access section) + (@stack-allocation section) + (@modular-arithmetic section) + (@recognized-idioms section) + (@global-and-always-bound-variables section) + (@miscellaneous-efficiency-issues section)) + +(defsection @slot-access (:title "Slot Access") + (@structure-object-slot-access section) + (@standard-object-slot-access section)) + +(defsection @structure-object-slot-access + (:title "Structure Object Slot Access") + "Structure slot accessors are efficient only if the compiler is + able to open code them: compiling a call to a structure slot + accessor before the structure is defined, declaring one NOTINLINE, + or passing it as a functional argument to another function causes + severe performance degradation.") + +(defsection @standard-object-slot-access + (:title "Standard Object Slot Access") + "The most efficient way to access a slot of a STANDARD-OBJECT is + by using SLOT-VALUE with a constant slot name argument inside a + DEFMETHOD body, where the variable holding the instance is a + specializer parameter of the method and is never assigned to. The + cost is roughly 1.6 times that of an open coded structure slot + accessor. + + Second most efficient way is to use a CLOS slot accessor, or + SLOT-VALUE with a constant slot name argument, but in circumstances + other than specified above. This may be up to 3 times as slow as the + method described above. + + Example: + + (defclass foo () ((bar))) + + ;; Fast: specializer and never assigned to + (defmethod quux ((foo foo) new) + (let ((old (slot-value foo 'bar))) + (setf (slot-value foo 'bar) new) + old)) + + ;; Slow: not a specializer + (defmethod quux ((foo foo) new) + (let* ((temp foo) + (old (slot-value temp 'bar))) + (setf (slot-value temp 'bar) new) + old)) + + ;; Slow: assignment to FOO + (defmethod quux ((foo foo) new) + (let ((old (slot-value foo 'bar))) + (setf (slot-value foo 'bar) new) + (setf foo new) + old)) + + Note that when profiling code such as this, the first few calls to the + generic function are not representative, as the dispatch mechanism is + lazily set up during those calls.") + +(defsection @stack-allocation (:title "Stack Allocation") + "SBCL has fairly extensive support for performing allocations on the + stack when a variable or function is declared DYNAMIC-EXTENT. The + DYNAMIC-EXTENT declarations are not verified but are simply trusted + as long as SB-EXT:*STACK-ALLOCATE-DYNAMIC-EXTENT* is true." + (sb-ext:*stack-allocate-dynamic-extent* variable) + "SBCL recognizes any value which a variable declared DYNAMIC-EXTENT + can take on as having dynamic extent. This means that, in addition + to the value a variable is bound to initially, a value assigned to a + variable by SETQ is also recognized as having dynamic extent when + the variable is declared DYNAMIC-EXTENT. Users can thus build + complex structures on the stack using iteration and SETQ. + + At present, SBCL implements stack allocation for the following kinds + of values when they are recognized as having dynamic extent: + + - &REST lists; + + - the results of CONS, LIST, LIST*, and VECTOR; + + - the result of simple forms of MAKE-ARRAY: stack allocation is + possible only if the resulting array is known to be both simple + and one-dimensional, and has a constant :ELEMENT-TYPE; + + > __Warning__: Stack space is limited, so allocation of a large + > vector may cause stack overflow. Stack overflow checks are + > done except in zero SAFETY policies. + + - closures defined with FLET or LABELS with a bound DYNAMIC-EXTENT + declaration; + + - anonymous closures defined with LAMBDA; + + - user-defined structures when the structure constructor defined using + DEFSTRUCT has been declared INLINE; + + > _Note_: Structures with _raw_ slots can currently be + > stack-allocated only on x86 and x86-64. A raw slot is one + > whose declared type is a subtype of exactly one of: + > DOUBLE-FLOAT, SINGLE-FLOAT, `(COMPLEX + > DOUBLE-FLOAT)`, `(COMPLEX SINGLE-FLOAT)`, or SB-EXT:WORD; but + > as an exception to the preceding, any subtype of FIXNUM is not + > stored as raw despite also being a subtype of SB-EXT:WORD. + + - otherwise-inaccessible parts of objects recognized to be dynamic + extent. The support for detecting when this applies is very + sophisticated. The compiler can do this detection when any value + form for a variable contains conditional allocations, function + calls, inlined functions, anonymous closures, or even other + variables. This allows stack allocation of complex structures. + + Examples: + + ;;; Declaiming a structure constructor inline before definition makes + ;;; stack allocation possible. + (declaim (inline make-thing)) + (defstruct thing obj next) + + ;;; Stack allocation of various objects bound to DYNAMIC-EXTENT + ;;; variables. + (let* ((list (list 1 2 3)) + (nested (cons (list 1 2) (list* 3 4 (list 5)))) + (vector (make-array 3 :element-type 'single-float)) + (thing (make-thing :obj list + :next (make-thing :obj (make-array 3)))) + (closure (let ((y ...)) (lambda () y)))) + (declare (dynamic-extent list nested vector thing closure)) + ...) + + ;;; Stack allocation of objects assigned to DYNAMIC-EXTENT variables. + (let ((x nil)) + (declare (dynamic-extent x)) + (setq x (list 1 2 3)) + (dotimes (i 10) + (setq x (cons i x))) + ...) + + ;;; Stack allocation of arguments to a local function is equivalent + ;;; to stack allocation of local variable values. + (flet ((f (x) + (declare (dynamic-extent x)) + ...)) + ... + (f (list 1 2 3)) + (f (cons (cons 1 2) (cons 3 4))) + ...) + + ;;; Stack allocation of &REST lists + (defun foo (&rest args) + (declare (dynamic-extent args)) + ...) + + As a notable exception to recognizing otherwise inaccessible parts + of other recognized dynamic extent values, SBCL does not as of + 1.0.48.21 propagate dynamic-extentness through &REST arguments -- + but another conforming implementation might, so portable code should + not rely on this. + + (declaim (inline foo)) + (defun foo (fun &rest arguments) + (declare (dynamic-extent arguments)) + (apply fun arguments)) + + (defun bar (a) + ;; SBCL will heap allocate the result of (LIST A), and stack + ;; allocate only the spine of the &rest list -- so this is + ;; safe but unportable. + ;; + ;; Another implementation, including earlier versions of SBCL + ;; might consider (LIST A) to be otherwise inaccessible and + ;; stack-allocate it as well! + (foo #'car (list a))) + + If dynamic extent constraints specified in the Common Lisp standard + are violated, the best that can happen is for the program to have + garbage in variables and return values; more commonly, the system + will crash. + + In particular, it is important to realize that this can interact in + suprising ways with the otherwise inaccessible parts criterion: + + (let* ((a (list 1 2 3)) + (b (cons a a))) + (declare (dynamic-extent b)) + ;; Unless A is accessed elsewhere as well, SBCL will consider + ;; it to be otherwise inaccessible -- it can only be accessed + ;; through B, after all -- and stack allocate it as well. + ;; + ;; Hence returning (CAR B) here is unsafe. + ...) + + SBCL also performs sophisticated escape analysis to enable automatic + stack allocation of local functions without any bound dynamic extent + declarations in many situations where the compiler can prove that no + uses escape (traditional Lisp terminology names this situation \"all + uses are downward funargs\"). For example, in the following + function, the local function `#'PREDICATEP` is stack allocated, + because the compiler understands that the built-in function + POSITION-IF only uses its first argument as a downward funarg: + + (let ((acc 0)) + (flet ((predicatep (num) (plusp (+ num off)))) + (dotimes (i 10) + (incf acc (position-if #'predicatep array))) + (if (plusp off) + (incf acc (if (positivep acc) 10 3)) + (incf acc (position-if #'predicatep array)))) + acc) + + Users can also declare that their own functions take downward + funargs by adding bound dynamic extent declarations on the function + arguments. + + (defun trivial-hof (fun arg) + (declare (dynamic-extent fun)) + (funcall fun 3 arg)) + + Currently, such dynamic extent declarations only cause stack + allocation of downward funargs at call sites on sufficiently unsafe + policy. This is partly because the compiler is currently not able to + detect incorrect usage of dynamic extent declarations. + + (defun autodxclosure1 (&optional (x 4)) + ;; Calling a higher-order function will only implicitly + ;; stack-allocate a funarg if the callee is trusted (a CL: + ;; function) or the caller is unsafe. + (declare (optimize speed (safety 0) (debug 0))) + (trivial-hof (lambda (a b) (+ a b x)) 92))") + +(defsection @modular-arithmetic (:title "Modular Arithmetic") + "Some numeric functions have a property: n lower bits of the + result depend only on n lower bits of (all or some) arguments. If + the compiler sees an expression of form `(LOGAND <EXPR> <MASK>)`, + where `<EXPR>` is a tree of such _good_ functions and `<MASK>` is + known to be of type `(UNSIGNED-BYTE <W>)`, where `<W>` is a _good_ + width, all intermediate results will be cut to `<W>` bits (but it is + not done for variables and constants!). This often results in an + ability to use simple machine instructions for the functions. + + Consider this example: + + (defun i (x y) + (declare (type (unsigned-byte 32) x y)) + (ldb (byte 32 0) (logxor x (lognot y)))) + + The result of `(LOGNOT Y)` will be negative and of type + `(SIGNED-BYTE 33)`, so a naive implementation on a 32-bit platform + is unable to use 32-bit arithmetic here. But modular arithmetic + optimizer is able to do it: because the result is cut down to 32 + bits, the compiler will replace LOGXOR and LOGNOT with versions + cutting results to 32 bits, and because terminals (here, expressions + `X` and `Y`) are also of type `(UNSIGNED-BYTE 32)`, 32-bit machine + arithmetic can be used. + + As of SBCL 0.8.5 good functions are `+`, `-`, LOGAND, LOGIOR, + LOGXOR, LOGNOT and their combinations; and ASH with the positive + second argument. Good widths are 32 on 32-bit CPUs and 64 on 64-bit + CPUs. While it is possible to support smaller widths as well, + currently this is not implemented." + (@signed-modular-arithmetic section)) + +(defsection @signed-modular-arithmetic (:title "Signed Modular Arithmetic") + "Sign-extending the result in the following way will be + translated into signed modular arithmetic: + + (defun add (a b) + (declare (type (signed-byte 64) a b)) + (let ((u (ldb (byte 64 0) (+ a b)))) + (logior u (- (mask-field (byte 1 63) u)))))") + +(defsection @recognized-idioms (:title "Recognized Idioms") + "Common Lisp doesn't directly expose all features present in + modern hardware. Some code patterns are recognized and turned into + more efficient hardware instructions without requiring the use of + internal features." + (@count-trailing-zeros section)) + +(defsection @count-trailing-zeros (:title "Count Trailing Zeros") + " (defun ctz (n) + (declare (type (unsigned-byte 64) n)) + (integer-length (ldb (byte 64 0) (lognor n (- n))))) + + is turned into hardware instructions on arm64 and x86-64. It returns + 64 when `N` is 0. `N` can also be `(SIGNED-BYTE 64)` or FIXNUM.") + +(defsection @global-and-always-bound-variables + (:title "Global and Always-bound Variables") + (sb-ext:defglobal macro) + "- [__declaration__] SB-EXT:GLOBAL + + Syntax: `(SB-EXT:GLOBAL &REST SYMBOLS)` + + Only valid as a global proclamation. + + Specifies that the named symbols cannot be proclaimed or locally + declared SPECIAL. Proclaiming an already special or constant + variable name as SB-EXT:GLOBAL signal an error. Allows more + efficient value lookup in threaded environments in addition to + expressing programmer intention. + + - [__declaration__] SB-EXT:ALWAYS-BOUND + + Syntax: `(SB-EXT:ALWAYS-BOUND &REST SYMBOLS)` + + Only valid as a global proclamation. + + Specifies that the named symbols are always bound. Inhibits + MAKUNBOUND of the named symbols. Proclaiming an unbound symbol + as SB-EXT:ALWAYS-BOUND signals an error. Allows the compiler to + elide boundness checks from value lookups.") + +(defsection @miscellaneous-efficiency-issues + (:title "Miscellaneous Efficiency Issues") + "FIXME: The material in the CMUCL manual about getting good + performance from the compiler should be reviewed, reformatted in + Texinfo, lightly edited for SBCL, and substituted into this + manual. In the meantime, the original CMUCL manual is still 95+% + correct for the SBCL version of the Python compiler. See the + sections + + - Advanced Compiler Use and Efficiency Hints + - Advanced Compiler Introduction + - More About Types in Python + - Type Inference + - Source Optimization + - Tail Recursion + - Local Call + - Block Compilation + - Inline Expansion + - Object Representation + - Numbers + - General Efficiency Hints + - Efficiency Notes + + Besides this information from the CMUCL manual, there are a few other + points to keep in mind. + + - The CMUCL manual doesn't seem to state it explicitly, but Python + has a mental block about type inference when assignment is + involved. Python is very aggressive and clever about inferring the + types of values bound with LET, LET*, inline function call, and so + forth. However, it's much more passive and dumb about inferring + the types of values assigned with SETQ, SETF, and friends. It + would be nice to fix this, but in the meantime don't expect that + just because it's very smart about types in most respects it will + be smart about types involved in assignments. (This doesn't affect + its ability to benefit from explicit type declarations involving + the assigned variables, only its ability to get by without + explicit type declarations.)" + ;; FIXME: Python dislikes assignments but not in type inference. The + ;; real problems are loop induction, closed over variables and + ;; aliases. + "- Since the time the CMUCL manual was written, CMUCL (and thus SBCL) + has gotten a generational garbage collector. This means that there + are some efficiency implications of various patterns of memory + usage which aren't discussed in the CMUCL manual. (Some new + material should be written about this.) + + - SBCL has some important known efficiency problems. Perhaps the + most important are + + - The garbage collector is not particularly efficient, at least + on platforms without the generational collector (as of SBCL + 0.8.9, all except x86). + + - Various aspects of the PCL implementation of CLOS are more + inefficient than necessary. + + Finally, note that Common Lisp defines many constructs which, in the + infamous phrase, \"could be compiled efficiently by a sufficiently + smart compiler\". The phrase is infamous because making a compiler + which actually is sufficiently smart to find all these optimizations + systematically is well beyond the state of the art of current + compiler technology. Instead, they're optimized on a case-by-case + basis by hand-written code, or not optimized at all if the + appropriate case hasn't been hand-coded. Some cases where no such + hand-coding has been done as of SBCL version 0.6.3 include + + - `(REDUCE #'F X)` where the type of `X` is known at compile time, + + - various bit vector operations, e.g. `(POSITION 0 SOME-BIT-VECTOR)`, + + - specialized sequence idioms, e.g. `(REMOVE ITEM LIST :COUNT 1)`, + + - cases where local compilation policy does not require excessive + type checking, e.g. `(LOCALLY (DECLARE (SAFETY 1)) (ASSOC ITEM LIST))` + (which currently performs safe ENDP checking internal to ASSOC). + + If your system's performance is suffering because of some construct + which could in principle be compiled efficiently, but which the SBCL + compiler can't in practice compile efficiently, consider writing a + patch to the compiler and submitting it for inclusion in the main + sources. Such code is often reasonably straightforward to write; + search the sources for the string `deftransform` to find many + examples (some straightforward, some less so).") diff --git a/contrib/sb-manual/doc/external-formats.lisp b/contrib/sb-manual/doc/external-formats.lisp new file mode 100644 index 000000000..ac09e3b63 --- /dev/null +++ b/contrib/sb-manual/doc/external-formats.lisp @@ -0,0 +1,136 @@ +(in-package :sb-manual) + +(defsection @external-formats (:title "External Formats") + "External formats determine the coding of characters from/to sequences + of octets when exchanging data with the outside world. Examples of + such exchanges are: + + - Character streams associated with files, sockets and process + input/output (see @STREAM-EXTERNAL-FORMATS and + @RUNNING-EXTERNAL-PROGRAMS) + + - Names of files + + - Foreign strings (see @FOREIGN-TYPES-AND-LISP-TYPES) + + - Posix interface (see @SB-POSIX) + + - Hostname- and protocol-related functions of the BSD-socket interface + (see @NETWORKING) + + Technically, external formats in SBCL are named objects describing + coding of characters as well as policies in case de- or encoding is + not possible. Each external format has a canonical name and zero or + more aliases. User code mostly interacts with external formats by + supplying external format designators to functions that use external + formats internally." + (@default-external-format section) + (@external-format-designators section) + (@character-coding-conditions section) + (@converting-between-strings-and-octet-vectors section) + (@supported-external-formats section)) + +(defsection @default-external-format (:title "The Default External Format") + (sb-ext:*default-external-format* variable) + (sb-ext:*default-source-external-format* variable) + ;; FIXME: Move this to @FFI? + (sb-ext:*default-c-string-external-format* variable)) + +(defsection @external-format-designators (:title "External Format Designators") + "In situations where an external format designator is required, such as + the :EXTERNAL-FORMAT argument in calls to OPEN or WITH-OPEN-FILE, + users may supply the name of an encoding to denote the external + format which is applying that encoding to Lisp characters. + + In addition to the basic encoding for an external format, options + controlling various special cases may be passed, by using a list + (whose first element must be an encoding name and whose rest is a + plist) as an external file format designator. + + More specifically, external format designators can take the + following forms: + + - :DEFAULT: Designates the current default external format (see + @DEFAULT-EXTERNAL-FORMAT). + + - `<keyword>`: Designates the supported external format that has + `<keyword>` as one of its names (see @SUPPORTED-EXTERNAL-FORMATS). + + - `(<keyword> . <options-plist>)`: Designates an external format + that is like the one designated by `<keyword>` with options as + specified in `<options-plist>`. + + Valid options for `<options-plist>` are: + + - `:NEWLINE <newline>` + + An external format with an explicit :NEWLINE option is like its + `<keyword>` parent but recognizes certain characters or + character sequences as newlines. For :LF (the default), the + `#\\Linefeed` character is treated as `#\\Newline` for both + input and output. For :CR, `#\\Return` is treated as + `#\\Newline`, while for :CRLF the two-character sequence + `#\\Return #\\Linefeed` is translated to and from + `#\\Newline`. + + - `:REPLACEMENT <replacement>` + + An external format with an explicit :REPLACEMENT option is like + its `<keyword>` parent but does not signal an error in case a + character or octet sequence cannot be en- or decoded. Instead, + it inserts `<replacement>` at the position in question. + `<replacement>` must be a string designator; that is, a + character or a string. + + For example: + + (with-open-file (stream pathname :external-format '(:utf-8 :replacement #\\?)) + (read-line stream)) + + will read the first line of `\\PATHNAME`, replacing any octet + sequence that is not valid in the UTF-8 external format with a + question mark character.") + +(defsection @character-coding-conditions (:title "Character Coding Conditions") + "De- or encoding characters using a given external format is not always + possible: + + - Decoding an octet vector using a given external format can fail if + it contains an octet or sequence of octets that does not have an + interpretation as a character according to the external format. + + - Conversely, a string may contain characters that a given external + format cannot encode. For example, the ASCII external format + cannot encode the character `#\\รถ`. + + Unless the external format governing the coding uses the + :REPLACEMENT option, SBCL will signal (continuable) errors under the + above circumstances. The types of the condition signaled are not + currently exported or documented but will be in future SBCL + versions.") + +(defsection @converting-between-strings-and-octet-vectors + (:title "Converting between Strings and Octet Vectors") + "To encode Lisp strings as octet vectors and decode octet vectors as + Lisp strings, the following SBCL-specific functions can be used:" + (sb-ext:string-to-octets function) + (sb-ext:octets-to-string function)) + +(eval-when (:compile-toplevel :load-toplevel :execute) + (defun list-external-formats-in-markdown () + (flet ((table (items) + (with-output-to-string (s) + (loop for (canonical-name . names) in items + do (format s "- `~S`~%~% ~{`~S`~^, ~}~%~%" + canonical-name names))))) + (let (result) + (loop for ef across sb-impl::*external-formats* + when (sb-impl::external-format-p ef) + do + (pushnew (sb-impl::ef-names ef) result :test #'equal)) + (table (sort result #'string< :key #'car)))))) + +(defsection @supported-external-formats (:title "Supported External Formats") + "The following lists the external formats supported by SBCL in + the form of the respective canonical name followed by the list of aliases:" + #.(list-external-formats-in-markdown)) diff --git a/contrib/sb-manual/doc/ffi.lisp b/contrib/sb-manual/doc/ffi.lisp new file mode 100644 index 000000000..7796381e3 --- /dev/null +++ b/contrib/sb-manual/doc/ffi.lisp @@ -0,0 +1,781 @@ +(in-package :sb-manual) + +(defsection @foreign-function-interface + (:title "Foreign Function Interface") + "This chapter describes SBCL's interface to C programs and + libraries (and, since C interfaces are a sort of _lingua franca_ + of the Unix world, to other programs and libraries in general). + + > _Note_: In the modern Lisp world, the usual term for this + > functionality is Foreign Function Interface, or FFI, where despite + > the mention of _function_ in this term, FFI also refers to direct + > manipulation of C data structures as well as functions. The + > traditional CMUCL terminology is Alien Interface, and while that + > older terminology is no longer used much in the system + > documentation, it still reflected in names in the implementation, + > notably in the name of the `SB-ALIEN` package." + (@introduction-to-the-foreign-function-interface section) + (@foreign-types section) + (@operations-on-foreign-values section) + (@foreign-variables section) + (@foreign-data-structure-examples section) + (@loading-shared-object-files section) + (@foreign-function-calls section) + (@calling-lisp-from-c section) + (@step-by-step-example-of-the-foreign-function-interface section)) + +(defsection @introduction-to-the-foreign-function-interface + (:title "Introduction to the Foreign Function Interface") + ;; AKA Introduction to Aliens in the CMU CL manual + "Because of Lisp's emphasis on dynamic memory allocation and garbage + collection, Lisp implementations use non-C-like memory + representations for objects. This representation mismatch creates + friction when a Lisp program must share objects with programs which + expect C data. There are three common approaches to establishing + communication: + + - The burden can be placed on the foreign program (and programmer) + by requiring the knowledge and use of the representations used + internally by the Lisp implementation. This can require a + considerable amount of \"glue\" code on the C side, and that code + tends to be sensitively dependent on the internal implementation + details of the Lisp system. + + - The Lisp system can automatically convert objects back and forth + between the Lisp and foreign representations. This is convenient, + but translation becomes prohibitively slow when large or complex + data structures must be shared. This approach is supported by the + SBCL FFI, and used automatically when passing integers and + strings. + + - The Lisp program can directly manipulate foreign objects through + the use of extensions to the Lisp language. + + SBCL, like CMUCL before it, relies primarily on the automatic + conversion and direct manipulation approaches. The `SB-ALIEN` + package provides a facility wherein foreign values of simple scalar + types are automatically converted and complex types are directly + manipulated in their foreign representation. Additionally the + lower-level System Area Pointers (or SAPs) can be used where + necessary to provide untyped access to foreign memory. + + Any foreign objects that can't automatically be converted into Lisp + values are represented by objects of type + SB-ALIEN-INTERNALS:ALIEN-VALUE Since Lisp is a dynamically typed + language, even foreign objects must have a run-time type; this type + information is provided by encapsulating the raw pointer to the + foreign data within an SB-ALIEN-INTERNALS:ALIEN-VALUE object. + + The type language and operations on foreign types are intentionally + similar to those of the C language.") + +(defsection @foreign-types (:title "Foreign Types") + "Alien types have a description language based on nested list + structure. For example the C type + + struct foo { + int a; + struct foo *b[100]; + }; + + has the corresponding SBCL FFI type + + (struct foo + (a int) + (b (array (* (struct foo)) 100)))" + (@defining-foreign-types section) + (@foreign-types-and-lisp-types section) + (@foreign-type-specifiers section)) + +(defsection @defining-foreign-types (:title "Defining Foreign Types") + "Types may be either named or anonymous. With structure and union + types, the name is part of the type specifier, allowing recursively + defined types such as: + + (struct foo (a (* (struct foo)))) + + An anonymous structure or union type is specified by using the name + NIL. The WITH-ALIEN macro defines a local scope which _captures_ any + named type definitions. Other types are not inherently named, but + can be given named abbreviations using the DEFINE-ALIEN-TYPE macro.") + +(defsection @foreign-types-and-lisp-types + (:title "Foreign Types and Lisp Types") + "The foreign types form a subsystem of the SBCL type system. An + ALIEN type specifier provides a way to use any foreign type as a + Lisp type specifier. For example, + + (typep foo '(alien (* int))) + + can be used to determine whether `FOO` is a pointer to a foreign + `int`. ALIEN type specifiers can be used in the same ways as + ordinary Lisp type specifiers (like STRING.) Alien type declarations + are subject to the same precise type checking as any other + declaration. See @PRECISE-TYPE-CHECKING. + + Note that the type identifiers used in the foreign type system + overlap with native Lisp type specifiers in some cases. For example, + the type specifier `(ALIEN SINGLE-FLOAT)` is identical to + SINGLE-FLOAT, since foreign floats are automatically converted to + Lisp floats. When TYPE-OF is called on an alien value that is not + automatically converted to a Lisp value, then it will return an + ALIEN type specifier.") + +(defsection @foreign-type-specifiers (:title "Foreign Type Specifiers") + "> _Note_: All foreign type names are exported from the `SB-ALIEN` + > package. Some foreign type names are also symbols in the + > `COMMON-LISP` package, in which case they are reexported from the + > `SB-ALIEN` package, so that e.g. it is legal to refer to + > SINGLE-FLOAT. + + These are the basic foreign type specifiers: + + - The foreign type specifier `(* <FOO>)` describes a pointer to an + object of type `<FOO>`. A pointed-to type `<FOO>` of T indicates a + pointer to anything, similar to `void *` in ANSI C. A null alien + pointer can be detected with the NULL-ALIEN function. + + - The foreign type specifier `(ARRAY <FOO> &REST <DIMENSIONS>)` + describes array of the specified `<DIMENSIONS>`, holding elements + of type `<FOO>`. Note that (unlike in C) `(* <FOO>)` and + `(ARRAY <FOO>)` are considered to be different types when + type checking is done. If equivalence of pointer and array types + is desired, it may be explicitly coerced using CAST. + + Arrays are accessed using DEREF, passing the indices + as additional arguments. Elements are stored in column-major order + (as in C), so the first dimension determines only the size of the + memory block, and not the layout of the higher dimensions. An array + whose first dimension is variable may be specified by using NIL as + the first dimension. Fixed-size arrays can be allocated as array + elements, structure slots or WITH-ALIEN variables. Dynamic arrays + can only be allocated using MAKE-ALIEN. + + - The foreign type specifier `(STRUCT <NAME> &REST <FIELDS>)` + describes a structure type with the specified `<NAME>` and + `<FIELDS>`. Fields are allocated at the same offsets used by the + implementation's C compiler, as guessed by the SBCL internals. + An optional :ALIGNMENT keyword argument can be specified for each + field to explicitly control the alignment of a field. If `<NAME>` + is NIL then the structure is anonymous. + + If a named foreign STRUCT specifier is passed to + DEFINE-ALIEN-TYPE or WITH-ALIEN, then this defines, + respectively, a new global or local foreign structure type. If + no `<FIELDS>` are specified, then the fields are taken from the + current (local or global) alien structure type definition of + `<NAME>`. + + - The foreign type specifier `(UNION <NAME> &REST <FIELDS>)` is + similar to STRUCT but describes a union type. All fields are + allocated at the same offset, and the size of the union is the + size of the largest field. The programmer must determine which + field is active from context. + + - The foreign type specifier `(ENUM <NAME> &REST <SPECS>)` describes + an enumeration type that maps between integer values and symbols. + If `<NAME>` is NIL, then the type is anonymous. Each element of + the `<SPECS>` list is either a Lisp symbol, or a list + `(<symbol> <value>)`. `<value>` is an integer. If `<value>` is not + supplied, then it defaults to one greater than the value for the + preceding spec (or to zero if it is the first spec). + + - The foreign type specifier `(SIGNED &OPTIONAL <BITS>)` specifies a + signed integer with the specified number of `<BITS>` precision. + The upper limit on integer precision is determined by the + machine's word size. If `<BITS>` is not specified, the maximum + size will be used. + + - The foreign type specifier `(INTEGER &OPTIONAL <BITS>)` is + equivalent to the corresponding type specifier using SIGNED + instead of INTEGER. + + - The foreign type specifier `(UNSIGNED &OPTIONAL <BITS>)` is like + corresponding type specifier using SIGNED except that the variable + is treated as an unsigned integer. + + - The foreign type specifier `(BOOLEAN &OPTIONAL <BITS>)` is similar + to an enumeration type but maps from Lisp NIL and T to C 0 and 1 + respectively. `<BITS>` determines the amount of storage allocated + to hold the truth value. + + - The foreign type specifier `\\SINGLE-FLOAT` describes a + floating-point number in IEEE single-precision format. + + - The foreign type specifier `\\DOUBLE-FLOAT` describes a + floating-point number in IEEE double-precision format. + + - The foreign type specifier `(FUNCTION <RESULT-TYPE> &REST + <ARG-TYPES>)` describes a foreign function that takes arguments of + the specified `<ARG-TYPES>` and returns a result of type + `<RESULT-TYPE>`. Note that the only context where a foreign + `\\FUNCTION` type is directly specified is in the argument to + ALIEN-FUNCALL. In all other contexts, foreign functions are + represented by foreign function pointer types: `(* (FUNCTION + ...))`. + + - The foreign type specifier `\\SYSTEM-AREA-POINTER` describes a + pointer which is represented in Lisp as a SYSTEM-AREA-POINTER + object. SBCL exports this type from `SB-ALIEN` because CMUCL did, + but tentatively (as of the first draft of this section of the + manual, SBCL 0.7.6) it is deprecated, since it doesn't seem to be + required by user code. + + - The foreign type specifier VOID is used in function types to + declare that no useful value is returned. Using ALIEN-FUNCALL to + call a VOID foreign function will return zero values. + + - The foreign type specifier `(C-STRING &KEY <external-format> + <element-type> <not-null>)` is similar to `(* CHAR)` but is + interpreted as a null-terminated string, and is automatically + converted into a Lisp string when accessed; or if the pointer is C + `\\NULL` or 0, then accessing it gives Lisp NIL unless + `<not-null>` is true, in which case a TYPE-ERROR is signalled. + + External format conversion is automatically done when Lisp + strings are passed to foreign code, or when foreign strings are + passed to Lisp code. If the type specifier has an explicit + `<external-format>`, that external format will be used. + Otherwise SB-EXT:*DEFAULT-C-STRING-EXTERNAL-FORMAT* will be + used. For example, when the following alien routine is called, + the Lisp string given as argument is converted to an \\EBCDIC + octet representation. + + (define-alien-routine test int (str (c-string :external-format :ebcdic-us))) + + Lisp strings of type BASE-STRING are stored with a trailing + `\\\\NUL` termination, so no copying (either by the user or the + implementation) is necessary when passing them to foreign code, + assuming that the `<EXTERNAL-FORMAT>` and `<ELEMENT-TYPE>` of + the C-STRING type are compatible with the internal + representation of the string. For an SBCL built with Unicode + support that means an `<external-format>` of :ASCII and an + `<ELEMENT-TYPE>` of BASE-CHAR. Without Unicode support the + `<EXTERNAL-FORMAT>` can also be :ISO-8859-1, and the + `<ELEMENT-TYPE>` can also be [CHARACTER][type]. If + `<EXTERNAL-FORMAT>` and `<ELEMENT-TYPE>` are not compatible, or + the string is a `(SIMPLE-ARRAY CHARACTER (*))`, this data is + copied by the implementation as required. + + Assigning a Lisp string to a C-STRING structure field or + variable stores the contents of the string to the memory already + pointed to by that variable. When a foreign object of type + `(* CHAR)` is assigned to a C-STRING, then the C-STRING pointer + is assigned to. This allows C-STRING pointers to be initialized. + For example: + + (cl:in-package \"CL-USER\") ; which USEs package \"SB-ALIEN\" + + (define-alien-type nil (struct foo (str c-string))) + + (defun make-foo (str) + (let ((my-foo (make-alien (struct foo)))) + (setf (slot my-foo 'str) (make-alien char (length str)) + (slot my-foo 'str) str) + my-foo)) + + Storing Lisp NIL in a C-STRING writes C `\\\\NULL` to the + variable." + "- `SB-ALIEN` also exports translations of these C type + specifiers as foreign type specifiers: + + CHAR, SHORT, INT, LONG, UNSIGNED-CHAR, UNSIGNED-SHORT, + UNSIGNED-INT, UNSIGNED-LONG, FLOAT, DOUBLE, SIZE-T, OFF-T") + +(defsection @operations-on-foreign-values + (:title "Operations On Foreign Values") + "This section describes how to read foreign values as Lisp values, + how to coerce foreign values to different kinds of foreign values, + and how to dynamically allocate and free foreign variables." + (@accessing-foreign-values section) + (@coercing-foreign-values section) + (@foreign-dynamic-allocation section)) + +(defsection @accessing-foreign-values (:title "Accessing Foreign Values") + (sb-alien:deref function) + (sb-alien:slot function) + (@untyped-memory section)) + +(defsection @untyped-memory (:title "Untyped memory") + "As noted at the beginning of the chapter, the System Area Pointer + facilities allow untyped access to foreign memory. SAPs can be + converted to and from the usual typed foreign values using SAP-ALIEN + and ALIEN-SAP, and also to and from integers (raw machine + addresses). They should thus be used with caution; corrupting the + Lisp heap or other memory with SAPs is trivial." + (sb-sys:int-sap function) + (sb-sys:sap-ref-32 function) + (sb-sys:sap= function) + "Similarly named functions exist for accessing other sizes of word, + other comparisons, and other conversions. The reader is invited to + use APROPOS and DESCRIBE for more details: + + (apropos \"sap\" :sb-sys)") + +(defsection @coercing-foreign-values (:title "Coercing Foreign Values") + (addr macro) + (cast macro) + (sap-alien macro) + (alien-sap function)) + +(defsection @foreign-dynamic-allocation (:title "Foreign Dynamic Allocation") + "Lisp code can call the C standard library functions `malloc` + and `free` to dynamically allocate and deallocate foreign variables. + The Lisp code uses the same allocator as foreign C code, so it's + OK for foreign code to call `free` on the result of Lisp MAKE-ALIEN, + or for Lisp code to call FREE-ALIEN on foreign objects allocated by + C code." + (make-alien macro) + (make-alien-string function) + (free-alien function)) + +(defsection @foreign-variables (:title "Foreign Variables") + "Both local (stack allocated) and external (C global) foreign + variables are supported." + (@local-foreign-variables section) + (@external-foreign-variables section)) + +(defsection @local-foreign-variables (:title "Local Foreign Variables") + (with-alien macro)) + +(defsection @external-foreign-variables (:title "External Foreign Variables") + "External foreign names are strings, and Lisp names are symbols. When + an external foreign value is represented using a Lisp variable, there + must be a way to convert from one name syntax into the other. The + macros EXTERN-ALIEN, DEFINE-ALIEN-VARIABLE and + DEFINE-ALIEN-ROUTINE use this conversion heuristic: + + - Alien names are converted to Lisp names by uppercasing and + replacing underscores with hyphens. + + - Conversely, Lisp names are converted to alien names by lowercasing + and replacing hyphens with underscores. + + - Both the Lisp symbol and alien string names may be separately + specified by using a list of the form + + (<alien-string> <lisp-symbol>)" + (define-alien-variable macro) + (get-errno function) + (extern-alien macro)) + +(defsection @foreign-data-structure-examples + (:title "Foreign Data Structure Examples") + "Now that we have alien types, operations and variables, we can + manipulate foreign data structures. This C declaration + + struct foo { + int a; + struct foo *b[100]; + }; + + can be translated into the following alien type: + + (define-alien-type nil + (struct foo + (a int) + (b (array (* (struct foo)) 100)))) + + Once the `FOO` alien type has been defined as above, the C + expression + + struct foo f; + f.b[7].a; + + can be translated in this way: + + (with-alien ((f (struct foo))) + (slot (deref (slot f 'b) 7) 'a) + ;; + ;; Do something with f... + ) + + Or consider this example of an external C variable and some accesses: + + struct c_struct { + short x, y; + char a, b; + int z; + c_struct *n; + }; + extern struct c_struct *my_struct; + my_struct->x++; + my_struct->a = 5; + my_struct = my_struct->n; + + which can be manipulated in Lisp like this: + + (define-alien-type nil + (struct c-struct + (x short) + (y short) + (a char) + (b char) + (z int) + (n (* c-struct)))) + (define-alien-variable \"my_struct\" (* c-struct)) + (incf (slot my-struct 'x)) + (setf (slot my-struct 'a) 5) + (setq my-struct (slot my-struct 'n))") + +(defsection @loading-shared-object-files (:title "Loading Shared Object Files") + "Foreign object files can be loaded into the running Lisp process by + calling LOAD-SHARED-OBJECT." + (load-shared-object function) + (unload-shared-object function)) + +(defsection @foreign-function-calls (:title "Foreign Function Calls") + "The foreign function call interface allows a Lisp program to call + many functions written in languages that use the C calling convention. + + Lisp sets up various signal handling routines and other environment + information when it first starts up, and expects these to be in + place at all times. The C functions called by Lisp should not change + the environment, especially the signal handlers: the signal handlers + installed by Lisp typically have interesting flags set (e.g to + request machine context information, or for signal delivery on an + alternate stack) which the Lisp runtime relies on for correct + operation. Precise details of how this works may change without + notice between versions; the source, or the brain of a friendly SBCL + developer, is the only documentation. Users of a Lisp built with the + :SB-THREAD feature should also read the section about threads, + @THREADING." + (alien-funcall function) + (alien-funcall-into function) + (define-alien-routine macro)) + +;; <!-- FIXME: This is a \"changebar\" section from the CMU CL manual. +;; I (WHN 2002-07-14) am not very familiar with this content, so +;; I'm not immediately prepared to try to update it for SBCL, and +;; I'm not feeling masochistic enough to work to encourage this +;; kind of low-level hack anyway. However, I acknowledge that callbacks +;; are sometimes really really necessary, so I include the original +;; text in case someone is hard-core enough to benefit from it. If +;; anyone brings the information up to date for SBCL, it belong +;; either in the main manual or on a CLiki SBCL Internals page. +;; LaTeX \subsection{Accessing Lisp Arrays} +;; LaTeX +;; LaTeX Due to the way \cmucl{} manages memory, the amount of memory that can +;; LaTeX be dynamically allocated by \code{malloc} or \funref{make-alien} is +;; LaTeX limited\footnote{\cmucl{} mmaps a large piece of memory for it's own +;; LaTeX use and this memory is typically about 8 MB above the start of the C +;; LaTeX heap. Thus, only about 8 MB of memory can be dynamically +;; LaTeX allocated.}. +;; +;; Empirically determined to be considerably >8Mb on this x86 linux +;; machine, but I don't know what the actual values are - dan 2003.09.01 +;; +;; Note that this technique is used in SB-GROVEL in the SBCL contrib +;; +;; LaTeX +;; LaTeX To overcome this limitation, it is possible to access the content of +;; LaTeX Lisp arrays which are limited only by the amount of physical memory +;; LaTeX and swap space available. However, this technique is only useful if +;; LaTeX the foreign function takes pointers to memory instead of allocating +;; LaTeX memory for itself. In latter case, you will have to modify the +;; LaTeX foreign functions. +;; LaTeX +;; LaTeX This technique takes advantage of the fact that \cmucl{} has +;; LaTeX specialized array types (\pxlref{specialized-array-types}) that match +;; LaTeX a typical C array. For example, a \code{(simple-array double-float +;; LaTeX (100))} is stored in memory in essentially the same way as the C +;; LaTeX array \code{double x[100]} would be. The following function allows us +;; LaTeX to get the physical address of such a Lisp array: +;; LaTeX \begin{example} +;; LaTeX (defun array-data-address (array) +;; LaTeX \"Return the physical address of where the actual data of an array is +;; LaTeX stored. +;; LaTeX +;; LaTeX ARRAY must be a specialized array type in CMU Lisp. This means ARRAY +;; LaTeX must be an array of one of the following types: +;; LaTeX +;; LaTeX double-float +;; LaTeX single-float +;; LaTeX (unsigned-byte 32) +;; LaTeX (unsigned-byte 16) +;; LaTeX (unsigned-byte 8) +;; LaTeX (signed-byte 32) +;; LaTeX (signed-byte 16) +;; LaTeX (signed-byte 8) +;; LaTeX \" +;; LaTeX (declare (type (or #+signed-array (array (signed-byte 8)) +;; LaTeX #+signed-array (array (signed-byte 16)) +;; LaTeX #+signed-array (array (signed-byte 32)) +;; LaTeX (array (unsigned-byte 8)) +;; LaTeX (array (unsigned-byte 16)) +;; LaTeX (array (unsigned-byte 32)) +;; LaTeX (array single-float) +;; LaTeX (array double-float)) +;; LaTeX array) +;; LaTeX (optimize (speed 3) (safety 0)) +;; LaTeX (ext:optimize-interface (safety 3))) +;; LaTeX ;; with-array-data will get us to the actual data. However, because +;; LaTeX ;; the array could have been displaced, we need to know where the +;; LaTeX ;; data starts. +;; LaTeX (lisp::with-array-data ((data array) +;; LaTeX (start) +;; LaTeX (end)) +;; LaTeX (declare (ignore end)) +;; LaTeX ;; DATA is a specialized simple-array. Memory is laid out like this: +;; LaTeX ;; +;; LaTeX ;; byte offset Value +;; LaTeX ;; 0 type code (should be 70 for double-float vector) +;; LaTeX ;; 4 4 * number of elements in vector +;; LaTeX ;; 8 1st element of vector +;; LaTeX ;; ... ... +;; LaTeX ;; +;; LaTeX (let ((addr (+ 8 (logandc1 7 (kernel:get-lisp-obj-address data)))) +;; LaTeX (type-size (let ((type (array-element-type data))) +;; LaTeX (cond ((or (equal type '(signed-byte 8)) +;; LaTeX (equal type '(unsigned-byte 8))) +;; LaTeX 1) +;; LaTeX ((or (equal type '(signed-byte 16)) +;; LaTeX (equal type '(unsigned-byte 16))) +;; LaTeX 2) +;; LaTeX ((or (equal type '(signed-byte 32)) +;; LaTeX (equal type '(unsigned-byte 32))) +;; LaTeX 4) +;; LaTeX ((equal type 'single-float) +;; LaTeX 4) +;; LaTeX ((equal type 'double-float) +;; LaTeX 8) +;; LaTeX (t +;; LaTeX (error \"Unknown specialized array element type\")))))) +;; LaTeX (declare (type (unsigned-byte 32) addr) +;; LaTeX (optimize (speed 3) (safety 0) (ext:inhibit-warnings 3))) +;; LaTeX (system:int-sap (the (unsigned-byte 32) +;; LaTeX (+ addr (* type-size start))))))) +;; LaTeX \end{example} +;; LaTeX +;; LaTeX Assume we have the C function below that we wish to use: +;; LaTeX \begin{example} +;; LaTeX double dotprod(double* x, double* y, int n) +;; LaTeX \{ +;; LaTeX int k; +;; LaTeX double sum = 0; +;; LaTeX +;; LaTeX for (k = 0; k < n; ++k) \{ +;; LaTeX sum += x[k] * y[k]; +;; LaTeX \} +;; LaTeX \} +;; LaTeX \end{example} +;; LaTeX The following example generates two large arrays in Lisp, and calls the C +;; LaTeX function to do the desired computation. This would not have been +;; LaTeX possible using \code{malloc} or \code{make-alien} since we need about +;; LaTeX 16 MB of memory to hold the two arrays. +;; LaTeX \begin{example} +;; LaTeX (define-alien-routine \"dotprod\" double +;; LaTeX (x (* double-float) :in) +;; LaTeX (y (* double-float) :in) +;; LaTeX (n int :in)) +;; LaTeX +;; LaTeX (let ((x (make-array 1000000 :element-type 'double-float)) +;; LaTeX (y (make-array 1000000 :element-type 'double-float))) +;; LaTeX ;; Initialize X and Y somehow +;; LaTeX (let ((x-addr (system:int-sap (array-data-address x))) +;; LaTeX (y-addr (system:int-sap (array-data-address y)))) +;; LaTeX (dotprod x-addr y-addr 1000000))) +;; LaTeX \end{example} +;; LaTeX In this example, it may be useful to wrap the inner \code{let} +;; LaTeX expression in an \code{unwind-protect} that first turns off garbage +;; LaTeX collection and then turns garbage collection on afterwards. This will +;; LaTeX prevent garbage collection from moving \code{x} and \code{y} after we +;; LaTeX have obtained the (now erroneous) addresses but before the call to +;; LaTeX \code{dotprod} is made. +;; LaTeX + + +(defsection @calling-lisp-from-c (:title "Calling Lisp From C") + "SBCL supports the calling of Lisp functions using the C calling + convention. This is useful for both defining callbacks and for creating + an interface for calling into Lisp as a shared library directly from C. + + The DEFINE-ALIEN-CALLABLE macro wraps Lisp code and creates a C + foreign function which can be called with the C calling convention. + On x86-64 and ARM64, callbacks may receive and return structures by + value." + (define-alien-callable macro) + "The ALIEN-CALLABLE-FUNCTION function returns the foreign callable + value associated with any name defined by DEFINE-ALIEN-CALLABLE, so + that we can, for example, pass the callable value to C as a + callback." + (alien-callable-function function) + "The WITH-ALIEN-CALLABLE macro wraps Lisp code and establishes + local C foreign functions which can be called with the C calling + convention. This macro is handy for passing callbacks which close over + Lisp values into C." + (with-alien-callable macro) + "Note that the garbage collector moves objects, and won't be able to fix + up any references in C variables. There are three mechanisms for + coping with this: + + - SB-EXT:PURIFY moves all live Lisp data into static or read-only + areas such that it will never be moved (or freed) again in the + life of the Lisp session + + - SB-SYS:WITH-PINNED-OBJECTS is a macro which arranges for some set + of objects to be pinned in memory for the dynamic extent of its + body forms. On ports which use the generational garbage + collector (most, as of this writing) this affects exactly the + specified objects. On other ports it is implemented by turning off + GC for the duration (so could be said to have a whole-world + granularity). + + - Disable GC, using the SB-EXT:WITHOUT-GCING macro." + (@lisp-as-a-shared-library section)) + +(defsection @lisp-as-a-shared-library (:title "Lisp as a Shared Library") + "SBCL supports the use of Lisp as a shared library that can be used by + C programs using the DEFINE-ALIEN-CALLABLE interface. See the + :CALLABLE-EXPORTS argument of SB-EXT:SAVE-LISP-AND-DIE for how to + save the Lisp image in a way that allows a C program to initialize + the Lisp runtime and the exported symbols. When SBCL is built as a + library, it exposes the symbol `initialize_lisp` which can be used + in conjunction with a core initializing global symbols to foreign + callables as function pointers and with object code allocating those + symbols to initialize the runtime properly. The arguments to + `initialize_lisp` are the same as the arguments to the main `sbcl` + program. + + > _Note_: There is currently no way to run exit hooks or otherwise + > undo Lisp initialization gracefully from C.") + +(defsection @step-by-step-example-of-the-foreign-function-interface + (:title "Step-By-Step Example of the Foreign Function Interface") + "This section presents a complete example of an interface to a somewhat + complicated C function. + + Suppose you have the following C function which you want to be able + to call from Lisp in the file `test.c`: + + struct c_struct + { + int x; + char *s; + }; + + struct c_struct *c_function (i, s, r, a) + int i; + char *s; + struct c_struct *r; + int a[10]; + { + int j; + struct c_struct *r2; + + printf(\"i = %d\n\", i); + printf(\"s = %s\n\", s); + printf(\"r->x = %d\n\", r->x); + printf(\"r->s = %s\n\", r->s); + for (j = 0; j < 10; j++) printf(\"a[%d] = %d.\n\", j, a[j]); + r2 = (struct c_struct *) malloc (sizeof(struct c_struct)); + r2->x = i + 5; + r2->s = \"a C string\"; + return(r2); + }; + + It is possible to call this C function from Lisp using the file + `test.lisp` containing + + (cl:defpackage \"TEST-C-CALL\" (:use \"CL\" \"SB-ALIEN\" \"SB-C-CALL\")) + (cl:in-package \"TEST-C-CALL\") + + ;;; Define the record C-STRUCT in Lisp. + (define-alien-type nil + (struct c-struct + (x int) + (s c-string))) + + ;;; Define the Lisp function interface to the C routine. It returns a + ;;; pointer to a record of type C-STRUCT. It accepts four parameters: + ;;; I, an int; S, a pointer to a string; R, a pointer to a C-STRUCT + ;;; record; and A, a pointer to the array of 10 ints. + ;;; + ;;; The INLINE declaration eliminates some efficiency notes about heap + ;;; allocation of alien values. + (declaim (inline c-function)) + (define-alien-routine c-function + (* (struct c-struct)) + (i int) + (s c-string) + (r (* (struct c-struct))) + (a (array int 10))) + + ;;; a function which sets up the parameters to the C function and + ;;; actually calls it + (defun call-cfun () + (with-alien ((ar (array int 10)) + (c-struct (struct c-struct))) + (dotimes (i 10) ; Fill array. + (setf (deref ar i) i)) + (setf (slot c-struct 'x) 20) + (setf (slot c-struct 's) \"a Lisp string\") + + (with-alien ((res (* (struct c-struct)) + (c-function 5 \"another Lisp string\" (addr c-struct) ar))) + (format t \"~&back from C function~%\") + (multiple-value-prog1 + (values (slot res 'x) + (slot res 's)) + + ;; Deallocate result. (after we are done referring to it: + ;; \"Pillage, *then* burn.\") + (free-alien res))))) + + To execute the above example, it is necessary to compile the C + routine, e.g. with `cc -c test.c && ld -shared -o test.so test.o`. + In order to enable incremental loading with some linkers, you may + need to say `cc -G 0 -c test.c`. + + Once the C code has been compiled, you can start up Lisp and load it + in: `sbcl`. Lisp should start up with its normal prompt. + + Within Lisp, compile the Lisp file: + + (compile-file \"test.lisp\") + + This step can be done separately. You don't have to recompile every + time. + + Within Lisp, load the foreign object file to define the necessary + symbols: + + (load-shared-object \"test.so\") + + Now you can load the compiled Lisp (fasl) file into Lisp: + + (load \"test.fasl\") + + And once the Lisp file is loaded, you can call the Lisp routine + that sets up the parameters and calls the C function: + + (test-c-call::call-cfun) + + The C routine should print the following information to standard output: + + i = 5 + s = another Lisp string + r->x = 20 + r->s = a Lisp string + a[0] = 0. + a[1] = 1. + a[2] = 2. + a[3] = 3. + a[4] = 4. + a[5] = 5. + a[6] = 6. + a[7] = 7. + a[8] = 8. + a[9] = 9. + + After return from the C function, + the Lisp wrapper function should print the following output: + + back from C function + + And upon return from the Lisp wrapper function, + before the next prompt is printed, the + Lisp read-eval-print loop should print the following return values: + + 10 + \"a C string\"") diff --git a/contrib/sb-manual/doc/intro.lisp b/contrib/sb-manual/doc/intro.lisp new file mode 100644 index 000000000..46dac049f --- /dev/null +++ b/contrib/sb-manual/doc/intro.lisp @@ -0,0 +1,500 @@ +(in-package :sb-manual) + +(defsection @introduction (:title "Introduction") + "SBCL is a mostly-conforming implementation of the ANSI Common Lisp + standard. This manual focuses on behavior which is specific to SBCL, + not on behavior which is common to all implementations of ANSI Common + Lisp." + (@ansi-conformance section) + (@extensions section) + (@idiosyncrasies section) + (@development-tools section) + (@more-sbcl-information section) + (@more-common-lisp-information section) + (@history-and-implementation-of-sbcl section)) + +(defsection @ansi-conformance (:title "ANSI Conformance") + "Essentially every type of non-conformance is considered a bug. (The + exceptions involve internal inconsistencies in the standard.) See + @REPORTING-BUGS. + + - PROG2 returns the primary value of its second form, as + specified in the _Arguments and Values_ section of the + specification for that operator, not that of its first form, as + specified in the _Description_. + + - The STRING type is considered to be the union of all types + `(ARRAY C (SIZE))` for all non-`NIL` subtypes `C` of CHARACTER, + excluding arrays specialized to the empty type. + + - The `:ORDER` long form option in DEFINE-METHOD-COMBINATION method + group specifiers accepts the value NIL as well as + :MOST-SPECIFIC-FIRST and :MOST-SPECIFIC-LAST, in order to allow + programmers to declare that the order of methods playing that role + in the method combination does not matter.") + +;;; FIXME: Document SERVE-EVENT? +(defsection @extensions (:title "Extensions") + "SBCL comes with numerous extensions, some in core and some in modules + loadable with REQUIRE. Unfortunately, not all of these extensions + have proper documentation yet. + + - __System Definition Tool:__ ASDF is a flexible and popular + protocol-oriented system definition tool by Daniel Barlow. + + - __Foreign Function Interface:__ The `SB-ALIEN` package allows + interfacing with C-code, loading shared object files, etc. See + @FOREIGN-FUNCTION-INTERFACE. + + @SB-GROVEL can be used to partially automate generation of + foreign function interface definitions. + + - __Recursive Event Loop:__ SBCL provides a recursive event + loop (`SERVE-EVENT`) for doing non-blocking IO on multiple streams + without using threads. + + - __Timeouts and Deadlines:__ SBCL allows restricting the execution + time of individual operations or parts of a computation using + :TIMEOUT arguments to certain blocking operations, synchronous + timeouts and asynchronous timeouts. The latter two affect operations + without explicit timeout support (such as standard functions and + macros). See @TIMEOUTS-AND-DEADLINES. + + - __Metaobject Protocol:__ The `SB-MOP` package provides an + implementation of the metaobject protocol for the Common Lisp + Object System as described in _The Art of the Metaobject Protocol_ + by Kiczales et al. + + - __Extensible Sequences:__ SBCL allows users to define subclasses + of the SEQUENCE class. See @EXTENSIBLE-SEQUENCES. + + - __Native Threads:__ SBCL has native threads on numerous platforms, + capable of taking advantage of SMP on multiprocessor machines. See + @THREADING. + + - __Network Interface:__ The `SB-BSD-SOCKETS` module is a low-level + networking interface, providing both TCP and UDP sockets. See + @NETWORKING. + + - __Introspective Facilities:__ The @SB-INTROSPECT module offers + numerous introspective extensions, including access to function + lambda-lists and a cross referencing facility. + + - __Operating System Interface:__ The `SB-EXT` package contains a + number of functions for running external processes, accessing + environment variables, etc. + + The @SB-POSIX module provides a lispy interface to standard + POSIX facilities. + + - __Extensible Streams:__ The package `SB-GRAY` provides an + implementation of @GRAY-STREAMS. + + The @SB-SIMPLE-STREAMS module is an implementation of the Simple + Streams API proposed by Franz Inc. + + - __Profiling:__ The `SB-PROFILE` package provides an exact, + per-function @DETERMINISTIC-PROFILER. + + The `SB-SPROF` module is SBCL's @STATISTICAL-PROFILER, capable + of call-graph generation and instruction level profiling, which + also supports allocation profiling. + + - __Customization Hooks:__ SBCL contains a number of extra-standard + customization hooks that can be used to tweak the behaviour of the + system. See @CUSTOMIZATION-HOOKS-FOR-USERS. + + - __sb-aclrepl:__ The @SB-ACLREPL module provides an Allegro-style + toplevel for SBCL, as an alternative to the classic CMUCL-style + one. + + - __CLTL2 Compatibility Layer:__ The SB-CLTL2 module provides + SB-CLTL2:COMPILER-LET and environment access functionality + described in _Common Lisp The Language, 2nd Edition_ which were + removed from the language during the ANSI standardization process. + + - __Executable Delivery:__ The :EXECUTABLE argument to + SB-EXT:SAVE-LISP-AND-DIE can produce a \"standalone\" executable + containing both an image of the current Lisp session and an SBCL + runtime. + + - __Bitwise Rotation:__ The @SB-ROTATE-BYTE module provides an + efficient primitive for bitwise rotation of integers, an operation + required by e.g. numerous cryptographic algorithms but not + available as a primitive in ANSI Common Lisp. + + - __Test Harness:__ The `SB-RT` module is a simple yet attractive + regression and unit-test framework. + + - __MD5 Sums:__ The @SB-MD5 module provides an implementation of the + MD5 message digest algorithm for Common Lisp, using the modular + arithmetic optimizations provided by SBCL.") + +(defsection @idiosyncrasies (:title "Idiosyncrasies") + "The information in this section describes some of the ways that SBCL + deals with choices that the ANSI standard leaves to the + implementation." + (@declarations section) + (@fasl-format section) + (@compiler-only-implementation section) + (@defining-constants section) + (@style-warnings section)) + +(defsection @declarations (:title "Declarations") + "Declarations are generally treated as assertions. This general + principle, and its implications, and the bugs which still keep the + compiler from quite satisfying this principle, are discussed in + @DECLARATIONS-AS-ASSERTIONS.") + +(defsection @fasl-format (:title "FASL format") + "SBCL fasl-format is binary compatible only with the exact SBCL version + it was generated with. While this is obviously suboptimal, it has + proven more robust than trying to maintain fasl compatibility across + versions: accidentally breaking things is far too easy, and can lead + to hard to diagnose bugs. + + The following snippet handles fasl recompilation automatically for + ASDF-based systems, and makes a good candidate for inclusion in the + user or system initialization file (see @INITIALIZATION-FILES). + + (require :asdf) + + ;;; If a fasl was stale, try to recompile and load (once). + (defmethod asdf:perform :around ((o asdf:load-op) + (c asdf:cl-source-file)) + (handler-case (call-next-method o c) + ;; If a fasl was stale, try to recompile and load (once). + (sb-ext:invalid-fasl () + (asdf:perform (make-instance 'asdf:compile-op) c) + (call-next-method))))") + +(defsection @compiler-only-implementation + (:title "Compiler-only Implementation") + "SBCL is essentially a compiler-only implementation of Common Lisp. + That is, for all but a few special cases, EVAL creates a lambda + expression, calls COMPILE on the lambda expression to create a + compiled function, and then calls FUNCALL on the resulting function + object. A more traditional interpreter is also available on default + builds; it is usually only called internally. This is explicitly + allowed by the ANSI standard but leads to some oddities; e.g. at + default settings, FUNCTIONP and COMPILED-FUNCTION-P are equivalent, + and they collapse into the same function when SBCL is built without + the interpreter.") + +(defsection @defining-constants (:title "Defining Constants") + "SBCL is quite strict about ANSI's definition of DEFCONSTANT. + ANSI says that doing DEFCONSTANT of the same symbol more than once + is undefined unless the new value is EQL to the old value. + Conforming to this specification is a nuisance when the \"constant\" + value is only constant under some weaker test like STRING= or EQUAL. + + It's especially annoying because, in SBCL, DEFCONSTANT takes effect + not only at load time but also at compile time, so that just + compiling and loading reasonable code like + + (defconstant +foobyte+ '(1 4)) + + runs into this undefined behavior. Many implementations of Common + Lisp try to help the programmer around this annoyance by silently + accepting the undefined code and trying to do what the programmer + probably meant. + + SBCL instead treats the undefined behavior as an error. Often such + code can be rewritten in portable ANSI Common Lisp which has the + desired behavior. E.g., the code above can be given an exactly + defined meaning by replacing DEFCONSTANT either with DEFPARAMETER or + with a customized macro which does the right thing, e.g. + + (defmacro define-constant (name value &optional doc) + `(defconstant ,name (if (boundp ',name) (symbol-value ',name) ,value) + ,@(when doc (list doc)))) + + or possibly along the lines of the SB-INT:DEFCONSTANT-EQX macro used + internally in the implementation of SBCL itself. In circumstances + where this is not appropriate, the programmer can handle the + condition type SB-EXT:DEFCONSTANT-UNEQL and choose either the + CONTINUE restart or ABORT restart as appropriate.") + +(defsection @style-warnings (:title "Style Warnings") + "SBCL gives style warnings about various kinds of perfectly legal code, + e.g. + + - multiple DEFUNs of the same symbol in different units; + + - special variables not named in the conventional `*foo*` style, and + lexical variables unconventionally named in the `*FOO*` style. + + This causes friction with people who point out that other ways of + organizing code (especially avoiding the use of DEFGENERIC) are just + as aesthetically stylish. However, these warnings should be read not + as _warning, bad aesthetics detected, you have no style_ but as + _warning, this style keeps the compiler from understanding the code + as well as you might like_. That is, unless the compiler warns about + such conditions, there's no way for the compiler to warn about some + programming errors which would otherwise be easy to + overlook. (Related bug: The warning about multiple DEFUNs is + pointlessly annoying when you compile and then load a function + containing DEFUN wrapped in EVAL-WHEN, and ideally should be + suppressed in that case, but still isn't as of SBCL 0.7.6.)") + +(defsection @development-tools (:title "Development Tools") + (@editor-integration section) + (@language-reference section) + (@generating-executables section)) + +(defsection @editor-integration (:title "Editor Integration") + "Though SBCL can be used running \"bare\", the recommended mode of + development is with an editor connected to SBCL, supporting not + only basic lisp editing (paren-matching, etc), but providing among + other features an integrated debugger, interactive compilation, and + automated documentation lookup. + + Currently _SLIME_ (Superior Lisp Interaction Mode for Emacs) + together with Emacs is recommended for use with SBCL, though other + options exist as well. Historically, the ILISP package at + <http://ilisp.cons.org/> provided similar functionality, but it does + not support modern SBCL versions. + + SLIME can be downloaded from <https://slime.common-lisp.dev/>.") + +(defsection @language-reference (:title "Language Reference") + "_\\CLHS_ (Common Lisp Hyperspec) is a hypertext version of the ANSI + standard, made freely available by LispWorks -- an invaluable + reference. + + See <https://www.lispworks.com/documentation/HyperSpec/Front/index.htm>.") + +(defsection @generating-executables (:title "Generating Executables") + "SBCL can generate stand-alone executables. The generated executables + include the SBCL runtime itself, so no restrictions are placed on + program functionality. For example, a deployed program can call + COMPILE and LOAD, which requires the compiler to be present in the + executable. For further information, SB-EXT:SAVE-LISP-AND-DIE.") + +(defsection @more-sbcl-information (:title "More SBCL Information") + (@sbcl-homepage section) + (@online-documentation section) + (@additional-documentation-files section) + (@internals-documentation section)) + +(defsection @sbcl-homepage (:title "SBCL Homepage") + "The SBCL website at <http://www.sbcl.org/> has some general + information, plus links to mailing lists devoted to SBCL, and to + archives of these mailing lists. Subscribing to the mailing lists + `sbcl-help` and `sbcl-announce` is recommended: both are fairly + low-volume, and help you keep abreast with SBCL development.") + +(defsection @online-documentation (:title "Online Documentation") + "Documentation for non-ANSI extensions for various commands is + available online from the SBCL executable itself. The extensions for + functions which have their own command prompts (e.g. the debugger, + and INSPECT) are documented in text available by typing `help` at + their command prompts. The extensions for functions which don't have + their own command prompt (such as TRACE) are described in their + documentation strings, unless your SBCL was compiled with an option + not to include documentation strings, in which case the + documentation strings are only readable in the source code.") + +(defsection @additional-documentation-files + (:title "Additional Documentation Files") + "Besides this user manual both SBCL source and binary distributions + include some other SBCL-specific documentation files, which should + be installed along with this manual on your system, e.g. in + `/usr/local/share/doc/sbcl/`. + + - `COPYING`: Licence and copyright summary. + + - `CREDITS`: Authorship information on various parts of SBCL. + + - `INSTALL`: Covers installing SBCL from both source and binary + distributions on your system, and also has some installation + related troubleshooting information. + + - `NEWS`: Summarizes changes between various SBCL versions.") + +(defsection @internals-documentation (:title "Internals Documentation") + "If you're interested in the development of the SBCL system itself, + then subscribing to `sbcl-devel` is a good idea. + + SBCL internals documentation -- besides comments in the source -- is + available in the Web Archive: + + <https://web.archive.org/web/20120814000933/http://sbcl-internals.cliki.net/index>. + + Some low-level information describing the programming details of the + conversion from CMUCL to SBCL is available in the + `doc/FOR-CMUCL-DEVELOPERS` file.") + +(defsection @more-common-lisp-information + (:title "More Common Lisp Information") + (@internet-community section) + (@third-party-libraries section) + (@common-lisp-books section)) + +(defsection @internet-community (:title "Internet Community") + "IRC channels on <https://libera.chat/>: + + - `#common-lisp`: \"Common Lisp, the #1=(programmable . #1#) + programming language\" + + - `#lispcafe`: \"The Lisp Cafรฉ; sit down, have a drink, chat about + anything, and enjoy your stay. | <https://www.cliki.net/lispcafe> | + Be insuperable to each other\". + + - `#sbcl`: \"Steel Bank Common Lisp Dev Hangout\" + + You can use <https://web.libera.chat> or a normal IRC client. + + Also, see <https://www.reddit.com/r/Common_Lisp/>, as well as + <https://www.lisp.org> and <https://cliki.net>, which contain + numerous pointers places in the net where lispers talks shop.") + +(defsection @third-party-libraries (:title "Third-party Libraries") + "For a wealth of information about free Common Lisp libraries and tools + we recommend checking out _CLiki_: <https://cliki.net/>. + + The most popular library manager is Quicklisp: + <https://www.quicklisp.org/beta/>.") + +(defsection @common-lisp-books (:title "Common Lisp Books") + "If you're not a programmer and you're trying to learn, many + introductory Lisp books are available. However, we don't have any + standout favorites. + + If you are an experienced programmer in other languages but need to + learn about Common Lisp, some books stand out: + + - Practical Common Lisp, by Peter Seibel + + An excellent introduction to the language, covering both the + basics and \"advanced topics\" like macros, CLOS, and packages. + Available both in print format and on the web: + <https://gigamonkeys.com/book/>. + + - Paradigms Of Artificial Intelligence Programming, by Peter Norvig + + Good information on general Common Lisp programming, and many + nontrivial examples. Whether or not your work is AI, it's a very + good book to look at. + + - On Lisp, by Paul Graham + + An in-depth treatment of macros, but not recommended as a first + Common Lisp book, since it is slightly pre-ANSI so you need to + be on your guard against non-standard usages, and since it + doesn't really even try to cover the language as a whole, + focusing solely on macros. Downloadable from + <https://www.paulgraham.com/onlisp.html>. + + - Object-Oriented Programming In Common Lisp, by Sonya Keene + + With the exception of _Practical Common Lisp_, most introductory + books don't emphasize CLOS. This one does. Even if you're very + knowledgeable about object oriented programming in the abstract, + it's worth looking at this book if you want to do any OO in + Common Lisp. Some abstractions in CLOS (especially multiple + dispatch) go beyond anything you'll see in most OO systems, and + there are a number of lesser differences as well. This book + tends to help with the culture shock. + + - Art Of Metaobject Programming, by Gregor Kiczales et al. + + Currently the prime source of information on the Common Lisp + Metaobject Protocol, which is supported by SBCL. Section + 2 (Chapters 5 and 6) are freely available at + <http://mop.lisp.se/www.alu.org/mop/>.") + +(defsection @history-and-implementation-of-sbcl + (:title "History and Implementation of SBCL") + "You can work productively with SBCL without knowing or understanding + anything about where it came from, how it is implemented, or how it + extends the ANSI Common Lisp standard. However, a little knowledge + can be helpful in order to understand error messages, to + troubleshoot problems, to understand why some parts of the system + are better debugged than others, and to anticipate which known bugs, + known performance problems, and missing extensions are likely to be + fixed, tuned, or added. + + SBCL is descended from CMUCL, which is itself descended from Spice + Lisp, including early implementations for the Mach operating system on + the IBM RT, back in the 1980s. Some design decisions from that time are + still reflected in the current implementation: + + - The system expects to be loaded into a fixed-at-compile-time + location in virtual memory, and also expects the location of all + of its heap storage to be specified at compile time. + + - The system overcommits memory, allocating large amounts of address + space from the system (often more than the amount of virtual + memory available) and then failing if it ends up using too much of + the allocated storage. + + - The system is implemented as a C program which is responsible for + supplying low-level services and loading a Lisp `.core` file. + + SBCL also inherited some newer architectural features from CMUCL. + The most important is that on some architectures it has a + generational garbage collector (GC), which has various + implications (mostly good) for performance. These are discussed in + another chapter, @EFFICIENCY. + + SBCL has diverged from CMUCL in that SBCL is now essentially a + compiler-only implementation of Common Lisp. This is a change in + implementation strategy, taking advantage of the freedom \"any of + these facilities might share the same execution strategy\" + guaranteed in CLHS `3.1` (Evaluation). It does not mean SBCL can't + be used interactively, and in fact the change is largely invisible + to the casual user, since SBCL still can and does execute code + interactively by compiling it on the fly. (It is visible if you know + how to look, like using COMPILED-FUNCTION-P; and it is visible in + the way that SBCL doesn't have many bugs which behave differently in + interpreted code than in compiled code.) What it means is that in + SBCL, the EVAL function only truly \"interprets\" a few easy kinds + of forms, such as symbols which are BOUNDP. More complicated forms + are evaluated by calling COMPILE and then calling FUNCALL on the + returned result. + + The direct ancestor of SBCL is the x86 port of CMUCL. This port was in + some ways the most cobbled-together of all the CMUCL ports, since a + number of strange changes had to be made to support the register-poor + x86 architecture. Some things (like tracing and debugging) do not work + particularly well there. SBCL should be able to improve in these areas + (and has already improved in some other areas), but it takes a while. + + On the x86 SBCL -- like the x86 port of CMUCL -- uses a + _conservative_ GC. This means that it doesn't maintain a strict + separation between tagged and untagged data, instead treating some + untagged data (e.g. raw floating point numbers) as possibly-tagged + data and so not collecting any Lisp objects that they point to. This + has some negative consequences for average time efficiency (though + possibly no worse than the negative consequences of trying to + implement an exact GC on a processor architecture as register-poor + as the X86) and also has potentially unlimited consequences for + worst-case memory efficiency. In practice, conservative garbage + collectors work reasonably well, not getting anywhere near the worst + case. But they can occasionally cause odd patterns of memory usage. + + The fork from CMUCL was based on a major rewrite of the system + bootstrap process. CMUCL has for many years tolerated a very unusual + \"build\" procedure which doesn't actually build the complete system + from scratch, but instead progressively overwrites parts of a + running system with new versions. This quasi-build procedure can + cause various bizarre bootstrapping hangups, especially when a major + change is made to the system. It also makes the connection between + the current source code and the current executable more tenuous than + in other software systems -- it's easy to accidentally build a CMUCL + system containing characteristics not reflected in the current + version of the source code. + + Other major changes since the fork from CMUCL include: + + - SBCL has removed many CMUCL extensions, (e.g. IP networking, + remote procedure call, Unix system interface, and X11 interface) + from the core system. Most of these are available as contributed + modules (distributed with SBCL) or third-party modules instead. + + - SBCL has deleted or deprecated some nonstandard features and code + complexity which helped efficiency at the price of + maintainability. For example, the SBCL compiler no longer + implements memory pooling internally (and so is simpler and more + maintainable, but generates more garbage and runs more slowly).") diff --git a/contrib/sb-manual/doc/networking.lisp b/contrib/sb-manual/doc/networking.lisp new file mode 120000 index 000000000..9003310a9 --- /dev/null +++ b/contrib/sb-manual/doc/networking.lisp @@ -0,0 +1 @@ +../../sb-bsd-sockets/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/package-locks.lisp b/contrib/sb-manual/doc/package-locks.lisp new file mode 100644 index 000000000..996d53865 --- /dev/null +++ b/contrib/sb-manual/doc/package-locks.lisp @@ -0,0 +1,276 @@ +(in-package :sb-manual) + +(defsection @package-locks (:title "Package Locks") + "None of the following sections apply to SBCL built without package + locking support. + + The interface described here is experimental: incompatible changes + in future SBCL releases are possible, even expected: the concept of + _implementation packages_ and the associated operators may be + renamed; more operations (such as naming restarts or catch tags) may + be added to the list of operations violating package locks." + (@package-lock-concepts section) + (@package-lock-dictionary section)) + +(defsection @package-lock-concepts (:title "Package Lock Concepts") + "Package locks protect against unintentional modifications of a package: + they provide similar protection to user packages as is mandated to + `COMMON-LISP` package by the ANSI specification. They are not, and + should not be used as, a security measure. + + Newly created packages are by default unlocked (see the :LOCK option + to DEFPACKAGE). + + The package `COMMON-LISP` and SBCL internal implementation packages + are locked by default, including `SB-EXT`. + + It may be beneficial to lock `COMMON-LISP-USER` as well, to ensure + that various libraries don't pollute it without asking, but this is + not currently done by default." + (@implementation-packages section) + (@package-lock-violations section) + (@package-locks-in-compiled-code section) + (@operations-violating-package-locks section)) + +(defsection @implementation-packages (:title "Implementation Packages") + "Each package has a list of associated implementation packages. A + locked package, and the symbols whose home package it is, can be + modified without violating package locks only when *PACKAGE* is + bound to one of the implementation packages of the locked package. + + Unless explicitly altered by DEFPACKAGE, + SB-EXT:ADD-IMPLEMENTATION-PACKAGE, or + SB-EXT:REMOVE-IMPLEMENTATION-PACKAGE each package is its own + (only) implementation package.") + +(defsection @package-lock-violations (:title "Package Lock Violations") + (@lexical-bindings-and-declarations section) + (@other-operations section)) + +(defsection @lexical-bindings-and-declarations + (:title "Lexical Bindings and Declarations") + "Lexical bindings or declarations that violate package locks cause a + compile-time warning, and a runtime PROGRAM-ERROR when the form that + violates package locks would be executed. + + A complete listing of operators affect by this is: LET, LET*, FLET, + LABELS, MACROLET, and SYMBOL-MACROLET, DECLARE. + + Package locks affecting both lexical bindings and declarations can + be disabled locally with the SB-EXT:DISABLE-PACKAGE-LOCKS + declaration, and re-enabled with the SB-EXT:ENABLE-PACKAGE-LOCKS + declaration. + + Example: + + (in-package :locked) + + (defun foo () ...) + + (defmacro with-foo (&body body) + `(locally (declare (disable-package-locks locked:foo)) + (flet ((foo () ...)) + (declare (enable-package-locks locked:foo)) ; re-enable for body + ,@body)))") + +(defsection @other-operations (:title "Other Operations") + "If an non-lexical operation violates a package lock, a continuable + error that is of a subtype of SB-EXT:PACKAGE-LOCK-VIOLATION + (subtype of PACKAGE-ERROR) is signalled when the operation is + attempted. + + Additional restarts may be established for continuable package lock + violations for interactive use. + + The actual type of the error depends on circumstances that caused + the violation: operations on packages signal errors of type + SB-EXT:PACKAGE-LOCKED-ERROR, and operations on symbols signal errors + of type SB-EXT:SYMBOL-PACKAGE-LOCKED-ERROR.") + +(defsection @package-locks-in-compiled-code + (:title "Package Locks in Compiled Code") + "If file-compiled code contains interned symbols, then loading that + code into an image without the said symbols will not cause a package + lock violation, even if the packages in question are locked. + + With the exception of interned symbols, behaviour is unspecified if + package locks affecting compiled code are not the same during + loading of the code or execution. + + Specifically, code compiled with packages unlocked may or may not + fail to signal package-lock-violations even if the packages are + locked at runtime, and code compiled with packages locked may or may + not signal spurious package-lock-violations at runtime even if the + packages are unlocked. + + In practice all this means that package-locks have a negligible + performance penalty in compiled code as long as they are not + violated.") + +(defsection @operations-violating-package-locks + (:title "Operations Violating Package Locks") + (@operations-on-packages section) + (@operations-on-symbols section)) + +(defsection @operations-on-packages (:title "Operations on Packages") + "The following actions cause a package lock violation if the package + operated on is locked, and *PACKAGE* is not an implementation + package of that package, and the action would cause a change in the + state of the package (so e.g. exporting already external symbols is + never a violation). Package lock violations caused by these + operations signal errors of type SB-EXT:PACKAGE-LOCKED-ERROR. + + - Shadowing a symbol in a package. + + - Importing a symbol to a package. + + - Uninterning a symbol from a package. + + - Exporting a symbol from a package. + + - Unexporting a symbol from a package. + + - Changing the packages used by a package. + + - Renaming a package. + + - Deleting a package. + + - Adding a new package local nickname to a package. + + - Removing an existing package local nickname to a package.") + +(defsection @operations-on-symbols (:title "Operations on Symbols") + "Following actions cause a package lock violation if the home package + of the symbol operated on is locked, and *PACKAGE* is not an + implementation package of that package. Package lock violations + caused by these action signal errors of type + SB-EXT:SYMBOL-PACKAGE-LOCKED-ERROR. + + These actions cause only one package lock violation per lexically + apparent violated package. + + Example: + + + ;;; Packages FOO and BAR are locked. + ;;; + ;;; Two lexically apparent violated packages: exactly two + ;;; package-locked-errors will be signalled. + + (defclass foo:point () + ((x :accessor bar:x) + (y :accessor bar:y))) + + - Binding or altering its value lexically or dynamically, or + establishing it as a symbol-macro. + + Exceptions: + + - If the symbol is not defined as a constant, global + symbol-macro or a global dynamic variable, it may be lexically + bound or established as a local symbol macro. + + - If the symbol is defined as a global dynamic variable, it may + be assigned or bound. + + - Defining, undefining, or binding it, or its setf name as a + function. + + Exceptions: + + - If the symbol is not defined as a function, macro, or special + operator it and its setf name may be lexically bound as a + function. + + - Defining, undefining, or binding it as a macro or compiler macro. + + Exceptions: + + - If the symbol is not defined as a function, macro, or special + operator it may be lexically bound as a macro. + + - Defining it as a type specifier or structure. + + - Defining it as a declaration with a declaration proclamation. + + - Declaring or proclaiming it special. + + - Declaring or proclaiming its type or ftype. + + Exceptions: + + - If the symbol may be lexically bound, the type of that binding + may be declared. + + - If the symbol may be lexically bound as a function, the ftype + of that binding may be declared. + + - Defining a setf expander for it. + + - Defining it as a method combination type. + + - Using it as the CLASS-NAME argument to (SETF FIND-CLASS). + + - Defining it as a hash table test using SB-EXT:DEFINE-HASH-TABLE-TEST.") + +(defsection @package-lock-dictionary (:title "Package Lock Dictionary") + "- [__declaration__] SB-EXT:DISABLE-PACKAGE-LOCKS + + Syntax: `(SB-EXT:DISABLE-PACKAGE-LOCKS &REST SYMBOLS)` + + Disables package locks affecting the named symbols during + compilation in the lexical scope of the declaration. Disabling + locks on symbols whose home package is unlocked, or disabling an + already disabled lock, has no effect. + + - [__declaration__] SB-EXT:ENABLE-PACKAGE-LOCKS + + Syntax: `(SB-EXT:ENABLE-PACKAGE-LOCKS &REST SYMBOLS)` + + Re-enables package locks affecting the named symbols during + compilation in the lexical scope of the declaration. Enabling + locks that were not first disabled with + SB-EXT:DISABLE-PACKAGE-LOCKS declaration, or enabling locks that + are already enabled has no effect." + + (sb-ext:package-lock-violation condition) + (sb-ext:package-locked-error condition) + (sb-ext:symbol-package-locked-error condition) + (sb-ext:package-locked-error-symbol function) + (sb-ext:package-locked-p function) + (sb-ext:lock-package function) + (sb-ext:unlock-package function) + (sb-ext:package-implemented-by-list function) + (sb-ext:package-implements-list function) + (sb-ext:add-implementation-package function) + (sb-ext:remove-implementation-package function) + (sb-ext:without-package-locks macro) + (sb-ext:with-unlocked-packages macro) + + "The DEFPACKAGE options are extended to include the following: + + - :LOCK `<boolean>` (defaults to NIL) + + If the argument to :LOCK is T, the package is locked, else it is + unlocked. Existing package are also affected. + + - :IMPLEMENT `<package-designator>*` + + The package is added as an implementation package to the + packages named. If :IMPLEMENT is not provided, it defaults to + the package itself. + + Example: + + (defpackage \"FOO\" (:export \"BAR\") (:lock t) (:implement)) + (defpackage \"FOO-INT\" (:use \"FOO\") (:implement \"FOO\" \"FOO-INT\")) + + ;;; is equivalent to + + (defpackage \"FOO\") (:export \"BAR\")) + (lock-package \"FOO\") + (remove-implementation-package \"FOO\" \"FOO\") + + (defpackage \"FOO-INT\" (:use \"BAR\")) + (add-implementation-package \"FOO-INT\" \"FOO\")") diff --git a/contrib/sb-manual/doc/pathnames.lisp b/contrib/sb-manual/doc/pathnames.lisp new file mode 100644 index 000000000..c0e4ff3ba --- /dev/null +++ b/contrib/sb-manual/doc/pathnames.lisp @@ -0,0 +1,152 @@ +(in-package :sb-manual) + +(defsection @pathnames (:title "Pathnames") + (@lisp-pathnames section) + (@native-filenames section)) + +(defsection @lisp-pathnames (:title "Lisp Pathnames") + "There are many aspects of ANSI Common Lisp's pathname support + which are implementation-defined and so need documentation." + (@home-directory-specifiers section) + (@the-sys-logical-pathname-host section)) + +;; FIXME: as a matter of ANSI conformance, we are required to document +;; implementation-defined stuff, which for pathnames (chapter 19 of CLtS) +;; includes: +;; +;; * Otherwise, the parsing of thing is implementation-defined. +;; (PARSE-NAMESTRING) +;; +;; * If thing contains an explicit host name and no explicit device name, +;; then it is implementation-defined whether parse-namestring will supply +;; the standard default device for that host as the device component of +;; the resulting pathname. (PARSE-NAMESTRING) +;; +;; * The specific nature of the search is implementation-defined. +;; (LOAD-LOGICAL-PATHNAME-TRANSLATIONS) +;; +;; * Any additional elements are implementation-defined. +;; (LOGICAL-PATHNAME-TRANSLATIONS) +;; +;; * The matching rules are implementation-defined but should be consistent +;; with directory. (PATHNAME-MATCH-P) +;; +;; * Any such additional translations are implementation-defined. +;; (TRANSLATE-LOGICAL-PATHNAMES) +;; +;; * ...or an implementation-defined portion of a component... +;; (TRANSLATE-PATHNAME) +;; +;; * The portion of source that is copied into the resulting pathname is +;; implementation-defined. (TRANSLATE-PATHNAME) +;; +;; * During the copying of a portion of source into the resulting +;; pathname, additional implementation-defined translations of case or +;; file naming conventions might occur. (TRANSLATE-PATHNAME) +;; +;; * In general, the syntax of namestrings involves the use of +;; implementation-defined conventions. (19.1.1) +;; +;; * The nature of the mapping between structure imposed by pathnames and +;; the structure, if any, that is used by the underlying file system is +;; implementation-defined. (19.1.2) +;; +;; * The mapping of the pathname components into the concepts peculiar to +;; each file system is implementation-defined. (19.1.2) +;; +;; * Whether separator characters are permitted as part of a string in a +;; pathname component is implementation-defined; (19.2.2.1.1) +;; +;; * Whether a value of :unspecific is permitted for any component on any +;; given file system accessible to the implementation is +;; implementation-defined. (19.2.2.2.3) +;; +;; * Other symbols and integers have implementation-defined meaning. +;; (19.2.2.4.6) + +(defsection @home-directory-specifiers (:title "Home Directory Specifiers") + "SBCL accepts the keyword :HOME and a list of the form + `(:HOME` `\"username\")` as a directory component immediately + following :ABSOLUTE. + + :HOME is represented in namestrings by `~/` and `(:HOME` + `\"username\")` by `~username/` at the start of the namestring. + Tilde-characters elsewhere in namestrings represent themselves. + + Home directory specifiers are resolved to home directory of the + current or specified user by SB-EXT:NATIVE-NAMESTRING, which is used + by the implementation to translate pathnames before passing them on + to operating system specific routines. + + Using `(:HOME` `\"user\")` form on Windows signals an error.") + +(defsection @the-sys-logical-pathname-host + (:title "The SYS Logical Pathname Host") + ;; The existence and meaning of SYS: logical pathnames is + ;; implementation-defined (CLHS 19.3.1.1.1). + "The logical pathname host named by `\"SYS\"` exists in SBCL. + Its LOGICAL-PATHNAME-TRANSLATIONS may be set by the site or the user + applicable to point to the locations of the system's sources; in + particular, the core system's source files match the logical + pathname `\"SYS:SRC;**;*.*.*\"`, and the contributed modules' source + files match `\"SYS:CONTRIB;**;*.*.*\"`." + (sb-ext:set-sbcl-source-location function)) + +(defsection @native-filenames (:title "Native Filenames") + "In some circumstances, what is wanted is a Lisp pathname object which + corresponds to a string produced by the Operating System. In this + case, some of the default parsing rules are inappropriate: most + filesystems do not have a native understanding of wild pathnames; + such functionality is often provided by shells above the OS, often + in mutually-incompatible ways. + + To allow the user to deal with this, the following functions are + provided: SB-EXT:PARSE-NATIVE-NAMESTRING and SB-EXT:NATIVE-PATHNAME + return the closest equivalent Lisp pathname to a given string + (appropriate for the Operating System), while + SB-EXT:NATIVE-NAMESTRING converts a non-wild pathname designator to + the equivalent native namestring, if possible. Some Lisp pathname + concepts (such as the :BACK directory component) have no direct + equivalents in most Operating Systems; the behaviour of + SB-EXT:NATIVE-NAMESTRING is unspecified if an inappropriate pathname + designator is passed to it. Additionally, note that conversion from + pathname to native filename and back to pathname should not be + expected to preserve equivalence under EQUAL." + (sb-ext:parse-native-namestring function) + (sb-ext:native-pathname function) + (sb-ext:native-namestring function) + "Because some file systems permit the names of directories to be + expressed in multiple ways, it is occasionally necessary to parse a + native file name as a directory name or to produce a native file + name that names a directory as a file. For these cases, + PARSE-NATIVE-NAMESTRING accepts the keyword argument + :AS-DIRECTORY to force a filename to parse as a directory, and + SB-EXT:NATIVE-NAMESTRING accepts the keyword argument :AS-FILE + to force a pathname to unparse as a file. For example, + + ; On Unix, the directory \"/tmp/\" can be denoted by \"/tmp/\" or \"/tmp\". + ; Under the default rules for native filenames, these parse and + ; unparse differently. + (defvar *p*) + (setf *p* (parse-native-namestring \"/tmp/\")) => #P\"/tmp/\" + (pathname-name *p*) => NIL + (pathname-directory *p*) => (:ABSOLUTE \"tmp\") + (native-namestring *p*) => \"/tmp/\" + + (setf *p* (parse-native-namestring \"/tmp\")) => #P\"/tmp\" + (pathname-name *p*) => \"tmp\" + (pathname-directory *p*) => (:ABSOLUTE) + (native-namestring *p*) => \"/tmp\" + + ; A non-NIL AS-DIRECTORY argument to PARSE-NATIVE-NAMESTRING forces + ; both the second string to parse the way the first does. + (setf *p* (parse-native-namestring \"/tmp\" + nil *default-pathname-defaults* + :as-directory t)) => #P\"/tmp/\" + (pathname-name *p*) => NIL + (pathname-directory *p*) => (:ABSOLUTE \"tmp\") + + ; A non-NIL AS-FILE argument to NATIVE-NAMESTRING forces the pathname + ; parsed from the first string to unparse as the second string. + (setf *p* (parse-native-namestring \"/tmp/\")) => #P\"/tmp/\" + (native-namestring *p* :as-file t) => \"/tmp\"") diff --git a/contrib/sb-manual/doc/profiling.lisp b/contrib/sb-manual/doc/profiling.lisp new file mode 100644 index 000000000..a8acd84f3 --- /dev/null +++ b/contrib/sb-manual/doc/profiling.lisp @@ -0,0 +1,158 @@ +(in-package :sb-manual) + +(defsection @profiling (:title "Profiling") + "SBCL includes both a deterministic profiler, that can collect + statistics on individual functions, and a more \"modern\", + statistical profiler. + + Inlined functions do not appear in the results reported by either." + (@deterministic-profiler section) + (@statistical-profiler section)) + +(defsection @deterministic-profiler (:title "Deterministic Profiler") + "The package `SB-PROFILE` provides a classic, per-function-call + profiler. + + > __Warning__: When profiling code executed by multiple threads in + > parallel, the consing attributed to each function is inaccurate." + (sb-profile:profile macro) + (sb-profile:unprofile macro) + (sb-profile:report function) + (sb-profile:reset function)) + +(defsection @statistical-profiler (:title "Statistical Profiler") + "The `SB-SPROF` module, loadable by + + (require :sb-sprof) + + provides an alternate profiler which works by taking samples of the + program execution at regular intervals, instead of instrumenting + functions as SB-PROFILE:PROFILE does. You might find `SB-SPROF` more + useful than the deterministic profiler when profiling functions in the + `COMMON-LISP` package, SBCL internals, or code where the instrumenting + overhead is excessive. + + Additionally `SB-SPROF` includes a limited deterministic profiler + which can be used for reporting the amounts of calls to some functions + during + + __Example usage:__ + + (in-package :cl-user) + + (require :sb-sprof) + + (declaim (optimize speed)) + + (defun cpu-test-inner (a i) + (logxor a + (* i 5) + (+ a i))) + + (defun cpu-test (n) + (let ((a 0)) + (dotimes (i (expt 2 n) a) + (setf a (cpu-test-inner a i))))) + + ;;;; CPU profiling + + ;;; Take up to 1000 samples of running (CPU-TEST 26), and give a flat + ;;; table report at the end. Profiling will end one the body has been + ;;; evaluated once, whether or not 1000 samples have been taken. + (sb-sprof:with-profiling (:max-samples 1000 + :report :flat + :loop nil) + (cpu-test 26)) + + ;;; Record call counts for functions defined on symbols in the CL-USER + ;;; package. + (sb-sprof:profile-call-counts \"CL-USER\") + + ;;; Take 1000 samples of running (CPU-TEST 24), and give a flat + ;;; table report at the end. The body will be re-evaluated in a loop + ;;; until 1000 samples have been taken. A sample count will be printed + ;;; after each iteration. + (sb-sprof:with-profiling (:max-samples 1000 + :report :flat + :loop t + :show-progress t) + (cpu-test 24)) + + ;;;; Allocation profiling + + (defun foo (&rest args) + (mapcar (lambda (x) (float x 1d0)) args)) + + (defun bar (n) + (declare (fixnum n)) + (apply #'foo (loop repeat n collect n))) + + (sb-sprof:with-profiling (:max-samples 10000 + :mode :alloc + :report :flat) + (bar 1000)) + + __Output:__ + + The flat report format will show a table of all functions that the + profiler encountered on the call stack during sampling, ordered by + the number of samples taken while executing that function. + + Self Total Cumul + Nr Count % Count % Count % Calls Function + ------------------------------------------------------------------------ + 1 69 24.4 97 34.3 69 24.4 67108864 CPU-TEST-INNER + 2 64 22.6 64 22.6 133 47.0 - SB-VM::GENERIC-+ + 3 39 13.8 256 90.5 172 60.8 1 CPU-TEST + 4 31 11.0 31 11.0 203 71.7 - SB-KERNEL:TWO-ARG-XOR + + For each function, the table will show three absolute and relative + sample counts. The `Self` column shows samples taken while directly + executing that function. The `Total` column shows samples taken + while executing that function or functions called from it (sampled + to a platform-specific depth). The `Cumul` column shows the sum of + all `Self` columns up to and including that line in the table. + + Additionally the `Calls` column will record the amount of calls that + were made to the function during the profiling run. This value will + only be reported for functions that have been explicitly marked for + call counting with SB-SPROF:PROFILE-CALL-COUNTS. + + The profiler also hooks into the disassembler such that instructions + which have been sampled are annotated with their relative frequency + of sampling. This information is not stored across different + sampling runs. + + ; 6CF: 702E JO L4 ; 6/242 samples + ; 6D1: D1E3 SHL EBX, 1 + ; 6D3: 702A JO L4 + ; 6D5: L2: F6C303 TEST BL, 3 ; 2/242 samples + ; 6D8: 756D JNE L8 + ; 6DA: 8BC3 MOV EAX, EBX ; 5/242 samples + ; 6DC: L3: 83F900 CMP ECX, 0 ; 4/242 samples + + __Platform support__ + + Allocation profiling is only supported on SBCL builds that use the + generational garbage collector. Tracking of call stacks at a depth + of more than two levels is only supported on x86 and x86-64. + + __Macros__" + (sb-sprof:with-profiling macro) + (sb-sprof:with-sampling macro) + "__Functions__" + (sb-sprof:map-traces function) + (sb-sprof:sample-pc function) + (sb-sprof:report function) + (sb-sprof:reset function) + (sb-sprof:start-profiling function) + (sb-sprof:stop-profiling function) + (sb-sprof:profile-call-counts function) + (sb-sprof:unprofile-call-counts function) + "__Variables__" + (sb-sprof:*max-samples* variable) + (sb-sprof:*sample-interval* variable) + "__Credits__ + + `SB-SPROF` is an SBCL port, with enhancements, of Gerd Moellmann's + statistical profiler for CMUCL.") diff --git a/contrib/sb-manual/doc/sb-aclrepl.lisp b/contrib/sb-manual/doc/sb-aclrepl.lisp new file mode 120000 index 000000000..d9924cd3c --- /dev/null +++ b/contrib/sb-manual/doc/sb-aclrepl.lisp @@ -0,0 +1 @@ +../../sb-aclrepl/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-concurrency.lisp b/contrib/sb-manual/doc/sb-concurrency.lisp new file mode 120000 index 000000000..0782cca0f --- /dev/null +++ b/contrib/sb-manual/doc/sb-concurrency.lisp @@ -0,0 +1 @@ +../../sb-concurrency/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-cover.lisp b/contrib/sb-manual/doc/sb-cover.lisp new file mode 120000 index 000000000..adf551ddd --- /dev/null +++ b/contrib/sb-manual/doc/sb-cover.lisp @@ -0,0 +1 @@ +../../sb-cover/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-grovel.lisp b/contrib/sb-manual/doc/sb-grovel.lisp new file mode 120000 index 000000000..1d037edf8 --- /dev/null +++ b/contrib/sb-manual/doc/sb-grovel.lisp @@ -0,0 +1 @@ +../../sb-grovel/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-introspect.lisp b/contrib/sb-manual/doc/sb-introspect.lisp new file mode 120000 index 000000000..40446900f --- /dev/null +++ b/contrib/sb-manual/doc/sb-introspect.lisp @@ -0,0 +1 @@ +../../sb-introspect/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-md5.lisp b/contrib/sb-manual/doc/sb-md5.lisp new file mode 120000 index 000000000..47fd42245 --- /dev/null +++ b/contrib/sb-manual/doc/sb-md5.lisp @@ -0,0 +1 @@ +../../sb-md5/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-posix.lisp b/contrib/sb-manual/doc/sb-posix.lisp new file mode 120000 index 000000000..61321ebcf --- /dev/null +++ b/contrib/sb-manual/doc/sb-posix.lisp @@ -0,0 +1 @@ +../../sb-posix/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-queue.lisp b/contrib/sb-manual/doc/sb-queue.lisp new file mode 120000 index 000000000..0c0d5860a --- /dev/null +++ b/contrib/sb-manual/doc/sb-queue.lisp @@ -0,0 +1 @@ +../../sb-queue/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-rotate-byte.lisp b/contrib/sb-manual/doc/sb-rotate-byte.lisp new file mode 120000 index 000000000..ae91bd039 --- /dev/null +++ b/contrib/sb-manual/doc/sb-rotate-byte.lisp @@ -0,0 +1 @@ +../../sb-rotate-byte/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-simd.lisp b/contrib/sb-manual/doc/sb-simd.lisp new file mode 120000 index 000000000..4d606ac98 --- /dev/null +++ b/contrib/sb-manual/doc/sb-simd.lisp @@ -0,0 +1 @@ +../../sb-simd/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sb-simple-streams.lisp b/contrib/sb-manual/doc/sb-simple-streams.lisp new file mode 120000 index 000000000..36866e7ba --- /dev/null +++ b/contrib/sb-manual/doc/sb-simple-streams.lisp @@ -0,0 +1 @@ +../../sb-simple-streams/manual.lisp \ No newline at end of file diff --git a/contrib/sb-manual/doc/sbcl.lisp b/contrib/sb-manual/doc/sbcl.lisp new file mode 100644 index 000000000..f2033a3f0 --- /dev/null +++ b/contrib/sb-manual/doc/sbcl.lisp @@ -0,0 +1,21 @@ +(in-package :sb-manual) + +(defsection @sbcl-manual (:title "SBCL Manual") + (@support-and-bugs section) + (@introduction section) + (@starting-and-stopping section) + (@compiler section) + (@debugger section) + (@efficiency section) + (@beyond-the-ansi-standard section) + (@external-formats section) + (@foreign-function-interface section) + (@pathnames section) + (@streams section) + (@package-locks section) + (@threading section) + (@timers section) + (@networking section) + (@profiling section) + (@contributed-modules section) + (@deprecation section)) diff --git a/contrib/sb-manual/doc/start-stop.lisp b/contrib/sb-manual/doc/start-stop.lisp new file mode 100644 index 000000000..aa29264a0 --- /dev/null +++ b/contrib/sb-manual/doc/start-stop.lisp @@ -0,0 +1,339 @@ +(in-package :sb-manual) + +(defsection @starting-and-stopping (:title "Starting and Stopping") + (@starting-sbcl section) + (@stopping-sbcl section) + (@command-line-options section) + (@initialization-files section) + (@initialization-and-exit-hooks section)) + +(defsection @starting-sbcl (:title "Starting SBCL") + (@running-from-shell section) + (@running-from-emacs section) + (@shebang-scripts section)) + +(defsection @running-from-shell (:title "Running from Shell") + "To run SBCL, type `sbcl` at the command line. + + You should end up in the toplevel _REPL_ (read-eval-print loop), + where you can interact with SBCL by typing expressions. + + $ sbcl + This is SBCL 0.8.13.60, an implementation of ANSI Common Lisp. + More information about SBCL is available at <http://www.sbcl.org/>. + + SBCL is free software, provided as is, with absolutely no warranty. + It is mostly in the public domain; some portions are provided under + BSD-style licenses. See the CREDITS and COPYING files in the + distribution for more information. + * (+ 2 2) + 4 + * (exit) + $ + + Also see @COMMAND-LINE-OPTIONS and @STOPPING-SBCL.") + +(defsection @running-from-emacs (:title "Running from Emacs") + "To run SBCL as an `inferior-lisp` from Emacs, in your `.emacs` do + something like: + + ;;; The SBCL binary and command-line arguments + (setq inferior-lisp-program \"/usr/local/bin/sbcl --noinform\") + + For more information on using SBCL with Emacs, see + @EDITOR-INTEGRATION.") + +(defsection @shebang-scripts (:title "Shebang Scripts") + "Standard Unix tools that are interpreters follow a common command line + protocol that is necessary to work with \"shebang scripts\". SBCL + supports this via the `--script` command line option (see + @COMMAND-LINE-OPTIONS). + + Example file (`hello.lisp`): + + #!/usr/local/bin/sbcl --script + (write-line \"Hello, World!\") + + Usage from the command line: + + $ ./hello.lisp + Hello, World! + + Note that SBCL skips the shebang line when it reads the file: + + $ sbcl --script hello.lisp + Hello, World!") + +(defsection @stopping-sbcl (:title "Stopping SBCL") + (@exit section) + (@end-of-file section) + (@saving-a-core-image section) + (@exit-on-errors section)) + +(defsection @exit (:title "Exit") + "SBCL can be stopped at any time by calling SB-EXT:EXIT, + optionally returning a specified numeric value to the calling + process. See @THREADING for information about terminating individual + threads." + (sb-ext:exit function)) + +(defsection @end-of-file (:title "End of File") + "By default SBCL also exits on end of input, caused either by user + pressing `Control-D` on an attached terminal, or end of input when + using SBCL as part of a shell pipeline.") + +(defsection @saving-a-core-image (:title "Saving a Core Image") + "SBCL has the ability to save its state as a file for later + execution. This functionality is important for its bootstrapping + process, and is also provided as an extension to the user." + (sb-ext:save-lisp-and-die function) + ;; When Swank is loaded, it sets this variable. + (sb-ext:*save-hooks* (variable nil)) + "In cases where the standard initialization files have already been loaded + into the saved core, and alternative ones should be used (or none at + all), SBCL allows customizing the initfile pathname computation." + (sb-ext:*sysinit-pathname-function* variable) + (sb-ext:*userinit-pathname-function* variable) + "To facilitate distribution of SBCL applications using external + resources, the filesystem location of the SBCL core file being used + is available from Lisp." + (sb-ext:*core-pathname* (variable "<site-specific>"))) + +(defsection @exit-on-errors (:title "Exit on Errors") + "SBCL can also be configured to exit if an unhandled error occurs, + which is mainly useful for acting as part of a shell pipeline; doing + so under most other circumstances would mean giving up large parts + of the flexibility and robustness of Common Lisp. See + @DEBUGGER-ENTRY and the command line option `--disable-debugger` in + @RUNTIME-OPTIONS.") + +(defsection @command-line-options (:title "Command Line Options") + "Command line options can be considered an advanced topic; for ordinary + interactive use, no command line arguments should be necessary. + + In order to understand the command line argument syntax for SBCL, it + is helpful to understand that the SBCL system is implemented as two + components, a low-level runtime environment written in \\C and a + higher-level system written in Common Lisp itself. Some command line + arguments are processed during the initialization of the low-level + runtime environment, some command line arguments are processed + during the initialization of the Common Lisp system, and any + remaining command line arguments are made available to user code via + SB-EXT:*POSIX-ARGV*. + + The full, unambiguous syntax for invoking SBCL at the command line + is: + + sbcl <runtime-option>* --end-runtime-options \\ + <toplevel-option>* --end-toplevel-options \\ + <user-option>* + + For convenience, `--end-runtime-options` and + `--end-toplevel-options` can be omitted, which can be convenient + when you are running the program interactively, and you can see that + no ambiguities are possible with the option values you are using. + Omitting these elements is probably a bad idea for any batch file + where any of the options are under user control, since it makes it + impossible for SBCL to detect erroneous command line input, so that + erroneous command line arguments will be passed on to the user + program even if they was intended for the runtime system or the Lisp + system." + (@runtime-options section) + (@toplevel-options section)) + +(defsection @runtime-options (:title "Runtime Options") + "- `--core <corefilename>` + + Run the specified Lisp core file instead of the default. Note + that if the Lisp core file is a user-created core file, it may + run a nonstandard toplevel which does not recognize the standard + toplevel options. + + - `--dynamic-space-size <megabytes>` + + Size of the dynamic space reserved on startup in megabytes. + Default value is platform dependent. + + - `--control-stack-size <megabytes>` + + Size of control stack reserved for each thread in megabytes. + Default value is 2. + + - `--tls-limit <positive integer>` + + Maximum number of thread-local symbols in threaded builds. + Default value is 4096. + + - `--noinform` + + Suppress the printing of any banner or other informational + message at startup. This makes it easier to write Lisp programs + which work cleanly in Unix pipelines. See also the `--noprint` + and `--disable-debugger` options. + + - `--disable-ldb` + + Disable the low-level debugger. Only effective if SBCL is + compiled with LDB. + + - `--lose-on-corruption` + + There are some dangerous low-level errors (for instance, control + stack exhausted, memory fault) that (or whose handlers) can + corrupt the image. By default, SBCL prints a warning, then tries + to continue and handle the error in Lisp, but this will not + always work, and SBCL may malfunction or even hang. With this + option, upon encountering such an error, SBCL will exit instead + of invoking LDB (if present and enabled). + + - `--script <filename>` + + As a _runtime_ option, this is equivalent to `--noinform` + `--disable-ldb` `--lose-on-corruption` + `--end-runtime-options` `--script` `<filename>`. See + the description of `--script` as a _toplevel_ option below. + If there are no other command line arguments following + `--script`, the filename argument can be omitted. + + - `--merge-core-pages` + + When platform support is present, provide hints to the operating + system that identical pages may be shared between processes + until they are written to. This can be useful to reduce the + memory usage on systems with multiple SBCL processes started + from similar but differently-named core files, or from + compressed cores. Without platform support, do nothing. By + default only compressed cores trigger hinting. + + - `--no-merge-core-pages` + + Ensures that no sharing hint is provided to the operating + system. + + - `--help` + + Print some basic information about SBCL, then exit. + + - `--version` + + Print SBCL's version information, then exit. + + In the future, runtime options may be added to control behaviour + such as lazy allocation of memory. + + Runtime options, including any `--end-runtime-options` option, are + stripped out of the command line before the Lisp toplevel logic gets + a chance to see it.") + +(defsection @toplevel-options (:title "Toplevel Options") + "The following options are processed and removed by the default + toplevel (see SB-EXT:SAVE-LISP-AND-DIE). + + - `--sysinit <filename>` + + Load `FILENAME` instead of the default system initialization + file (see @INITIALIZATION-FILES). + + - `--no-sysinit` + + Don't load a system-wide initialization file. If this option is + given, the `--sysinit` option is ignored. + + - `--userinit <filename>` + + Load `FILENAME` instead of the default user initialization file + (see @INITIALIZATION-FILES.) + + - `--no-userinit` + + Don't load a user initialization file. If this option is given, + the `--userinit` option is ignored. + + - `--eval <command>` + + After executing any initialization file, but before starting the + read-eval-print loop on standard input, read and evaluate + `COMMAND`. More than one `--eval` option can be used, and all + will be read and executed, in the order they appear on the + command line. + + - `--load <filename>` + + This is equivalent to `--eval '(load \"<filename>\")'`. The + special syntax is intended to reduce quoting headaches when + invoking SBCL from shell scripts. + + - `--noprint` + + When ordinarily the toplevel \"read-eval-print loop\" would be + executed, execute a \"read-eval loop\" instead, i.e. don't print + a prompt and don't echo results. Combined with the `--noinform` + runtime option, this makes it easier to write Lisp \"scripts\" + which work cleanly in Unix pipelines. + + - `--disable-debugger` + + By default when SBCL encounters an error, it enters the builtin + debugger, allowing interactive diagnosis and possible + intercession. This option disables the debugger, causing errors + to print a backtrace and exit with status 1 instead. When given, + this option takes effect before loading of initialization files + or processing `--eval` and `--load` options. See + SB-EXT:DISABLE-DEBUGGER and @DEBUGGER-ENTRY. + + - `--script <filename>` + + Implies `--no-userinit` `--no-sysinit` `--disable-debugger` + `--end-toplevel-options`. + + Causes the system to load the specified file instead of entering + the read-eval-print-loop, and exit afterwards. If the file + begins with a shebang line, it is ignored. + + If there are no other command line arguments following, the + filename can be omitted: this causes the script to be loaded + from standard input instead. Shebang lines in standard input + script are currently _not_ ignored. + + In either case, if there is an unhandled error (e.g. end of + file, or a broken pipe) on either standard input, standard + output, or standard error, the script silently exits with code + 0. This allows e.g. safely piping output from SBCL to `head -n1` + or similar. + + Additionally, the option sets *COMPILE-VERBOSE* and + *LOAD-VERBOSE* to NIL while loading the file to avoid + potentially verbose diagnostic messages printed on the standard + output.") + +(defsection @initialization-files (:title "Initialization Files") + "SBCL processes initialization files with READ and EVAL, + not LOAD; hence initialization files can be used to set startup + *PACKAGE* and *READTABLE*, and for proclaiming a global optimization + policy. + + - __System Initialization File:__ Defaults to `$SBCL_HOME/sbclrc`, + or if that doesn't exist to `/etc/sbclrc`. Can be overridden with + the command line option `--sysinit` or `--no-sysinit` (see + @TOPLEVEL-OPTIONS). + + The system initialization file is intended for system + administrators and software packagers to configure locations of + installed third party modules, etc. + + - __User Initialization File:__ Defaults to `$HOME/.sbclrc`. Can be + overridden with the command line option `--userinit` or + `--no-userinit` (see @TOPLEVEL-OPTIONS). + + The user initialization file is intended for personal + customizations, such as loading certain modules at startup, + defining convenience functions to use in the REPL, handling + automatic recompilation of FASLs (see @FASL-FORMAT), etc. + + Neither initialization file is required.") + +(defsection @initialization-and-exit-hooks + (:title "Initialization and Exit Hooks") + "SBCL provides hooks into the system initialization and exit." + (sb-ext:*init-hooks* variable) + (sb-ext:*exit-hooks* variable)) diff --git a/contrib/sb-manual/doc/streams.lisp b/contrib/sb-manual/doc/streams.lisp new file mode 100644 index 000000000..35d6aa11f --- /dev/null +++ b/contrib/sb-manual/doc/streams.lisp @@ -0,0 +1,322 @@ +(in-package :sb-manual) + +(defsection @streams (:title "Streams") + "Streams which read or write Lisp character data from or to the outside + world -- files, sockets or other external entities -- require the + specification of a conversion between the external, binary data and + the Lisp characters. In ANSI Common Lisp, this is done by specifying + the :EXTERNAL-FORMAT argument when the stream is created. The major + information required is an _encoding_, specified by a keyword naming + that encoding; however, it is also possible to specify refinements + to that encoding as additional options to the external format + designator. + + In addition, SBCL supports various extensions of ANSI Common Lisp + streams: + + - _Bivalent Streams_: A type of stream that can read and write both + CHARACTER and `(UNSIGNED-BYTE 8)` values. + + - _Gray Streams_: User-overloadable CLOS classes whose instances can + be used as Lisp streams (e.g. passed as the first argument to + FORMAT). + + - _Simple Streams_: The bundled contrib module `SB-SIMPLE-STREAMS` + implements a subset of the Franz Allegro simple-streams proposal." + (@stream-external-formats section) + (@bivalent-streams section) + (@gray-streams section) + (@sb-simple-streams section)) + +(defsection @stream-external-formats (:title "Stream External Formats") + "The function STREAM-EXTERNAL-FORMAT returns the canonical name of + the external format (See @EXTERNAL-FORMATS) used by the stream for + character-based input and/or output. + + When constructing file streams, for example using OPEN or + WITH-OPEN-FILE, the external format to use is specified via the + :EXTERNAL-FORMAT argument which accepts an external format + designator (see @EXTERNAL-FORMAT-DESIGNATORS).") + +(defsection @bivalent-streams (:title "Bivalent Streams") + "A _bivalent stream_ can be used to read and write both + CHARACTER and `(UNSIGNED-BYTE 8)` values. A bivalent stream is + created by calling OPEN with the argument :ELEMENT-TYPE + :DEFAULT. On such a stream, both binary and character data can be + read and written with the usual input and output functions. + + Streams are _not_ created bivalent by default for performance + reasons. Bivalent streams are incompatible with `FAST-READ-CHAR`, an + internal optimization in SBCL's stream machinery that bulk-converts + octets to characters and implements a fast path through READ-CHAR.") + +(defsection @gray-streams (:title "Gray Streams") + "The Gray Streams interface is a widely supported extension that + provides for definition of CLOS-extensible stream classes. Gray + stream classes are implemented by adding methods to generic + functions analogous to Common Lisp's standard I/O functions. + Instances of Gray stream classes may be used with any I/O operation + where a non-Gray stream can, provided that all required methods have + been implemented suitably." + (@gray-streams-classes section) + (@methods-common-to-all-streams section) + (@input-stream-methods section) + (@character-input-stream-methods section) + (@output-stream-methods section) + (@character-output-stream-methods section) + (@binary-stream-methods section) + (@gray-streams-examples section)) + +(defsection @gray-streams-classes (:title "Gray Streams classes") + "The defined Gray Stream classes are these:" + (sb-gray:fundamental-stream class) + (sb-gray:fundamental-input-stream class) + "The function INPUT-STREAM-P will return true of any generalized + instance of SB-GRAY:FUNDAMENTAL-INPUT-STREAM." + (sb-gray:fundamental-output-stream class) + "The function OUTPUT-STREAM-P will return true of any generalized + instance of SB-GRAY:FUNDAMENTAL-OUTPUT-STREAM." + (sb-gray:fundamental-binary-stream class) + "Note that instantiable subclasses of SB-GRAY:FUNDAMENTAL-BINARY-STREAM + should provide (or inherit) an applicable method for the generic + function STREAM-ELEMENT-TYPE." + (sb-gray:fundamental-character-stream class) + (sb-gray:fundamental-binary-input-stream class) + (sb-gray:fundamental-binary-output-stream class) + (sb-gray:fundamental-character-input-stream class) + (sb-gray:fundamental-character-output-stream class)) + +(defsection @methods-common-to-all-streams + (:title "Methods common to all streams") + "These generic functions can be specialized on any generalized instance + of fundamental-stream." + (stream-element-type generic-function) + (close generic-function) + (sb-gray:stream-file-position generic-function)) + +(defsection @input-stream-methods (:title "Input stream methods") + "These generic functions may be specialized on any generalized instance + of fundamental-input-stream." + (sb-gray:stream-clear-input generic-function) + (sb-gray:stream-read-sequence generic-function)) + +(defsection @character-input-stream-methods + (:title "Character input stream methods") + "These generic functions are used to implement subclasses of + SB-GRAY:FUNDAMENTAL-INPUT-STREAM:" + (sb-gray:stream-peek-char generic-function) + (sb-gray:stream-read-char-no-hang generic-function) + (sb-gray:stream-read-char generic-function) + (sb-gray:stream-read-line generic-function) + (sb-gray:stream-listen generic-function) + (sb-gray:stream-unread-char generic-function)) + +(defsection @output-stream-methods (:title "Output stream methods") + "These generic functions are used to implement subclasses of + SB-GRAY:FUNDAMENTAL-OUTPUT-STREAM:" + (sb-gray:stream-clear-output generic-function) + (sb-gray:stream-finish-output generic-function) + (sb-gray:stream-force-output generic-function) + (sb-gray:stream-write-sequence generic-function)) + +(defsection @character-output-stream-methods + (:title "Character output stream methods") + "These generic functions are used to implement subclasses of + SB-GRAY:FUNDAMENTAL-CHARACTER-OUTPUT-STREAM:" + (sb-gray:stream-advance-to-column generic-function) + (sb-gray:stream-fresh-line generic-function) + (sb-gray:stream-line-column generic-function) + (sb-gray:stream-line-length generic-function) + (sb-gray:stream-start-line-p generic-function) + (sb-gray:stream-terpri generic-function) + (sb-gray:stream-write-char generic-function) + (sb-gray:stream-write-string generic-function)) + +(defsection @binary-stream-methods (:title "Binary stream methods") + "The following generic functions are available for subclasses of + SB-GRAY:FUNDAMENTAL-BINARY-STREAM:" + (sb-gray:stream-read-byte generic-function) + (sb-gray:stream-write-byte generic-function)) + +(defsection @gray-streams-examples (:title "Gray Streams Examples") + "Below are two classes of stream that can be conveniently defined as + wrappers for Common Lisp streams. These are meant to serve as + examples of minimal implementations of the protocols that must be + followed when defining Gray streams. Realistic uses of the Gray + Streams API would implement the various methods that can do I/O in + batches, such as SB-GRAY:STREAM-READ-LINE, + SB-GRAY:STREAM-WRITE-STRING, SB-GRAY:STREAM-READ-SEQUENCE, and + SB-GRAY:STREAM-WRITE-SEQUENCE." + (@character-counting-input-stream section) + (@output-prefixing-character-stream section)) + +(defsection @character-counting-input-stream + (:title "Character Counting Input Stream") + " It is occasionally handy for programs that process input files to + count the number of characters and lines seen so far, and the number + of characters seen on the current line, so that useful messages may + be reported in case of parsing errors, etc. Here is a character + input stream class that keeps track of these counts. Note that all + character input streams must implement SB-GRAY:STREAM-READ-CHAR and + SB-GRAY:STREAM-UNREAD-CHAR. + + (defclass wrapped-stream (fundamental-stream) + ((stream :initarg :stream :reader stream-of))) + + (defmethod stream-element-type ((stream wrapped-stream)) + (stream-element-type (stream-of stream))) + + (defmethod close ((stream wrapped-stream) &key abort) + (close (stream-of stream) :abort abort)) + + (defclass wrapped-character-input-stream + (wrapped-stream fundamental-character-input-stream) + ()) + + (defmethod stream-read-char ((stream wrapped-character-input-stream)) + (read-char (stream-of stream) nil :eof)) + + (defmethod stream-unread-char ((stream wrapped-character-input-stream) + char) + (unread-char char (stream-of stream))) + + (defclass counting-character-input-stream + (wrapped-character-input-stream) + ((char-count :initform 1 :accessor char-count-of) + (line-count :initform 1 :accessor line-count-of) + (col-count :initform 1 :accessor col-count-of) + (prev-col-count :initform 1 :accessor prev-col-count-of))) + + (defmethod stream-read-char ((stream counting-character-input-stream)) + (with-accessors ((inner-stream stream-of) (chars char-count-of) + (lines line-count-of) (cols col-count-of) + (prev prev-col-count-of)) stream + (let ((char (call-next-method))) + (cond ((eql char :eof) + :eof) + ((char= char #\Newline) + (incf lines) + (incf chars) + (setf prev cols) + (setf cols 1) + char) + (t + (incf chars) + (incf cols) + char))))) + + (defmethod stream-unread-char ((stream counting-character-input-stream) + char) + (with-accessors ((inner-stream stream-of) (chars char-count-of) + (lines line-count-of) (cols col-count-of) + (prev prev-col-count-of)) stream + (cond ((char= char #\Newline) + (decf lines) + (decf chars) + (setf cols prev)) + (t + (decf chars) + (decf cols) + char)) + (call-next-method))) + + The default methods for SB-GRAY:STREAM-READ-CHAR-NO-HANG, + SB-GRAY:STREAM-PEEK-CHAR, SB-GRAY:STREAM-LISTEN, + SB-GRAY:STREAM-CLEAR-INPUT, SB-GRAY:STREAM-READ-LINE, and + SB-GRAY:STREAM-READ-SEQUENCE should be sufficient (though the last + two will probably be slower than methods that forwarded directly). + + Here's a sample use of this class: + + (with-input-from-string (input \"1 2 + 3 :foo \") + (let ((counted-stream (make-instance 'counting-character-input-stream + :stream input))) + (loop for thing = (read counted-stream) while thing + unless (numberp thing) do + (error \"Non-number ~S (line ~D, column ~D)\" thing + (line-count-of counted-stream) + (- (col-count-of counted-stream) + (length (format nil \"~S\" thing)))) + end + do (print thing)))) + + Output: + + 1 + 2 + 3 + Non-number :FOO (line 2, column 5) + [Condition of type SIMPLE-ERROR]") + +(defsection @output-prefixing-character-stream + (:title "Output Prefixing Character Stream") + "One use for a wrapped output stream might be to prefix each line of + text with a timestamp, e.g. for a logging stream. Here's a simple + stream that does this, though without any fancy line-wrapping. Note + that all character output stream classes must implement + SB-GRAY:STREAM-WRITE-CHAR and SB-GRAY:STREAM-LINE-COLUMN. + + (defclass wrapped-stream (fundamental-stream) + ((stream :initarg :stream :reader stream-of))) + + (defmethod stream-element-type ((stream wrapped-stream)) + (stream-element-type (stream-of stream))) + + (defmethod close ((stream wrapped-stream) &key abort) + (close (stream-of stream) :abort abort)) + + (defclass wrapped-character-output-stream + (wrapped-stream fundamental-character-output-stream) + ((col-index :initform 0 :accessor col-index-of))) + + (defmethod stream-line-column ((stream wrapped-character-output-stream)) + (col-index-of stream)) + + (defmethod stream-write-char ((stream wrapped-character-output-stream) + char) + (with-accessors ((inner-stream stream-of) (cols col-index-of)) stream + (write-char char inner-stream) + (if (char= char #\Newline) + (setf cols 0) + (incf cols)))) + + (defclass prefixed-character-output-stream + (wrapped-character-output-stream) + ((prefix :initarg :prefix :reader prefix-of))) + + (defgeneric write-prefix (prefix stream) + (:method ((prefix string) stream) (write-string prefix stream)) + (:method ((prefix function) stream) (funcall prefix stream))) + + (defmethod stream-write-char ((stream prefixed-character-output-stream) + char) + (with-accessors ((inner-stream stream-of) (cols col-index-of) + (prefix prefix-of)) stream + (when (zerop cols) + (write-prefix prefix inner-stream)) + (call-next-method))) + + As with the example input stream, this implements only the minimal + protocol. A production implementation should also provide methods + for at least SB-GRAY:STREAM-WRITE-STRING, + SB-GRAY:STREAM-WRITE-SEQUENCE. + + And here's a sample use of this class: + + (flet ((format-timestamp (stream) + (apply #'format stream \"[~2@*~2,' D:~1@*~2,'0D:~0@*~2,'0D] \" + (multiple-value-list (get-decoded-time))))) + (let ((output (make-instance 'prefixed-character-output-stream + :stream *standard-output* + :prefix #'format-timestamp))) + (loop for string in '(\"abc\" \"def\" \")ghi\") do + (write-line string output) + (sleep 1)))) + + Output: + + [ 0:30:05] abc + [ 0:30:06] def + [ 0:30:07] ghi + NIL") diff --git a/contrib/sb-manual/doc/support-and-bugs.lisp b/contrib/sb-manual/doc/support-and-bugs.lisp new file mode 100644 index 000000000..1e33365d5 --- /dev/null +++ b/contrib/sb-manual/doc/support-and-bugs.lisp @@ -0,0 +1,123 @@ +(in-package :sb-manual) + +(defsection @support-and-bugs (:title "Getting Support and Reporting Bugs") + (@volunteer-support section) + (@commercial-support section) + (@reporting-bugs section)) + +(defsection @volunteer-support (:title "Volunteer Support") + "Your primary source of SBCL support should probably be the mailing + list `sbcl-help`: in addition to other users SBCL developers monitor + this list and are available for advice. As an anti-spam measure + subscription is required for posting: + + <https://lists.sourceforge.net/lists/listinfo/sbcl-help> + + Remember that the people answering your question are volunteers, so + you stand a much better chance of getting a good answer if you ask a + good question. + + Before sending mail, check the list archives at either + + <http://sourceforge.net/mailarchive/forum.php?forum_name=sbcl-help> + + or + + <http://news.gmane.org/gmane.lisp.steel-bank.general> + + to see if your question has been answered already. Checking the bug + database is also worth it (see @REPORTING-BUGS), to see if the issue + is already known. + + For general advice on asking good questions, see + + <http://www.catb.org/~esr/faqs/smart-questions.html>.") + +(defsection @commercial-support (:title "Commercial Support") + "There is no formal organization developing SBCL, but if you need a + paid support arrangement or custom SBCL development, we maintain the + list of companies and consultants below. Use it to identify service + providers with appropriate skills and interests, and contact them + directly. + + The SBCL project cannot verify the accuracy of the information or + the competence of the people listed, and they have provided their + own blurbs below: you must make your own judgement of suitability + from the available information - refer to the links they provide, + the CREDITS file, mailing list archives, CVS commit messages, and so + on. Please feel free to ask for advice on the sbcl-help list. + + (At present, no companies or consultants wish to advertise paid + support or custom SBCL development in this manual).") + +(defsection @reporting-bugs (:title "Reporting Bugs") + "SBCL uses Launchpad to track bugs. The bug database is available at + + <https://bugs.launchpad.net/sbcl> + + Reporting bugs there requires registering at Launchpad. However, + bugs can also be reported on the mailing list `sbcl-bugs`, + which is moderated but does _not_ require subscribing. + + Simply send email to `sbcl-bugs@lists.sourceforge.net` and the bug + will be checked and added to Launchpad by SBCL maintainers." + (@how-to-report-bugs-effectively section) + (@how-to-report-signal-related-bugs section)) + +(defsection @how-to-report-bugs-effectively + (:title "How to Report Bugs Effectively") + "Please include enough information in a bug report that someone reading + it can reproduce the problem, i.e. don't write + + Subject: apparent bug in PRINT-OBJECT (or *PRINT-LENGTH*?) + PRINT-OBJECT doesn't seem to work with *PRINT-LENGTH*. Is this a bug? + + but instead + + Subject: apparent bug in PRINT-OBJECT (or *PRINT-LENGTH*?) + In sbcl-1.2.3 running under OpenBSD 4.5 on my Alpha box, when + I compile and load the file + (DEFSTRUCT (FOO (:PRINT-OBJECT (LAMBDA (X Y) + (LET ((*PRINT-LENGTH* 4)) + (PRINT X Y))))) + X Y) + then at the command line type + (MAKE-FOO) + the program loops endlessly instead of printing the object. + + A more in-depth discussion on reporting bugs effectively can be + found at + + <http://www.chiark.greenend.org.uk/~sgtatham/bugs.html>.") + +(defsection @how-to-report-signal-related-bugs + (:title "How to Report Signal-related Bugs") + "If you run into a signal related bug, you are getting fatal errors + such as `signal N is [un]blocked` or just hangs, and you want to + send a useful bug report then: + + - Compile SBCL with ldb enabled (feature `:sb-ldb`, see + `base-target-features.lisp-expr`). + + - Isolate a smallish test case, run it. + + - If it just hangs kill it with `SIGABRT`: `kill -ABRT <pidof sbcl>`. + + - Print the backtrace from ldb by typing `ba`. + + - Attach gdb: `gdb -p <pidof sbcl>` and get backtraces for all + threads: `thread apply all ba`. + + - If multiple threads are in play then still in gdb, try to get Lisp + backtrace for all threads: `thread apply all call + backtrace_from_fp($ebp, 100, 0)`. Substitute `$ebp` with `$rbp` on + x86-64. The backtraces will appear in the stdout of the SBCL + process. + + - Send a report with the backtraces and the output (both stdout and + stderr) produced by SBCL. + + - Don't forget to include OS and SBCL version. + + - If available, include information on outcome of the same test with + other versions of SBCL, OS, ...") diff --git a/contrib/sb-manual/doc/threading.lisp b/contrib/sb-manual/doc/threading.lisp new file mode 100644 index 000000000..b7d2569c5 --- /dev/null +++ b/contrib/sb-manual/doc/threading.lisp @@ -0,0 +1,333 @@ +(in-package :sb-manual) + +(defsection @threading (:title "Threading") + "SBCL supports a fairly low-level threading interface that maps onto + the host operating system's concept of threads or lightweight + processes. This means that threads may take advantage of hardware + multiprocessing on machines that have more than one CPU, but it does + not allow Lisp control of the scheduler. This is found in the + `SB-THREAD` package. + + Threads are part of the default build on x86[-64]/ARM64 Linux and + Windows. + + They are also supported on: x86[-64] Darwin (Mac OS X), x86[-64] + FreeBSD, x86 SunOS (Solaris), PPC Linux, ARM64 Linux, RISC-V Linux. + On these platforms threads must be explicitly enabled at build-time, + see `INSTALL` for directions." + (@threading-basics section) + (@special-variables section) + (@atomic-operations section) + (@mutex-support section) + (@semaphores section) + (@waitqueue/condition-variables section) + (@barriers section) + (@sessions/debugging section) + (@foreign-threads section) + (@implementation-on-linux-x86oids section)) + +(defsection @threading-basics (:title "Threading Basics") + "``` + (make-thread (lambda () (write-line \"Hello, world\"))) + ```" + (@thread-objects section) + (@running-threads section) + (@asynchronous-operations section) + (@miscellaneous-operations section) + (@error-conditions section)) + +(defsection @thread-objects (:title "Thread Objects") + (sb-thread:thread structure) + (sb-thread:*current-thread* variable) + (sb-thread:list-all-threads function) + (sb-thread:thread-alive-p function) + (sb-thread:thread-name function) + (sb-thread:main-thread-p function) + (sb-thread:main-thread function)) + +(defsection @running-threads (:title "Running Threads") + (sb-thread:make-thread function) + (sb-thread:return-from-thread macro) + (sb-thread:abort-thread function) + (sb-thread:join-thread function) + (sb-thread:thread-yield function)) + +(defsection @asynchronous-operations (:title "Asynchronous Operations") + (sb-thread:interrupt-thread function) + (sb-thread:terminate-thread function)) + +(defsection @miscellaneous-operations (:title "Miscellaneous Operations") + (sb-thread:symbol-value-in-thread function)) + +(defsection @error-conditions (:title "Error Conditions") + (sb-thread:thread-error condition) + (sb-thread:thread-error-thread function) + (sb-thread:symbol-value-in-thread-error condition) + (sb-thread:interrupt-thread-error condition) + (sb-thread:join-thread-error condition)) + +(defsection @special-variables (:title "Special Variables") + "The interaction of special variables with multiple threads is mostly + as one would expect, with behaviour very similar to other + implementations. + + - Global special values are visible across all threads. + + - Bindings (e.g. using LET) are local to the thread. + + - Threads do not inherit dynamic bindings from the parent thread. + + The last point means that + + (defparameter *x* 0) + (let ((*x* 1)) + (sb-thread:make-thread (lambda () (print *x*)))) + + prints `0` and not `1`. + + Note, however, that there is a hard limit on the number of distinct + symbols that can be bound dynamically in threaded builds (see + `--tls-limit` in @RUNTIME-OPTIONS). Exceeding this limit triggers + the low-level error `Thread local storage exhausted.`") + +(defsection @atomic-operations (:title "Atomic Operations") + "Following atomic operations are particularly useful for implementing + lockless algorithms." + (sb-ext:atomic-decf macro) + (sb-ext:atomic-incf macro) + (sb-ext:atomic-pop macro) + (sb-ext:atomic-push macro) + (sb-ext:atomic-update macro) + (sb-ext:compare-and-swap macro) + "Our SB-EXT:COMPARE-AND-SWAP is user-extensible by defining functions + named `(CAS <PLACE>)`, allowing users to add CAS support to new + places." + (sb-ext:cas macro) + (sb-ext:get-cas-expansion function)) + +(defsection @mutex-support (:title "Mutex Support") + "Mutexes are used for controlling access to a shared resource. One + thread is allowed to hold the mutex, others which attempt to take it + will be made to wait until it's free. Threads are woken in the order + that they go to sleep. + + (defpackage :demo (:use \"CL\" \"SB-THREAD\" \"SB-EXT\")) + + (in-package :demo) + + (defvar *a-mutex* (make-mutex :name \"my lock\")) + + (defun thread-fn () + (format t \"Thread ~A running ~%\" *current-thread*) + (with-mutex (*a-mutex*) + (format t \"Thread ~A got the lock~%\" *current-thread*) + (sleep (random 5))) + (format t \"Thread ~A dropped lock, dying now~%\" *current-thread*)) + + (make-thread #'thread-fn) + (make-thread #'thread-fn)" + (sb-thread:mutex structure) + (sb-thread:with-mutex macro) + (sb-thread:with-recursive-lock macro) + (sb-thread:make-mutex function) + (sb-thread:mutex-name function) + (sb-thread:mutex-owner function) + (sb-thread:mutex-value function) + (sb-thread:grab-mutex function) + (sb-thread:release-mutex function)) + +(defsection @semaphores (:title "Semaphores") + "Semaphores are among other things useful for keeping track of a + countable resource, e.g. messages in a queue, and sleep when the + resource is exhausted." + (sb-thread:semaphore structure) + (sb-thread:make-semaphore function) + (sb-thread:signal-semaphore function) + (sb-thread:wait-on-semaphore function) + (sb-thread:try-semaphore function) + (sb-thread:semaphore-count function) + (sb-thread:semaphore-name function) + (sb-thread:semaphore-notification structure) + (sb-thread:make-semaphore-notification function) + (sb-thread:semaphore-notification-status function) + (sb-thread:clear-semaphore-notification function)) + +(defsection @waitqueue/condition-variables + (:title "Waitqueue/condition variables") + "These are based on the POSIX condition variable design, hence the + annoyingly CL-conflicting name. For use when you want to check a + condition and sleep until it's true. For example: you have a shared + queue, a writer process checking _queue is empty_ and one or more + readers that need to know when _queue is not empty_. It sounds + simple but is astonishingly easy to deadlock if another process runs + when you weren't expecting it to. + + There are three components: + + - the condition itself (not represented in code) + + - the condition variable (a.k.a. waitqueue) which proxies for it + + - a lock to hold while testing the condition + + Important stuff to be aware of: + + - when calling condition-wait, you must hold the mutex. + condition-wait will drop the mutex while it waits, and obtain it + again before returning for whatever reason; + + - likewise, you must be holding the mutex around calls to + SB-THREAD:CONDITION-NOTIFY; + + - a process may return from SB-THREAD:CONDITION-WAIT in several + circumstances: it is not guaranteed that the underlying condition + has become true. You must check that the resource is ready for + whatever you want to do to it. + + (defvar *buffer-queue* (make-waitqueue)) + (defvar *buffer-lock* (make-mutex :name \"buffer lock\")) + + (defvar *buffer* (list nil)) + + (defun reader () + (with-mutex (*buffer-lock*) + (loop + (condition-wait *buffer-queue* *buffer-lock*) + (loop + (unless *buffer* (return)) + (let ((head (car *buffer*))) + (setf *buffer* (cdr *buffer*)) + (format t \"reader ~A woke, read ~A~%\" + *current-thread* head)))))) + + (defun writer () + (loop + (sleep (random 5)) + (with-mutex (*buffer-lock*) + (let ((el (intern + (string (code-char + (+ (char-code #\A) (random 26))))))) + (setf *buffer* (cons el *buffer*))) + (condition-notify *buffer-queue*)))) + + (make-thread #'writer) + (make-thread #'reader) + (make-thread #'reader)" + (sb-thread:waitqueue structure) + (sb-thread:make-waitqueue function) + (sb-thread:waitqueue-name function) + (sb-thread:condition-wait function) + (sb-thread:condition-notify function) + (sb-thread:condition-broadcast function)) + +(defsection @barriers (:title "Barriers") + "These are based on the Linux kernel barrier design, which is in turn + based on the Alpha CPU memory model. They are presently implemented for + x86, x86-64, PPC, ARM64, and RISC-V systems, and behave as compiler + barriers on all other CPUs. + + In addition to explicit use of the SB-THREAD:BARRIER macro, the + following functions and macros also serve as :MEMORY barriers: + + - SB-EXT:ATOMIC-DECF, SB-EXT:ATOMIC-INCF, SB-EXT:ATOMIC-PUSH, + and SB-EXT:ATOMIC-POP + + - SB-EXT:COMPARE-AND-SWAP + + - SB-THREAD:GRAB-MUTEX, SB-THREAD:RELEASE-MUTEX, + SB-THREAD:WITH-MUTEX and SB-THREAD:WITH-RECURSIVE-LOCK + + - SB-THREAD:SIGNAL-SEMAPHORE, SB-THREAD:TRY-SEMAPHORE and + SB-THREAD:WAIT-ON-SEMAPHORE + + - SB-THREAD:CONDITION-WAIT, SB-THREAD:CONDITION-NOTIFY and + SB-THREAD:CONDITION-BROADCAST." + (sb-thread:barrier macro)) + +(defsection @sessions/debugging (:title "Sessions/Debugging") + "If the user has multiple views onto the same Lisp image (for example, + using multiple terminals, or a windowing system, or network access) + they are typically set up as multiple _sessions_ such that each view + has its own collection of foreground, background, and stopped + threads. A thread which wishes to create a new session can use + SB-THREAD:WITH-NEW-SESSION to remove itself from the current + session (which it shares with its parent and siblings) and create a + fresh one." + (sb-thread:with-new-session macro) + (sb-thread:make-listener-thread function) + "Within a single session, threads arbitrate between themselves for + the user's attention. A thread may be in one of three notional + states: foreground, background, or stopped. When a background + process attempts to print a repl prompt or to enter the debugger, it + will stop and print a message saying that it has stopped. The user + at his leisure may switch to that thread to find out what it needs. + If a background thread enters the debugger, selecting any restart + will put it back into the background before it resumes. Arbitration + for the input stream is managed by calls to + SB-THREAD:GET-FOREGROUND (which may block) and + SB-THREAD:RELEASE-FOREGROUND." + (sb-thread:get-foreground function) + (sb-thread:release-foreground function)) + +(defsection @foreign-threads (:title "Foreign threads") + "Direct calls to `pthread_create(3)` (instead of SB-THREAD:MAKE-THREAD) + create threads that SBCL is not aware of, these are called foreign + threads. Currently, it is not possible to run Lisp code in such + threads. This means that the Lisp side signal handlers cannot work. + The best solution is to start foreign threads with signals blocked, + but since third party libraries may create threads, it is not always + feasible to do so. As a workaround, upon receiving a signal in a + foreign thread, SBCL changes the thread's sigmask to block all + signals that it wants to handle and resends the signal to the + current process which should land in a thread that does not block + it, that is, a Lisp thread. + + The resignalling trick cannot work for synchronously triggered signals + (`SIGSEGV` and co), take care not to trigger any. Resignalling for + synchronously triggered signals in foreign threads is subject to + `--lose-on-corruption`, see @RUNTIME-OPTIONS.") + +(defsection @implementation-on-linux-x86oids + (:title "Implementation on Linux x86oids") + "Threading is implemented using pthreads and some Linux specific bits + like futexes. + + On x86, the per-thread local bindings for special variables is + achieved using the `%fs` segment register to point to a per-thread + storage area. This may cause interesting results if you link to + foreign code that expects threading or creates new threads, and the + thread library in question uses %fs in an incompatible way. On + x86-64 the r12 register has a similar role. + + Queues require the `futex(2)` system call to be available: this is + the reason for the NPTL requirement. We test at runtime that this + system call exists. + + Garbage collection is done with the existing Conservative + Generational GC. Allocation is done in small (typically 8k) regions: + each thread has its own region so this involves no stopping. + However, when a region fills, a lock must be obtained while another + is allocated, and when a collection is required, all processes are + stopped. This is achieved by sending them signals, which may make + for interesting behaviour if they are interrupted in system calls. + The streams interface is believed to handle the required system call + restarting correctly, but this may be a consideration when making + other blocking calls e.g. from foreign library code. + + Large amounts of the SBCL library have not been inspected for + thread-safety. Some of the obviously unsafe areas have large locks + around them, so compilation and fasl loading, for example, cannot be + parallelized. Work is ongoing in this area. + + A new thread by default is created in the same POSIX process group and + session as the thread it was created by. This has an impact on + keyboard interrupt handling: pressing your terminal's intr key + (typically `Control-C`) will interrupt all processes in the + foreground process group, including Lisp threads that SBCL considers + to be notionally _background_. This is undesirable, so background + threads are set to ignore the `SIGINT` signal. + + `SB-THREAD:MAKE-LISTENER-THREAD` in addition to creating a new Lisp + session makes a new POSIX session, so that pressing `Control-C` in + one window will not interrupt another listener - this has been found + to be embarrassing.") diff --git a/contrib/sb-manual/doc/timers.lisp b/contrib/sb-manual/doc/timers.lisp new file mode 100644 index 000000000..1a28cda70 --- /dev/null +++ b/contrib/sb-manual/doc/timers.lisp @@ -0,0 +1,41 @@ +(in-package :sb-manual) + +(defsection @timers (:title "Timers") + "SBCL supports a system-wide event scheduler implemented on top of + `setitimer(2)` that also works with threads but does not require a + separate scheduler thread. + + The following example schedules a timer that writes `Hello, world` + after two seconds. + + (schedule-timer (make-timer (lambda () + (write-line \"Hello, world\") + (force-output))) + 2) + + It should be noted that writing timer functions requires special + care, as the dynamic environment in which they run is unpredictable: + dynamic variable bindings, locks held, etc, all depend on whatever + code was running when the timer fired. The following example should + serve as a cautionary tale: + + (defvar *foo* nil) + + (defun show-foo () + (format t \"~&foo=~S~%\" *foo*) + (force-output t)) + + (defun demo () + (schedule-timer (make-timer #'show-foo) 0.5) + (schedule-timer (make-timer #'show-foo) 1.5) + (let ((*foo* t)) + (sleep 1.0)) + (let ((*foo* :surprise!)) + (sleep 2.0)))" + (sb-ext:timer structure) + (sb-ext:make-timer function) + (sb-ext:timer-name function) + (sb-ext:timer-scheduled-p function) + (sb-ext:schedule-timer function) + (sb-ext:unschedule-timer function) + (sb-ext:list-all-timers function)) diff --git a/contrib/sb-md5/manual.lisp b/contrib/sb-md5/manual.lisp new file mode 100644 index 000000000..58302279d --- /dev/null +++ b/contrib/sb-md5/manual.lisp @@ -0,0 +1,18 @@ +(in-package :sb-manual) + +(defsection @sb-md5 (:title "sb-md5") + ;; FIXME: cite + "The `SB-MD5` module implements the RFC1321 MD5 Message Digest + Algorithm." + (sb-md5:md5sum-file function) + (sb-md5:md5sum-sequence function) + (sb-md5:md5sum-stream function) + (sb-md5:md5sum-string function) + "The implementation for CMUCL was largely done by Pierre Mai, with help + from members of the `cmucl-help` mailing list. Since CMUCL and SBCL + are similar in many respects, it was not too difficult to extend the + low-level implementation optimizations for CMUCL to SBCL. Following + this, SBCL's compiler was extended to implement efficient + compilation of modular arithmetic (@MODULAR-ARITHMETIC), which + enabled the implementation to be expressed in portable arithmetical + terms, apart from the use of @SB-ROTATE-BYTE for bitwise rotation.") diff --git a/contrib/sb-posix/manual.lisp b/contrib/sb-posix/manual.lisp new file mode 100644 index 000000000..9f4a70421 --- /dev/null +++ b/contrib/sb-posix/manual.lisp @@ -0,0 +1,174 @@ +(in-package :sb-manual) + +(defsection @sb-posix (:title "sb-posix") + "Sb-posix is the supported interface for calling out to the operating + system. + + > _Note_: The functionality contained in the package `SB-UNIX` is + > for SBCL internal use only; its contents are likely to change from + > version to version. + + The scope of this interface is \"operating system calls on a typical + Unixlike platform\". This is section 2 of the Unix manual, plus + section 3 calls that are (a) typically found in libc, but (b) not + part of the C standard. For example, we intend to provide support + for `opendir(3)` and `readdir(3)` but not for `printf(3)`. That + said, if your favourite system call is not included yet, you are + encouraged to submit a patch to the SBCL mailing list. + + Some facilities are omitted where they offer absolutely no + additional use over some portable function, or would be actively + dangerous to the consistency of Lisp. Not all functions are + available on all platforms. + + Sb-posix functions do not implicitly take measures to provide + thread-safety or reentrancy beyond whatever the underlying C library + does, except in cases where doing so is necessary to maintain the + consistency of the Lisp image. For example, the bindings to the user + and group database accessing functions are neither thread-safe nor + reentrant unless the underlying libc happens to make them so (but + see @SB-POSIX-EXTENSIONS-TO-POSIX)." + (@sb-posix-lisp-names section) + (@sb-posix-types section) + (@sb-posix-function-parameters section) + (@sb-posix-function-return-values section) + (@sb-posix-lisp-objects-and-c-structures section) + (@sb-posix-idiosyncracies section) + (@sb-posix-extensions-to-posix section)) + +(defsection @sb-posix-lisp-names (:title "Lisp names for C names") + "All symbols are in the `SB-POSIX` package. This package contains a + Lisp function for each supported Unix system call or function, a + variable or constant for each supported Unix constant, an object + type for each supported Unix structure type, and a slot name for + each supported Unix structure member. A symbol name is derived from + the C binding's name, by (a) uppercasing, then (b) removing leading + underscores (`#\\_`) then replacing remaining underscore characters + with the hyphen (`#\\-`). The requirement to uppercase is so that in + a standard upcasing reader the user may write `sb-posix:creat` + instead of `sb-posix:|creat|` as would otherise be required. + + No other changes to \"Lispify\" symbol names are made, so + `creat` becomes `\\\\CREAT`, not `\\\\CREATE`. + + The user is encouraged not to `(USE-PACKAGE :SB-POSIX)` but instead + to use the `SB-POSIX:` prefix on all references, as some of the + symbols symbols contained in the `SB-POSIX` package have the same + name as CL symbols (e.g. OPEN, CLOSE, SIGNAL). Also, see + @PACKAGE-LOCAL-NICKNAMES.") + +(defsection @sb-posix-types (:title "Types") + "Generally, marshalling between Lisp and C data types is done using + SBCL's FFI. See @FOREIGN-FUNCTION-INTERFACE. + + Some functions accept objects such as filenames or file descriptors. + In the C binding to POSIX, these are represented as strings and + small integers respectively. For the Lisp programmer's convenience + we introduce designators such that CL pathnames or open streams can + be passed to these functions. For example, SB-POSIX:RENAME accepts + both pathnames and strings as its arguments." + (@sb-posix-file-descriptors section) + (@sb-posix-filenames section)) + +(defsection @sb-posix-file-descriptors (:title "File-descriptors") + (sb-posix:file-descriptor type) + (sb-posix:file-descriptor-designator type) + (sb-posix:file-descriptor function)) + +(defsection @sb-posix-filenames (:title "Filenames") + (sb-posix:filename type) + (sb-posix:filename-designator type) + (sb-posix:filename function)) + +(defsection @sb-posix-function-parameters (:title "Function Parameters") + "The calling convention is modelled after that of CMUCL's `UNIX` + package: in particular, it's like the C interface except that: + + - Length arguments are omitted or optional where the sensible value + is obvious. For example, `\\read` would be defined this way: + + (read fd buffer &optional (length (length buffer))) => bytes-read + + - Where C simulates \"out\" parameters using pointers (for instance, + in `pipe(2)` or `socketpair(2)`), these may be optional or omitted + in the Lisp interface: if not provided, appropriate objects will + be allocated and returned (using multiple return values if + necessary). + + - Some functions accept objects such as filenames or file + descriptors. Wherever these are specified as such in the C + bindings, the Lisp interface accepts designators for them as + specified in the @SB-POSIX-TYPES section above. + + - A few functions have been included in sb-posix that do not + correspond exactly with their C counterparts. These are described + in @SB-POSIX-IDIOSYNCRACIES.") + +(defsection @sb-posix-function-return-values (:title "Function Return Values") + "The return value is usually the same as for the C binding, except in + error cases: where the C function is defined as returning some + sentinel value and setting `errno` on error, we instead signal an + error of type SB-POSIX:SYSCALL-ERROR. The actual error + value (`errno`) is stored in this condition and can be accessed with + SB-POSIX:SYSCALL-ERRNO. + + We do not automatically translate the returned value into lispy + objects -- for example, SB-POSIX:OPEN returns a small integer, not a + stream. Exception: boolean-returning functions (or, more commonly, + macros) do not return a C integer but instead a Lisp boolean.") + +(defsection @sb-posix-lisp-objects-and-c-structures + (:title "Lisp Objects and C structures") + "Sb-posix provides various Lisp object types to stand in for C + structures in the POSIX library. Lisp bindings to C functions that + accept, manipulate, or return C structures accept, manipulate, or + return instances of these Lisp types instead of instances of alien + types. + + The names of the Lisp types are chosen according to the general + rules described above. For example Lisp objects of type + SB-POSIX:STAT stand in for C structures of type `struct stat`. + + Accessors are provided for each standard field in the structure. + These are named `<STRUCTURE-NAME>-<FIELD-NAME>` where the two + components are chosen according to the general name conversion + rules, with the exception that in cases where all fields in a given + structure have a common prefix, that prefix is omitted. For example, + `stat.st_dev` in C becomes `\\STAT-DEV` in Lisp. + + Because sb-posix might not support all semi-standard or + implementation-dependent members of all structure types on your + system (patches welcome), here is an enumeration of all supported + Lisp objects corresponding to supported POSIX structures, and the + supported slots for those structures." + (sb-posix:flock class) + (sb-posix:passwd class) + (sb-posix:group class) + (sb-posix:stat class) + (sb-posix:termios class) + (sb-posix:timeval class)) + +(defsection @sb-posix-idiosyncracies + (:title "Functions with Idiosyncratic Bindings") + "A few functions in sb-posix don't correspond directly to their C + counterparts." + (sb-posix:getcwd function) + (sb-posix:readlink function) + (sb-posix:syslog function)) + +(defsection @sb-posix-extensions-to-posix (:title "Extensions to POSIX") + "Some of POSIX's standardized operators are not safe to use on their + own, so `SB-POSIX` exports a few helpers that do not correspond + exactly to functionality present in the POSIX standard. + + The user and group database accessing routines are not required to + be thread-safe or reentrant and so can only be used safely if all + clients coordinate around their use. Since it would be logically + impossible for independently developed programs to coordinate, + `SB-POSIX` exports two iteration macros, SB-POSIX:DO-PASSWDS and + SB-POSIX:DO-GROUPS, each of which iterates over the respective + database while preventing the keyed accesses (SB-POSIX:GETPWNAM, + SB-POSIX:GETPWUID, SB-POSIX:GETGRNAM, SB-POSIX:GETGRGID) from + running until iteration completes." + (sb-posix:do-passwds macro) + (sb-posix:do-groups macro)) diff --git a/contrib/sb-queue/manual.lisp b/contrib/sb-queue/manual.lisp new file mode 100644 index 000000000..079e0d508 --- /dev/null +++ b/contrib/sb-queue/manual.lisp @@ -0,0 +1,5 @@ +(in-package :sb-manual) + +(defsection @sb-queue (:title "sb-queue") + "Since SBCL 1.0.38, the `SB-QUEUE` module has been merged into the + `SB-CONCURRENCY` module. See @SB-CONCURRENCY.") diff --git a/contrib/sb-rotate-byte/manual.lisp b/contrib/sb-rotate-byte/manual.lisp new file mode 100644 index 000000000..0700ecd88 --- /dev/null +++ b/contrib/sb-rotate-byte/manual.lisp @@ -0,0 +1,13 @@ +(in-package :sb-manual) + +(defsection @sb-rotate-byte (:title "sb-rotate-byte") + ;; FIXME: Copy the spec to the manual here. + "The `SB-ROTATE-BYTE` module offers an interface to bitwise + rotation, with an efficient implementation for operations which can + be performed directly using the platform's arithmetic routines. It + implements the specification at <http://www.cliki.net/ROTATE-BYTE>." + ;; FIXME: cite + "Bitwise rotation is a component of various cryptographic or hashing + algorithms: MD5, SHA-1, etc.; often these algorithms are specified + on 32-bit rings." + (sb-rotate-byte:rotate-byte function)) diff --git a/contrib/sb-simd/manual.lisp b/contrib/sb-simd/manual.lisp new file mode 100644 index 000000000..82d7de487 --- /dev/null +++ b/contrib/sb-simd/manual.lisp @@ -0,0 +1,233 @@ +(in-package :sb-manual) + +(defsection @sb-simd (:title "sb-simd") + "The `SB-SIMD` module provides a convenient interface for SIMD + programming in SBCL. It provides one package per SIMD instruction + set, plus functions and macros for querying whether an instruction + set is available and what functions and data types it exports." + (@data-types section) + (@casts section) + (@constructors section) + (@unpackers section) + (@reinterpret-casts section) + (@associatives section) + (@reducers section) + (@rounding section) + (@comparisons section) + (@conditionals section) + (@loads-and-stores section) + (@specialized-scalar-operations section) + (@instruction-set-dispatch section)) + +(defsection @data-types (:title "Data Types") + "The central data type in sb-simd is the SIMD pack. A SIMD pack + is very similar to a specialized vector, except that its length must + be a particular power of two that depends on its element type and + the underlying hardware. The set of element types that are supported + for SIMD packs is similar to that of SBCL's specialized array + element types, except that there is currently no support for SIMD + packs of complex numbers or characters. + + The supported scalar types are `F32`, `F64`, `S<N>`, and `U<N>`, + where `<N>` is either 8, 16, 32, or 64. These scalar types are + abbreviations for the Common Lisp types SINGLE-FLOAT, DOUBLE-FLOAT, + SIGNED-BYTE, and UNSIGNED-BYTE, respectively. For each scalar data + type `X`, there exists one or more SIMD data type `X.Y` with `Y` + elements. For example, in AVX there are two supported SIMD data + types with element type `F64`, namely `F64.2` (128 bit) and + `F64.4` (256 bit). + + SIMD packs are regular Common Lisp objects that have a type, a + class, and can be passed as function arguments. The price for this + is that SIMD packs have both a boxed and an unboxed representation. + The unboxed representation of a SIMD pack has zero overhead and fits + into a CPU register but can only be used within a function and when + the compiler can statically determine the SIMD pack's type. + Otherwise, the SIMD pack is boxed, i.e. spilled to the heap together + with its type information. In practice, boxing of SIMD packs can + usually be avoided via inlining, or by loading and storing them to + specialized arrays instead of passing them around as function + arguments.") + +(defsection @casts (:title "Casts") + "For each scalar data type `X`, there is a function named `X` + that is equivalent to `(LAMBDA (V) (COERCE V 'X))`. For each SIMD + data type `X.Y`, there is a function named `X.Y` that ensures that + its argument is of type `X.Y`, or, if the argument is a number, + calls the cast function of `X` and broadcasts the result. + + All functions provided by sb-simd (apart from the casts themselves) + implicitly cast each argument to its expected type. So, to add the + number five to each single float in a SIMD pack `X` of type `F32.8`, + it is sufficient to write `(F32.8+ X 5)`. We don't mention this + implicit conversion explicitly in the following sections, so if any + function description states that an argument must be of type `X.Y`, + the argument can actually be of any type that is a suitable argument + of the cast function named `X.Y`.") + +(defsection @constructors (:title "Constructors") + "For each SIMD data type `X.Y`, there is a constructor named + `MAKE-X.Y` that takes `Y` arguments of type `X` and returns a SIMD + pack whose elements are the supplied values.") + +(defsection @unpackers (:title "Unpackers") + "For each SIMD data type `X.Y`, there is a function named + `X.Y-VALUES` that returns, as `Y` multiple values, the elements of + the supplied SIMD pack of type `X.Y`.") + +(defsection @reinterpret-casts (:title "Reinterpret Casts") + "For each SIMD data type `X.Y`, there is a function named + `X.Y!` that takes any SIMD pack or scalar datum and interprets its + bits as a SIMD pack of type `X.Y`. If the supplied datum has more + bits than the resulting value, the excess bits are discarded. If the + supplied datum has less bits than the resulting value, the missing + bits are assumed to be zero.") + +(defsection @associatives (:title "Associatives") + "For each associative binary function, e.g. `TWO-ARG-X.Y-OP`, there + is a function `X.Y-OP` that takes any number of arguments and + combines them with this binary function in a tree-like fashion. If + the binary function has an identity element, it is possible to call + the function with zero arguments, in which case the identity element + is returned. If there is no identity element, the function must + receive at least one argument. + + Examples of associative functions are `SB-SIMD-AVX:F32.8+`, for + summing any number of 256 bit packs of single floats, and + `SB-SIMD-FMA:U8.32-MAX`, for computing the element-wise maximum of + one or more 256 bit packs of 8 bit integers.") + +(defsection @reducers (:title "Reducers") + "For binary functions `TWO-ARG-X.Y-OP` that are not associative but + have a neutral element, there are functions `X.Y-OP` that take any + positive number of arguments and return the reduction of all + arguments with the binary function. In the special case of a single + supplied argument, the binary function is invoked on the neutral + element and that argument. Reducers have been introduced to generate + Lisp-style subtraction and division functions. + + Examples of reducers are `SB-SIMD-AVX:F32.8/`, for successively + dividing a pack of 32 bit single floats by all further supplied + packs of 32 bit single floats, or `SB-SIMD-FMA:U32.8-` for + subtracting any number of supplied packs of 32 bit unsigned integers + from the first supplied one, except in the case of a single + argument, where `SB-SIMD-FMA:U32.8-` simply negates all values in + the pack.") + +(defsection @rounding (:title "Rounding") + "For each floating-point SIMD data type `X.Y`, there are several + functions that round the values of a supplied SIMD pack to nearby + floating-point values whose fractional digits are all zero. Those + functions are `X.Y-ROUND`, `X.Y-FLOOR`, `X.Y-CEILING`, and + `X.Y-TRUNCATE`, and they have the same semantics as the one argument + versions of CL:ROUND, CL:FLOOR, CL:CEILING, and CL:TRUNCATE, + respectively.") + +(defsection @comparisons (:title "Comparisons") + "For each SIMD data type `X.Y`, there exist conversion functions + `X.Y<`, `X.Y<=`, `X.Y>`, `X.Y>=`, and `X.Y=` that check whether the + supplied arguments are strictly monotonically increasing, + monotonically increasing, strictly monotonically decreasing, + monotonically decreasing, equal, or nowhere equal, respectively. In + contrast to the Common Lisp functions `<`, `<=`, `>`, `>=`, `=`, and + `/=`, the SIMD comparison functions don't return a generalized + boolean but a SIMD pack of unsigned integers with `Y` elements. + The bits of each unsigned integer are either all one, if the values + of the arguments at that position satisfy the test, or all zero, if + they don't. We call a SIMD packs of such unsigned integers a mask.") + +(defsection @conditionals (:title "Conditionals") + "The SIMD paradigm is inherently incompatible with fine-grained control + flow. A piece of code containing an IF special form cannot be + vectorized in a straightforward way, because doing so would require + as many instruction pointers and processor states as there are + values in the desired SIMD data type. Instead, most SIMD instruction + sets provide an operator for selecting values from one of two + supplied SIMD packs based on a mask. The mask is a SIMD pack with as + many elements as the other two arguments, but whose elements are + unsigned integers whose bits must be either all zeros or all ones. + This selection mechanism can be used to emulate the effect of an IF + special form, at the price that both operands have to be computed + each time. + + In sb-simd, all conditional operations and comparisons emit suitable + mask fields, and there is a `X.Y-IF` function for each SIMD data + type with element type `X` and number of elements `Y` whose first + arguments must be a suitable mask, whose second and third argument + must be objects that can be converted to the SIMD data type `X.Y`, + and that returns a value of type `X.Y` where each element is from + the second operand if the corresponding mask bits are set, and from + the third operand if the corresponding mask bits are not set.") + +(defsection @loads-and-stores (:title "Loads and Stores") + "In practice, a SIMD pack `X.Y` is usually not constructed by + calling its constructor but by loading `Y` consecutive elements from + a specialized array with element type `X`. The functions for doing + so are called `X.Y-AREF` and `X.Y-ROW-MAJOR-AREF`, and have similar + semantics as Common Lisp's AREF and ROW-MAJOR-AREF. In addition to + that, some instruction sets provide the functions + `X.Y-NON-TEMPORAL-AREF` and `X.Y-NON-TEMPORAL-ROW-MAJOR-AREF`, for + accessing a memory location without loading the referenced values + into the CPU's cache. + + For each function `X.Y-FOO` for loading SIMD packs from an array, + there also exists a corresponding function `(SETF X.Y-FOO)` for + storing a SIMD pack in the specified memory location. An exception + to this rule is that some instruction sets (e.g., SSE) only provide + functions for non-temporal stores but not for the corresponding + non-temporal loads. + + One difficulty when treating the data of a Common Lisp array as a + SIMD pack is that some hardware instructions require a particular + alignment of the address being referenced. Luckily, most + architectures provide instructions for unaligned loads and stores + that are, at least on modern CPUs, not slower than their aligned + equivalents. So by default we translate all array references as + unaligned loads and stores. An exception are the instructions for + non-temporal loads and stores, that always require a certain + alignment. We do not handle this case specially, so without special + handling by the user, non-temporal loads and stores will only work + on certain array indices that depend on the actual placement of that + array in memory.") + +(defsection @specialized-scalar-operations + (:title "Specialized Scalar Operations") + "Finally, for each SIMD function `X.Y-OP` that applies a certain + operation `OP` element-wise to the `Y` elements of type `X`, there + exists also a functions `X-OP` for applying that operation only to a + single element. For example, the SIMD function `F64.4+` has a + corresponding function `F64+` that differs from `CL:+` in that it + only accepts arguments of type double float, and that it adds its + supplied arguments in a fixed order that is the same as the one used + by `F64.4`. + + There are good reasons for exporting scalar functions from a SIMD + library, too. The most obvious one is that they obey the same naming + convention and hence make it easier to locate the correct functions. + Another benefit is that the semantics of each scalar operation is + precisely the same as that of the corresponding SIMD function, so + they can be used to write reference implementations for testing. A + final reason is that these scalar functions can be used to simplify + the life of tools for automatic vectorization.") + +(defsection @instruction-set-dispatch (:title "Instruction Set Dispatch") + "One challenge that is unique to image-based programming systems such as + Lisp is that a program can run on one machine, be dumped as an image, + and then resumed on another machine. While nobody expects this feature + to work across machines with different architectures, it is quite likely + that the machine where the image is dumped and the one where execution + is resumed provide different instruction set extensions. + + As a practical example, consider a game developer that develops software + on an x86-64 machine with all SIMD extensions up to AVX2, but then dumps + it as an image and ships it to a customer whose machine only supports + SIMD extensions up to SSE2. Ideally, the image should contain multiple + optimized versions of all crucial functions, and dynamically select the + most appropriate version based on the instruction set extensions that + are actually available. + + This kind of run time instruction set dispatch is explicitly + supported by means of the SB-SIMD-INTERNALS:INSTRUCTION-SET-CASE + macro. The code resulting from an invocation of this macro compiles + to an efficient jump table whose index is recomputed on each startup + of the Lisp image.") diff --git a/contrib/sb-simple-streams/manual.lisp b/contrib/sb-simple-streams/manual.lisp new file mode 100644 index 000000000..cb85fe371 --- /dev/null +++ b/contrib/sb-simple-streams/manual.lisp @@ -0,0 +1,22 @@ +(in-package :sb-manual) + +(defsection @sb-simple-streams (:title "Simple Streams") + "Simple streams are an extensible streams protocol that avoids some + problems with @GRAY-STREAMS. + + Documentation about simple streams is available at: + + <http://www.franz.com/support/documentation/6.2/doc/streams.htm> + + The implementation should be considered Alpha-quality; the basic + framework is there, but many classes are just stubs at the moment. + + See `\\\\SYS:CONTRIB;SB-SIMPLE-STREAMS;SIMPLE-STREAM-TEST.LISP` for + things that should work. + + Known differences to the ACL behaviour: + + - `SB-SIMPLE-STREAMS:OPEN` does not return a `SIMPLE-STREAM` by + default. See its :CLASS argument. + + - `WRITE-VECTOR` is unimplemented.")