Merge sb-simd.

This commit contains the sb-simd contrib in its most recent form.  The
original sb-simd repository will be archived, and development of sb-simd
will now be a part of the regular SBCL development.  Thanks for everyone
who made this work possible!

contrib/Makefile

@@ -3,7 +3,7 @@ vpath %.fasl ../obj/sbcl-home/contrib/
 contribs = asdf sb-posix sb-bsd-sockets sb-introspect sb-cltl2 sb-aclrepl \
      sb-sprof sb-capstone sb-md5 sb-capstone sb-executable sb-gmp sb-mpfr \
      sb-queue sb-rotate-byte sb-simple-streams sb-concurrency sb-cover \
-     sb-graph
+     sb-graph sb-simd
 
 active_contribs = $(filter-out $(CONTRIB_BLOCKLIST),$(contribs))
 
@@ -51,3 +51,5 @@ sb-simple-streams.fasl: asdf.fasl sb-posix.fasl sb-bsd-sockets.fasl sb-rt.fasl
 	sh ./build-contrib $(basename $(@F))
 sb-graph.fasl: asdf.fasl sb-rt.fasl
 	sh ./build-contrib $(basename $(@F))
+sb-simd.fasl: asdf.fasl
+	sh ./build-contrib $(basename $(@F))

contrib/sb-simd/CREDITS

@@ -0,0 +1,8 @@
+sb-simd was developed by:
+
+Marco Heisig
+Bela Pecsek
+Tomas Wain
+Stas Boukarev
+
+The detailed project history is archived at github.com/marcoheisig/sb-simd

contrib/sb-simd/Makefile

@@ -0,0 +1,2 @@
+SYSTEM=sb-simd
+include ../asdf-module.mk

contrib/sb-simd/code/constants.lisp

@@ -0,0 +1,50 @@
+(in-package #:sb-simd-internals)
+
+(defconstant most-positive-f32 most-positive-single-float)
+(defconstant most-negative-f32 most-negative-single-float)
+(defconstant most-positive-f64 most-positive-double-float)
+(defconstant most-negative-f64 most-negative-double-float)
+
+(defconstant most-positive-u1  (1- (expt 2  1)))
+(defconstant most-positive-u2  (1- (expt 2  2)))
+(defconstant most-positive-u4  (1- (expt 2  4)))
+(defconstant most-positive-u8  (1- (expt 2  8)))
+(defconstant most-positive-u16 (1- (expt 2 16)))
+(defconstant most-positive-u32 (1- (expt 2 32)))
+(defconstant most-positive-u64 (1- (expt 2 64)))
+
+(defconstant  +u8-true+ most-positive-u8)
+(defconstant +u16-true+ most-positive-u16)
+(defconstant +u32-true+ most-positive-u32)
+(defconstant +u64-true+ most-positive-u64)
+
+(defconstant  +u8-false+ 0)
+(defconstant +u16-false+ 0)
+(defconstant +u32-false+ 0)
+(defconstant +u64-false+ 0)
+
+(defconstant most-positive-s8  (1- (expt 2 (1-  8))))
+(defconstant most-positive-s16 (1- (expt 2 (1- 16))))
+(defconstant most-positive-s32 (1- (expt 2 (1- 32))))
+(defconstant most-positive-s64 (1- (expt 2 (1- 64))))
+
+(defconstant most-negative-s8  (- (expt 2 (1-  8))))
+(defconstant most-negative-s16 (- (expt 2 (1- 16))))
+(defconstant most-negative-s32 (- (expt 2 (1- 32))))
+(defconstant most-negative-s64 (- (expt 2 (1- 64))))
+
+(defconstant  +s8-true+ -1)
+(defconstant +s16-true+ -1)
+(defconstant +s32-true+ -1)
+(defconstant +s64-true+ -1)
+
+(defconstant  +s8-false+ 0)
+(defconstant +s16-false+ 0)
+(defconstant +s32-false+ 0)
+(defconstant +s64-false+ 0)
+
+(defconstant +f32-true+ (sb-kernel:make-single-float +s32-true+))
+(defconstant +f64-true+ (sb-kernel:make-double-float +s32-true+ +u32-true+))
+
+(defconstant +f32-false+ (sb-kernel:make-single-float +s32-false+))
+(defconstant +f64-false+ (sb-kernel:make-double-float +s32-false+ +u32-false+))

contrib/sb-simd/code/cpu-identification.lisp

@@ -0,0 +1,72 @@
+(in-package #:sb-simd-internals)
+
+#+x86-64
+(progn
+  (defun cpuid (eax &optional (ecx 0))
+    (declare (type (unsigned-byte 32) eax ecx))
+    (sb-vm::%cpu-identification eax ecx))
+
+  (defun sse-supported-p ()
+    (and (>= (cpuid 0) 1)
+         (logbitp 25 (nth-value 3 (cpuid 1)))))
+
+  (defun sse2-supported-p ()
+    (and (>= (cpuid 0) 1)
+         (logbitp 26 (nth-value 3 (cpuid 1)))))
+
+  (defun sse3-supported-p ()
+    (and (>= (cpuid 0) 1)
+         (logbitp 0 (nth-value 2 (cpuid 1)))))
+
+  (defun ssse3-supported-p ()
+    (and (>= (cpuid 0) 1)
+         (logbitp 9 (nth-value 2 (cpuid 1)))))
+
+  (defun sse4.1-supported-p ()
+    (and (>= (cpuid 0) 1)
+         (logbitp 19 (nth-value 2 (cpuid 1)))))
+
+  (defun sse4.2-supported-p ()
+    (and (>= (cpuid 0) 1)
+         (logbitp 20 (nth-value 2 (cpuid 1)))))
+
+  (defun avx-supported-p ()
+    (and (>= (cpuid 0) 1)
+         (logbitp 28 (nth-value 2 (cpuid 1)))))
+
+  (defun avx2-supported-p ()
+    (and (>= (cpuid 0) 7)
+         (logbitp 5 (nth-value 1 (cpuid 7)))))
+
+  (defun fma-supported-p ()
+    (and (>= (cpuid 0) 1)
+         (logbitp 12 (nth-value 2 (cpuid 1))))))
+
+#-x86-64
+(progn
+  (defun sse-supported-p ()
+    nil)
+
+  (defun sse2-supported-p ()
+    nil)
+
+  (defun sse3-supported-p ()
+    nil)
+
+  (defun ssse3-supported-p ()
+    nil)
+
+  (defun sse4.1-supported-p ()
+    nil)
+
+  (defun sse4.2-supported-p ()
+    nil)
+
+  (defun avx-supported-p ()
+    nil)
+
+  (defun avx2-supported-p ()
+    nil)
+
+  (defun fma-supported-p ()
+    nil))

contrib/sb-simd/code/define-arefs.lisp

@@ -0,0 +1,179 @@
+(in-package #:sb-simd-internals)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;
+;;; Auxiliary Functions and Macros
+
+(declaim (notinline wrong-number-of-subscripts))
+(defun wrong-number-of-subscripts (array number-of-subscripts)
+  (error "Wrong number of subcripts, ~S, for an array of rank ~S."
+         number-of-subscripts
+         (array-rank array)))
+
+(declaim (notinline invalid-subscript))
+(defun invalid-subscript (subscript array axis limit)
+  (declare (ignore array))
+  (error "Invalid array subscript ~S for axis ~S, ~
+          should be a non-negative integer below ~S."
+         subscript axis limit))
+
+;; This function doesn't have to be particularly fast, because all index
+;; computations where the number of subscripts is known at compile time are
+;; expanded by compiler macros and WITH-ROW-MAJOR-SIMD-INDEX.  The only way
+;; to reach this function is when an array indexing function is supplied as
+;; the first argument to APPLY or FUNCALL.
+(defun array-row-major-simd-index (array simd-width &rest subscripts)
+  (let ((rank (array-rank array))
+        (length (length subscripts)))
+    (unless (= rank length)
+      (wrong-number-of-subscripts array length))
+    (let ((stride 1)
+          (index 0))
+      (declare (index stride index))
+      (loop for axis from (1- rank) downto 0
+            for subscript = (nth axis subscripts)
+            for dimension = (array-dimension array axis)
+            for width = simd-width then 1 do
+              (unless (<= -1 subscript (- dimension width))
+                (invalid-subscript subscript array axis (1+ (- dimension width))))
+              (incf index (* stride subscript))
+              (setf stride (* stride dimension)))
+      index)))
+
+(defmacro with-row-major-simd-index
+    ((index array simd-width &rest indices) &body body &environment env)
+  (check-type index symbol)
+  (check-type array symbol)
+  (check-type simd-width (integer 1))
+  (dolist (index indices)
+    (check-type index symbol))
+  (let* ((length (length indices))
+         (rank-binding `(,(gensym "RANK") (array-rank ,array)))
+         (rank (first rank-binding))
+         (dimension-bindings
+           (loop for axis below length
+                 collect `(,(gensym "DIMENSION") (array-dimension ,array ,axis))))
+         (dimensions (mapcar #'first dimension-bindings))
+         (stride-bindings
+           (loop for axis from (- length 2) downto 0
+                 for old-stride = nil then new-stride
+                 for new-stride = (gensym "STRIDE")
+                 for stride-binding = `(,new-stride ,(nth axis dimensions))
+                   then `(,new-stride (index* ,(nth axis dimensions) ,old-stride))
+                 collect stride-binding))
+         (strides (reverse (mapcar #'first stride-bindings)))
+         (index-form
+           `(index+
+             ,@(loop for stride in strides
+                     for index in indices
+                     collect `(index* ,stride ,index))
+             ,(first (last indices)))))
+    `(let (,rank-binding)
+       (unless (= ,rank ,length)
+         (wrong-number-of-subscripts ,array ,length))
+       (let (,@dimension-bindings)
+         (declare (ignorable ,@dimensions))
+         ,@(when (sb-c:policy env (plusp sb-c::insert-array-bounds-checks))
+             (loop for axis from 0
+                   for dimension in dimensions
+                   for index in indices
+                   for limit = (if (= axis (1- length)) `(- ,dimension ,(1- simd-width)) dimension)
+                   collect `(unless (< -1 ,index ,limit)
+                              (invalid-subscript ,index ,array ,axis ,limit))))
+         (let* (,@stride-bindings)
+           (let ((,index ,index-form))
+             ,@body))))))
+
+(defun setf-row-major-aref (v a i)
+  (setf (row-major-aref a i) v))
+
+(define-compiler-macro setf-row-major-aref (v a i)
+  (let* ((v-binding `(,(gensym "VALUE") ,v))
+         (v (first v-binding)))
+    `(let (,v-binding)
+       (setf (row-major-aref ,a ,i) ,v))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;
+;;; Array Load and Store Instructions
+
+(macrolet
+    ((define-aref (load-record-name)
+       (with-accessors ((load load-record-name)
+                        (aref load-record-aref)
+                        (row-major-aref load-record-row-major-aref)
+                        (value-record load-record-value-record)
+                        (vector-record load-record-vector-record))
+           (find-function-record load-record-name)
+         (let ((simd-width (value-record-simd-width value-record))
+               (element-type
+                 (second
+                  (value-record-type vector-record))))
+           `(progn
+              (define-inline ,row-major-aref (array index)
+                (declare (type (array ,element-type) array)
+                         (index index))
+                (,load array index))
+              (defun ,aref (array &rest indices)
+                (declare (type (array ,element-type) array))
+                (,load
+                 array
+                 (apply #'array-row-major-simd-index array ,simd-width indices)))
+              (define-compiler-macro ,aref (array &rest indices)
+                (let* ((index (gensym "INDEX"))
+                       (array-binding `(,(gensym "ARRAY") ,array))
+                       (index-bindings
+                         (loop for index-form in indices
+                               collect `(,(gensym "INDEX") ,index-form)))
+                       (array (first array-binding))
+                       (indices (mapcar #'first index-bindings)))
+                  `(let (,array-binding ,@index-bindings)
+                     (declare (type (array ,',element-type) ,array))
+                     (with-row-major-simd-index (,index ,array ,',simd-width ,@indices)
+                       (,',load ,array ,index)))))))))
+     (define-setf-aref (store-record-name)
+       (with-accessors ((store store-record-name)
+                        (aref store-record-aref)
+                        (row-major-aref store-record-row-major-aref)
+                        (value-record store-record-value-record)
+                        (vector-record store-record-vector-record))
+           (find-function-record store-record-name)
+         (let ((value-type (value-record-name value-record))
+               (simd-width (value-record-simd-width value-record))
+               (element-type
+                 (second
+                  (value-record-type vector-record))))
+           `(progn
+              (define-inline (setf ,row-major-aref) (value array index)
+                (declare (type (array ,element-type) array)
+                         (index index))
+                (,store (,value-type value) array index))
+              (defun (setf ,aref) (value array &rest indices)
+                (declare (type (array ,element-type) array))
+                (,store
+                 (,value-type value)
+                 array
+                 (apply #'array-row-major-simd-index array ,simd-width indices)))
+              (define-compiler-macro (setf ,aref) (value array &rest indices)
+                (let* ((value-binding `(,(gensym "VALUE") ,value))
+                       (array-binding `(,(gensym "ARRAY") ,array))
+                       (index-bindings
+                         (loop for index-form in indices
+                               collect `(,(gensym "INDEX") ,index-form)))
+                       (indices (mapcar #'first index-bindings))
+                       (value (first value-binding))
+                       (array (first array-binding))
+                       (index (gensym "INDEX")))
+                  `(let (,value-binding ,array-binding ,@index-bindings)
+                     (declare (type (array ,',element-type) ,array))
+                     (with-row-major-simd-index (,index ,array ,',simd-width ,@indices)
+                       (,',store (,',value-type ,value) ,array ,index)))))))))
+     (define-arefs ()
+       `(progn
+          ,@(loop for load-record in (filter-function-records #'load-record-p)
+                  for name = (load-record-name load-record)
+                  collect `(define-aref ,name))
+          ,@(loop for store-record in (filter-function-records #'store-record-p)
+                  for name = (store-record-name store-record)
+                  collect `(define-setf-aref ,name)))))
+  (define-arefs))

contrib/sb-simd/code/define-associatives.lisp

@@ -0,0 +1,58 @@
+(in-package #:sb-simd-internals)
+
+(macrolet
+    ((define-associative (associative-record-name)
+       (with-accessors ((name associative-record-name)
+                        (binary-operation associative-record-binary-operation)
+                        (identity-element associative-record-identity-element))
+           (find-function-record associative-record-name)
+         (with-accessors ((binary-operation instruction-record-name)
+                          (result-records instruction-record-result-records)
+                          (argument-records instruction-record-argument-records)
+                          (associative instruction-record-associative)) binary-operation
+           (assert associative)
+           (destructuring-bind ((value-record) (arg1-record arg2-record))
+               (list result-records argument-records)
+             (assert (eq value-record arg1-record))
+             (assert (eq value-record arg2-record))
+             (let ((type (value-record-name value-record)))
+               (if (not identity-element)
+                   `(progn
+                      (defun ,name (arg &rest more-args)
+                        (let ((result (,type arg)))
+                          (declare (,type result))
+                          (loop for arg in more-args
+                                do (setf result (,binary-operation result (,type arg))))
+                          result))
+                      (define-compiler-macro ,name (&whole whole &rest args)
+                        (let ((n (length args)))
+                          (case n
+                            (0 whole)
+                            (1 `(,',type ,(first args)))
+                            (otherwise
+                             `(,',binary-operation
+                               (,',name ,@(subseq args 0 (floor n 2)))
+                               (,',name ,@(subseq args (floor n 2)))))))))
+                   `(progn
+                      (defun ,name (&rest args)
+                        (if (null args)
+                            (,type ,identity-element)
+                            (let ((result (,type (first args))))
+                              (declare (,type result))
+                              (loop for arg in (rest args)
+                                    do (setf result (,binary-operation result (,type arg))))
+                              result)))
+                      (define-compiler-macro ,name (&rest args)
+                        (let ((n (length args)))
+                          (case n
+                            (0 `(,',type ,,identity-element))
+                            (1 `(,',type ,(first args)))
+                            (otherwise
+                             `(,',binary-operation
+                               (,',name ,@(subseq args 0 (floor n 2)))
+                               (,',name ,@(subseq args (floor n 2)))))))))))))))
+     (define-associatives ()
+       `(progn
+          ,@(loop for associative-record in (filter-function-records #'associative-record-p)
+                  collect `(define-associative ,(associative-record-name associative-record))))))
+  (define-associatives))

contrib/sb-simd/code/define-comparisons.lisp

@@ -0,0 +1,47 @@
+(in-package #:sb-simd-internals)
+
+(macrolet
+    ((define-comparison (comparision-record-name)
+       (with-accessors ((name comparison-record-name)
+                        (cmp comparison-record-cmp)
+                        (and comparison-record-and)
+                        (truth comparison-record-truth))
+           (find-function-record comparision-record-name)
+         (with-accessors ((cmp instruction-record-name)
+                          (result-records instruction-record-result-records)
+                          (argument-records instruction-record-argument-records)) cmp
+           (destructuring-bind ((result-record) (argument-record other-argument-record))
+               (list result-records argument-records)
+             (assert (eq argument-record other-argument-record))
+             (let ((and (function-record-name and))
+                   (result-type (value-record-name result-record))
+                   (argument-type (value-record-name argument-record)))
+               `(progn
+                  (defun ,name (arg &rest more-args)
+                    (if (null more-args)
+                        (progn (,argument-type arg) (,result-type ,truth))
+                        (let* ((a (,argument-type arg))
+                               (b (,argument-type (first more-args)))
+                               (result (,cmp a b)))
+                          (declare (,argument-type a b)
+                                   (,result-type result))
+                          (loop for elt in (rest more-args)
+                                do (shiftf a b (,argument-type elt))
+                                do (setf result (,and result (,cmp a b))))
+                          result)))
+                  (define-compiler-macro ,name (arg &rest more-args)
+                    (if (null more-args)
+                        `(progn (,',argument-type ,arg) (,',result-type ,',truth))
+                        (let ((bindings
+                                (loop for arg in (list* arg more-args)
+                                      collect
+                                      (list (gensym "ARG") (list ',argument-type arg)))))
+                          `(let ,bindings
+                             (,',and ,@(loop for ((a nil) (b nil) . rest) on bindings
+                                             collect `(,',cmp ,a ,b)
+                                             until (null rest)))))))))))))
+     (define-comparisons ()
+       `(progn
+          ,@(loop for comparison-record in (filter-function-records #'comparison-record-p)
+                  collect `(define-comparison ,(comparison-record-name comparison-record))))))
+  (define-comparisons))

contrib/sb-simd/code/define-custom-vops.lisp

@@ -0,0 +1,167 @@
+(in-package #:sb-vm)
+
+(macrolet
+    ((define-custom-vop (name &body clauses)
+       (with-accessors ((name sb-simd-internals:instruction-record-name)
+                        (vop sb-simd-internals:instruction-record-vop)
+                        (argument-records sb-simd-internals:instruction-record-argument-records)
+                        (result-records sb-simd-internals:instruction-record-result-records)
+                        (cost sb-simd-internals:instruction-record-cost)
+                        (encoding sb-simd-internals:instruction-record-encoding))
+           (sb-simd-internals:find-function-record name)
+         (assert (eq encoding :custom))
+         (labels ((find-clauses (key)
+                    (remove key clauses :test-not #'eq :key #'first))
+                  (find-clause (key)
+                    (let ((found (find-clauses key)))
+                      (assert (= 1 (length found)))
+                      (rest (first found)))))
+           `(sb-c:define-vop (,vop)
+              (:translate ,vop)
+              (:policy :fast-safe)
+              (:arg-types ,@(mapcar #'sb-simd-internals:value-record-primitive-type argument-records))
+              (:result-types ,@(mapcar #'sb-simd-internals:value-record-primitive-type result-records))
+              (:args
+               ,@(loop for arg in (find-clause :args)
+                       for argument-record in argument-records
+                       collect `(,@arg :scs ,(sb-simd-internals:value-record-scs argument-record))))
+              ,@(find-clauses :info)
+              ,@(find-clauses :temporary)
+              (:results
+               ,@(loop for result in (find-clause :results)
+                       for result-record in result-records
+                       collect `(,@result :scs ,(sb-simd-internals:value-record-scs result-record))))
+              (:generator ,cost ,@(find-clause :generator)))))))
+  ;; SSE
+  (macrolet ((def (name cmp)
+               `(define-custom-vop ,name
+                    (:args (a :target tmp) (b))
+                  (:temporary (:sc single-reg :from (:argument 0)) tmp)
+                  (:results (dst))
+                  (:generator
+                   (unless (location= a tmp)
+                     (inst xorps tmp tmp)
+                     (inst movss tmp a))
+                   (inst cmpss ,cmp tmp b)
+                   (inst movq dst tmp)))))
+    (def sb-simd-sse::two-arg-f32= :eq)
+    (def sb-simd-sse::two-arg-f32/= :neq)
+    (def sb-simd-sse::two-arg-f32< :lt)
+    (def sb-simd-sse::two-arg-f32<= :le)
+    (def sb-simd-sse::two-arg-f32> :nle)
+    (def sb-simd-sse::two-arg-f32>= :nlt))
+  (define-custom-vop sb-simd-sse::f32-from-s64
+    (:args (src))
+    (:results (dst))
+    (:generator
+     (inst xorps dst dst)
+     (inst cvtsi2ss dst src)))
+  (define-custom-vop sb-simd-sse::f32!-from-p128
+    (:args (src :target dst))
+    (:temporary (:sc single-sse-reg :from (:argument 0)) tmp)
+    (:results (dst))
+    (:generator
+     (move tmp src)
+     (inst xorps dst dst)
+     (inst movss dst tmp)))
+  ;; SSE2
+  (macrolet ((def (name cmp)
+               `(define-custom-vop ,name
+                  (:args (a :target tmp) (b))
+                  (:temporary (:sc single-reg :from (:argument 0)) tmp)
+                  (:results (dst))
+                  (:generator
+                   (unless (location= a tmp)
+                     (inst xorpd tmp tmp)
+                     (inst movsd tmp a))
+                   (inst cmpsd ,cmp tmp b)
+                   (inst movq dst tmp)))))
+    (def sb-simd-sse2::two-arg-f64= :eq)
+    (def sb-simd-sse2::two-arg-f64/= :neq)
+    (def sb-simd-sse2::two-arg-f64< :lt)
+    (def sb-simd-sse2::two-arg-f64<= :le)
+    (def sb-simd-sse2::two-arg-f64> :nle)
+    (def sb-simd-sse2::two-arg-f64>= :nlt))
+  (define-custom-vop sb-simd-sse2::f64-from-s64
+    (:args (src))
+    (:results (dst))
+    (:generator
+     (inst xorpd dst dst)
+     (inst cvtsi2sd dst src)))
+  (define-custom-vop sb-simd-sse2::f64!-from-p128
+    (:args (src :target tmp))
+    (:temporary (:sc double-sse-reg :from (:argument 0)) tmp)
+    (:results (dst))
+    (:generator
+     (move tmp src)
+     (inst xorpd dst dst)
+     (inst movsd dst tmp)))
+  ;; AVX
+  (macrolet ((def (name cmp)
+               `(define-custom-vop ,name
+                  (:args (a :target tmp) (b))
+                  (:temporary (:sc single-reg :from (:argument 0)) tmp)
+                  (:results (dst))
+                  (:generator
+                   (unless (location= a tmp)
+                     (inst vxorps tmp tmp tmp))
+                   (inst vcmpss ,cmp tmp a b)
+                   (inst vmovq dst tmp)))))
+    (def sb-simd-avx::two-arg-f32= :eq)
+    (def sb-simd-avx::two-arg-f32/= :neq)
+    (def sb-simd-avx::two-arg-f32< :lt)
+    (def sb-simd-avx::two-arg-f32<= :le)
+    (def sb-simd-avx::two-arg-f32> :nle)
+    (def sb-simd-avx::two-arg-f32>= :nlt))
+  (macrolet ((def (name cmp)
+               `(define-custom-vop ,name
+                  (:args (a :target tmp) (b))
+                  (:temporary (:sc single-reg :from (:argument 0)) tmp)
+                  (:results (dst))
+                  (:generator
+                   (unless (location= a tmp)
+                     (inst vxorpd tmp tmp tmp))
+                   (inst vcmpsd ,cmp tmp a b)
+                   (inst vmovq dst tmp)))))
+    (def sb-simd-avx::two-arg-f64= :eq)
+    (def sb-simd-avx::two-arg-f64/= :neq)
+    (def sb-simd-avx::two-arg-f64< :lt)
+    (def sb-simd-avx::two-arg-f64<= :le)
+    (def sb-simd-avx::two-arg-f64> :nle)
+    (def sb-simd-avx::two-arg-f64>= :nlt))
+  (define-custom-vop sb-simd-avx::f32-from-s64
+    (:args (src :to :save))
+    (:results (dst))
+    (:generator
+     (inst vxorpd dst dst dst)
+     (inst vcvtsi2ss dst dst src)))
+  (define-custom-vop sb-simd-avx::f64-from-s64
+    (:args (src :to :save))
+    (:results (dst))
+    (:generator
+     (inst vxorpd dst dst dst)
+     (inst vcvtsi2sd dst dst src)))
+  (define-custom-vop sb-simd-avx::f32!-from-p128
+    (:args (src :to :save))
+    (:results (dst))
+    (:generator
+     (inst vxorps dst dst dst)
+     (inst movss dst src)))
+  (define-custom-vop sb-simd-avx::f32!-from-p256
+    (:args (src :to :save))
+    (:results (dst))
+    (:generator
+     (inst vxorps dst dst dst)
+     (inst movss dst src)))
+  (define-custom-vop sb-simd-avx::f64!-from-p128
+    (:args (src :to :save))
+    (:results (dst))
+    (:generator
+     (inst vxorpd dst dst dst)
+     (inst movsd dst src)))
+  (define-custom-vop sb-simd-avx::f64!-from-p256
+    (:args (src :to :save))
+    (:results (dst))
+    (:generator
+     (inst vxorpd dst dst dst)
+     (inst movsd dst src))))

contrib/sb-simd/code/define-ifs.lisp

@@ -0,0 +1,28 @@
+(in-package #:sb-simd-internals)
+
+(macrolet
+    ((define-if (if-record-name)
+       (with-accessors ((name if-record-name)
+                        (blend if-record-blend))
+           (find-function-record if-record-name)
+         (with-accessors ((blend instruction-record-name)
+                          (result-records instruction-record-result-records)
+                          (argument-records instruction-record-argument-records)) blend
+           (destructuring-bind (a-record b-record mask-record) argument-records
+             (assert (eq a-record b-record))
+             (assert (= (value-record-bits a-record)
+                        (value-record-bits mask-record)))
+             (destructuring-bind (result-record) result-records
+               (assert (eq result-record a-record))
+               (let ((value-type (value-record-name a-record))
+                     (mask-type (value-record-name mask-record)))
+                 `(define-inline ,name (mask a b)
+                    (the ,value-type
+                         (,blend (,value-type b)
+                                 (,value-type a)
+                                 (,mask-type mask))))))))))
+     (define-ifs ()
+       `(progn
+          ,@(loop for if-record in (filter-function-records #'if-record-p)
+                  collect `(define-if ,(if-record-name if-record))))))
+  (define-ifs))

contrib/sb-simd/code/define-instruction-vops.lisp

@@ -0,0 +1,181 @@
+(in-package #:sb-vm)
+
+(macrolet
+    ((define-instruction-vop (instruction-record-name)
+       (with-accessors ((name sb-simd-internals:instruction-record-name)
+                        (vop sb-simd-internals:instruction-record-vop)
+                        (mnemonic sb-simd-internals:instruction-record-mnemonic)
+                        (argument-records sb-simd-internals:instruction-record-argument-records)
+                        (result-records sb-simd-internals:instruction-record-result-records)
+                        (cost sb-simd-internals:instruction-record-cost)
+                        (pure sb-simd-internals:instruction-record-pure)
+                        (always-translatable sb-simd-internals:instruction-record-always-translatable)
+                        (associative sb-simd-internals:instruction-record-associative)
+                        (prefix sb-simd-internals:instruction-record-prefix)
+                        (suffix sb-simd-internals:instruction-record-suffix)
+                        (encoding sb-simd-internals:instruction-record-encoding))
+           (sb-simd-internals:find-function-record instruction-record-name)
+         (let* ((asyms (sb-simd-internals:prefixed-symbols "A" (length argument-records)))
+                (rsyms (sb-simd-internals:prefixed-symbols "R" (length result-records)))
+                (defknown
+                    `(defknown ,vop
+                         (,@(mapcar #'sb-simd-internals:value-record-name argument-records))
+                         (values ,@(mapcar #'sb-simd-internals:value-record-name result-records) &optional)
+                         (,@(when (and always-translatable (not (eq encoding :fake-vop)))
+                              '(always-translatable))
+                          ,@(when pure '(foldable flushable movable)))
+                       :overwrite-fndb-silently t))
+                (arg-types
+                  (mapcar #'sb-simd-internals:value-record-primitive-type argument-records))
+                (result-types
+                  (mapcar #'sb-simd-internals:value-record-primitive-type result-records))
+                (args
+                  (loop for asym in asyms
+                        for argument-record in argument-records
+                        when (symbolp (sb-simd-internals:value-record-primitive-type argument-record))
+                          collect `(,asym :scs ,(sb-simd-internals:value-record-scs argument-record))))
+                (info
+                  (loop for asym in asyms
+                        for argument-record in argument-records
+                        unless (symbolp (sb-simd-internals:value-record-primitive-type argument-record))
+                          collect asym))
+                (results
+                  (loop for rsym in rsyms
+                        for result-record in result-records
+                        collect `(,rsym :scs ,(sb-simd-internals:value-record-scs result-record)))))
+           (ecase encoding
+             ((:fake-vop :custom)
+              `(progn ,defknown))
+             (:standard
+              (assert mnemonic)
+              `(progn
+                 ,defknown
+                 (define-vop (,vop)
+                   (:translate ,vop)
+                   (:policy :fast-safe)
+                   (:args ,@args)
+                   (:info ,@info)
+                   (:results ,@results)
+                   (:arg-types ,@arg-types)
+                   (:result-types ,@result-types)
+                   (:generator
+                    ,cost
+                    (inst ,mnemonic ,@prefix ,@rsyms ,@asyms ,@suffix)))))
+             (:move
+              (assert mnemonic)
+              (let ((src (first asyms))
+                    (dst (first rsyms)))
+                `(progn
+                   ,defknown
+                   (define-vop (,vop)
+                     (:translate ,vop)
+                     (:policy :fast-safe)
+                     (:args (,@(first args) :target ,dst) ,@(rest args))
+                     (:info ,@info)
+                     (:results ,@results)
+                     (:arg-types ,@arg-types)
+                     (:result-types ,@result-types)
+                     (:generator
+                      ,cost
+                      (unless (location= ,dst ,src)
+                        (inst ,mnemonic ,@prefix ,@rsyms ,@asyms ,@suffix)))))))
+             (:sse
+              (assert mnemonic)
+              (let ((x (first asyms))
+                    (y (second asyms))
+                    (rest (rest (rest asyms)))
+                    (r (first rsyms)))
+                `(progn
+                   ,defknown
+                   (define-vop (,vop)
+                     (:translate ,vop)
+                     (:policy :fast-safe)
+                     (:args (,@(first args) :target ,r) ,@(rest args))
+                     (:temporary (:sc ,(first (sb-simd-internals:value-record-scs (first argument-records)))) tmp)
+                     (:info ,@info)
+                     (:results ,@results)
+                     (:arg-types ,@arg-types)
+                     (:result-types ,@result-types)
+                     (:generator
+                      ,cost
+                      (cond ((location= ,x ,r)
+                             (inst ,mnemonic ,@prefix ,r ,y ,@rest ,@suffix))
+                            ((or (not (tn-p ,y))
+                                 (not (location= ,y ,r)))
+                             (move ,r ,x)
+                             (inst ,mnemonic ,@prefix ,r ,y ,@rest ,@suffix))
+                            (t
+                             (move tmp ,x)
+                             (inst ,mnemonic ,@prefix tmp ,y ,@rest ,@suffix)
+                             (move ,r tmp))))))))
+             (:sse+xmm0
+              (assert mnemonic)
+              (let ((x (first asyms))
+                    (y (second asyms))
+                    (z (third asyms))
+                    (r (first rsyms)))
+                `(progn
+                   ,defknown
+                   (define-vop (,vop)
+                     (:translate ,vop)
+                     (:policy :fast-safe)
+                     (:args (,@(first args) :target ,r) ,(second args) (,@(third args) :target xmm0))
+                     (:temporary (:sc ,(first (sb-simd-internals:value-record-scs (first argument-records)))) tmp)
+                     (:temporary (:sc ,(first (sb-simd-internals:value-record-scs (second argument-records)))
+                                  :from (:argument 0) :to :result :offset 0) xmm0)
+                     (:info ,@info)
+                     (:results ,@results)
+                     (:arg-types ,@arg-types)
+                     (:result-types ,@result-types)
+                     (:generator
+                      ,cost
+                      (move xmm0 ,z)
+                      (cond ((location= ,x ,r)
+                             (inst ,mnemonic ,@prefix ,r ,y xmm0 ,@suffix))
+                            ((or (not (tn-p ,y))
+                                 (not (location= ,y ,r)))
+                             (move ,r ,x)
+                             (inst ,mnemonic ,@prefix ,r ,y xmm0 ,@suffix))
+                            (t
+                             (move tmp ,x)
+                             (inst ,mnemonic ,@prefix tmp ,y xmm0 ,@suffix)
+                             (move ,r tmp))))))))
+             (:fma
+              (assert mnemonic)
+              (let ((x (first asyms))
+                    (y (second asyms))
+                    (z (third asyms))
+                    (rest (rest (rest (rest asyms))))
+                    (r (first rsyms)))
+                `(progn
+                   ,defknown
+                   (define-vop (,vop)
+                     (:translate ,vop)
+                     (:policy :fast-safe)
+                     (:args (,@(first args) :target ,r) ,@(rest args))
+                     (:temporary (:sc ,(first (sb-simd-internals:value-record-scs (first argument-records)))) tmp)
+                     (:info ,@info)
+                     (:results ,@results)
+                     (:arg-types ,@arg-types)
+                     (:result-types ,@result-types)
+                     (:generator
+                      ,cost
+                      (cond ((location= ,x ,r)
+                             (inst ,mnemonic ,@prefix ,r ,y ,z ,@rest ,@suffix))
+                            ((and (or (not (tn-p ,y))
+                                      (not (location= ,y ,r)))
+                                  (or (not (tn-p ,z))
+                                      (not (location= ,z ,r))))
+                             (move ,r ,x)
+                             (inst ,mnemonic ,@prefix ,r ,y ,z ,@rest ,@suffix))
+                            (t
+                             (move tmp ,x)
+                             (inst ,mnemonic ,@prefix tmp ,y ,z ,@rest ,@suffix)
+                             (move ,r tmp))))))))))))
+     (define-instruction-vops ()
+       `(progn
+          ,@(loop for instruction-record
+                    in (sb-simd-internals:filter-available-function-records
+                        #'sb-simd-internals:instruction-record-p)
+                  collect `(define-instruction-vop ,(sb-simd-internals:instruction-record-name instruction-record))))))
+  (define-instruction-vops))

contrib/sb-simd/code/define-instructions.lisp

@@ -0,0 +1,30 @@
+(in-package #:sb-simd-internals)
+
+(macrolet
+    ((define-instruction (name)
+       (with-accessors ((name instruction-record-name)
+                        (vop instruction-record-vop)
+                        (argument-records instruction-record-argument-records)
+                        (encoding instruction-record-encoding)
+                        (instruction-set instruction-record-instruction-set))
+           (find-function-record name)
+         (let ((argument-record-names (mapcar #'record-name argument-records))
+               (argument-symbols (prefixed-symbols "ARGUMENT-" (length argument-records))))
+           (if (not (instruction-set-available-p instruction-set))
+               `(define-missing-instruction ,name
+                  :required-arguments ,argument-symbols)
+               ;; Define the actual instruction as a wrapper around the VOP
+               ;; that attempts to cast all arguments to the correct types.
+               `(define-inline ,name ,argument-symbols
+                  (let ,(loop for argument-symbol in argument-symbols
+                              for type in (mapcar #'value-record-name argument-records)
+                              collect `(,argument-symbol (,type ,argument-symbol)))
+                    (with-primitive-arguments
+                        ,(mapcar #'list argument-symbols argument-record-names)
+                      (,vop ,@argument-symbols))))))))
+     (define-instructions ()
+       `(progn
+          ,@(loop for instruction-record in (filter-function-records #'instruction-record-p)
+                  for name = (instruction-record-name instruction-record)
+                  collect `(define-instruction ,name)))))
+  (define-instructions))

contrib/sb-simd/code/define-modify-macros.lisp

@@ -0,0 +1,142 @@
+(in-package #:sb-simd)
+
+(define-modify-macro f32-incf (&optional (num 1f0)) two-arg-f32+)
+(define-modify-macro f32-decf (&optional (num 1f0)) two-arg-f32-)
+
+(define-modify-macro f64-incf (&optional (num 1d0)) two-arg-f64+)
+(define-modify-macro f64-decf (&optional (num 1d0)) two-arg-f64-)
+
+(define-modify-macro u8-incf (&optional (num 1)) two-arg-u8+)
+(define-modify-macro u8-decf (&optional (num 1)) two-arg-u8-)
+
+(define-modify-macro u16-incf (&optional (num 1)) two-arg-u16+)
+(define-modify-macro u16-decf (&optional (num 1)) two-arg-u16-)
+
+(define-modify-macro u32-incf (&optional (num 1)) two-arg-u32+)
+(define-modify-macro u32-decf (&optional (num 1)) two-arg-u32-)
+
+(define-modify-macro u64-incf (&optional (num 1)) two-arg-u64+)
+(define-modify-macro u64-decf (&optional (num 1)) two-arg-u64-)
+
+(define-modify-macro s8-incf (&optional (num 1)) two-arg-s8+)
+(define-modify-macro s8-decf (&optional (num 1)) two-arg-s8-)
+
+(define-modify-macro s16-incf (&optional (num 1)) two-arg-s16+)
+(define-modify-macro s16-decf (&optional (num 1)) two-arg-s16-)
+
+(define-modify-macro s32-incf (&optional (num 1)) two-arg-s32+)
+(define-modify-macro s32-decf (&optional (num 1)) two-arg-s32-)
+
+(define-modify-macro s64-incf (&optional (num 1)) two-arg-s64+)
+(define-modify-macro s64-decf (&optional (num 1)) two-arg-s64-)
+
+(in-package #:sb-simd-sse)
+
+(define-modify-macro f32-incf (&optional (num 1f0)) two-arg-f32+)
+(define-modify-macro f32-decf (&optional (num 1f0)) two-arg-f32-)
+
+(define-modify-macro f32.4-incf (&optional (num 1f0)) two-arg-f32.4+)
+(define-modify-macro f32.4-decf (&optional (num 1f0)) two-arg-f32.4-)
+
+(in-package #:sb-simd-sse2)
+
+(define-modify-macro f64-incf (&optional (num 1d0)) two-arg-f64+)
+(define-modify-macro f64-decf (&optional (num 1d0)) two-arg-f64-)
+
+(define-modify-macro f64.2-incf (&optional (num 1d0)) two-arg-f64.2+)
+(define-modify-macro f64.2-decf (&optional (num 1d0)) two-arg-f64.2-)
+
+(define-modify-macro u8.16-incf (&optional (num 1)) two-arg-u8.16+)
+(define-modify-macro u8.16-decf (&optional (num 1)) two-arg-u8.16-)
+
+(define-modify-macro u16.8-incf (&optional (num 1)) two-arg-u16.8+)
+(define-modify-macro u16.8-decf (&optional (num 1)) two-arg-u16.8-)
+
+(define-modify-macro u32.4-incf (&optional (num 1)) two-arg-u32.4+)
+(define-modify-macro u32.4-decf (&optional (num 1)) two-arg-u32.4-)
+
+(define-modify-macro u64.2-incf (&optional (num 1)) two-arg-u64.2+)
+(define-modify-macro u64.2-decf (&optional (num 1)) two-arg-u64.2-)
+
+(define-modify-macro s8.16-incf (&optional (num 1)) two-arg-s8.16+)
+(define-modify-macro s8.16-decf (&optional (num 1)) two-arg-s8.16-)
+
+(define-modify-macro s16.8-incf (&optional (num 1)) two-arg-s16.8+)
+(define-modify-macro s16.8-decf (&optional (num 1)) two-arg-s16.8-)
+
+(define-modify-macro s32.4-incf (&optional (num 1)) two-arg-s32.4+)
+(define-modify-macro s32.4-decf (&optional (num 1)) two-arg-s32.4-)
+
+(define-modify-macro s64.2-incf (&optional (num 1)) two-arg-s64.2+)
+(define-modify-macro s64.2-decf (&optional (num 1)) two-arg-s64.2-)
+
+(in-package #:sb-simd-avx)
+
+(define-modify-macro f32-incf (&optional (num 1f0)) two-arg-f32+)
+(define-modify-macro f32-decf (&optional (num 1f0)) two-arg-f32-)
+
+(define-modify-macro f64-incf (&optional (num 1d0)) two-arg-f64+)
+(define-modify-macro f64-decf (&optional (num 1d0)) two-arg-f64-)
+
+(define-modify-macro f32.4-incf (&optional (num 1f0)) two-arg-f32.4+)
+(define-modify-macro f32.4-decf (&optional (num 1f0)) two-arg-f32.4-)
+
+(define-modify-macro f32.8-incf (&optional (num 1f0)) two-arg-f32.8+)
+(define-modify-macro f32.8-decf (&optional (num 1f0)) two-arg-f32.8-)
+
+(define-modify-macro f64.2-incf (&optional (num 1d0)) two-arg-f64.2+)
+(define-modify-macro f64.2-decf (&optional (num 1d0)) two-arg-f64.2-)
+
+(define-modify-macro f64.4-incf (&optional (num 1d0)) two-arg-f64.4+)
+(define-modify-macro f64.4-decf (&optional (num 1d0)) two-arg-f64.4-)
+
+(define-modify-macro u8.16-incf (&optional (num 1)) two-arg-u8.16+)
+(define-modify-macro u8.16-decf (&optional (num 1)) two-arg-u8.16-)
+
+(define-modify-macro u16.8-incf (&optional (num 1)) two-arg-u16.8+)
+(define-modify-macro u16.8-decf (&optional (num 1)) two-arg-u16.8-)
+
+(define-modify-macro u32.4-incf (&optional (num 1)) two-arg-u32.4+)
+(define-modify-macro u32.4-decf (&optional (num 1)) two-arg-u32.4-)
+
+(define-modify-macro u64.2-incf (&optional (num 1)) two-arg-u64.2+)
+(define-modify-macro u64.2-decf (&optional (num 1)) two-arg-u64.2-)
+
+(define-modify-macro s8.16-incf (&optional (num 1)) two-arg-s8.16+)
+(define-modify-macro s8.16-decf (&optional (num 1)) two-arg-s8.16-)
+
+(define-modify-macro s16.8-incf (&optional (num 1)) two-arg-s16.8+)
+(define-modify-macro s16.8-decf (&optional (num 1)) two-arg-s16.8-)
+
+(define-modify-macro s32.4-incf (&optional (num 1)) two-arg-s32.4+)
+(define-modify-macro s32.4-decf (&optional (num 1)) two-arg-s32.4-)
+
+(define-modify-macro s64.2-incf (&optional (num 1)) two-arg-s64.2+)
+(define-modify-macro s64.2-decf (&optional (num 1)) two-arg-s64.2-)
+
+(in-package #:sb-simd-avx2)
+
+(define-modify-macro u8.32-incf (&optional (num 1)) two-arg-u8.32+)
+(define-modify-macro u8.32-decf (&optional (num 1)) two-arg-u8.32-)
+
+(define-modify-macro u16.16-incf (&optional (num 1)) two-arg-u16.16+)
+(define-modify-macro u16.16-decf (&optional (num 1)) two-arg-u16.16-)
+
+(define-modify-macro u32.8-incf (&optional (num 1)) two-arg-u32.8+)
+(define-modify-macro u32.8-decf (&optional (num 1)) two-arg-u32.8-)
+
+(define-modify-macro u64.4-incf (&optional (num 1)) two-arg-u64.4+)
+(define-modify-macro u64.4-decf (&optional (num 1)) two-arg-u64.4-)
+
+(define-modify-macro s8.32-incf (&optional (num 1)) two-arg-s8.32+)
+(define-modify-macro s8.32-decf (&optional (num 1)) two-arg-s8.32-)
+
+(define-modify-macro s16.16-incf (&optional (num 1)) two-arg-s16.16+)
+(define-modify-macro s16.16-decf (&optional (num 1)) two-arg-s16.16-)
+
+(define-modify-macro s32.8-incf (&optional (num 1)) two-arg-s32.8+)
+(define-modify-macro s32.8-decf (&optional (num 1)) two-arg-s32.8-)
+
+(define-modify-macro s64.4-incf (&optional (num 1)) two-arg-s64.4+)
+(define-modify-macro s64.4-decf (&optional (num 1)) two-arg-s64.4-)
+

contrib/sb-simd/code/define-reducers.lisp

@@ -0,0 +1,37 @@
+(in-package #:sb-simd-internals)
+
+(macrolet
+    ((define-reducer (reducer-record-name)
+       (with-accessors ((name reducer-record-name)
+                        (binary-operation reducer-record-binary-operation)
+                        (initial-element reducer-record-initial-element))
+           (find-function-record reducer-record-name)
+         (with-accessors ((binary-operation instruction-record-name)
+                          (result-records instruction-record-result-records)
+                          (argument-records instruction-record-argument-records)) binary-operation
+           (destructuring-bind ((value-record) (arg1-record arg2-record))
+               (list result-records argument-records)
+             (assert (eq value-record arg1-record))
+             (assert (eq value-record arg2-record))
+             (let ((type (value-record-name value-record)))
+               `(progn
+                  (defun ,name (arg &rest more-args)
+                    (if (null more-args)
+                        (,binary-operation (,type ,initial-element) (,type arg))
+                        (let ((result (,type arg)))
+                          (declare (,type result))
+                          (loop for arg in more-args
+                                do (setf result (,binary-operation result (,type arg))))
+                          result)))
+                  (define-compiler-macro ,name (arg &rest more-args)
+                    (cond ((null more-args)
+                           `(,',binary-operation ,',initial-element (,',type ,arg)))
+                          (t (reduce
+                              (lambda (a b) `(,',binary-operation (,',type ,a) (,',type ,b)))
+                              more-args
+                              :initial-value `(,',type ,arg)))))))))))
+     (define-reducers ()
+       `(progn
+          ,@(loop for reducer-record in (filter-function-records #'reducer-record-p)
+                  collect `(define-reducer ,(reducer-record-name reducer-record))))))
+  (define-reducers))

contrib/sb-simd/code/define-reffers.lisp

@@ -0,0 +1,44 @@
+(in-package #:sb-simd-internals)
+
+;;; In this file, we define reffers for scalars that can be defined in
+;;; terms of built-in Common Lisp functions.
+(macrolet
+    ((define-reffers ()
+       `(progn
+          ,@(loop for record in (filter-function-records
+                                 (lambda (function-record)
+                                   (eq (symbol-package
+                                        (parse-function-name
+                                         (function-record-name function-record)))
+                                       (find-package "SB-SIMD"))))
+                  for name = (function-record-name record)
+                  for value-record = (first (function-record-result-records record))
+                  for type = (value-record-name value-record)
+                  when (aref-record-p record) collect
+                    `(progn
+                       (defun ,name (array &rest subscripts)
+                         (declare (type (array ,type) array))
+                         (apply #'aref array subscripts))
+                       (define-compiler-macro ,name (array &rest subscripts)
+                         `(aref (the (array ,',type) ,array) ,@subscripts)))
+                  when (setf-aref-record-p record) collect
+                    `(progn
+                       (defun ,name (value array &rest subscripts)
+                         (declare (type ,type value)
+                                  (type (array ,type) array))
+                         (setf (apply #'aref array subscripts) value))
+                       (define-compiler-macro ,name (value array &rest subscripts)
+                         (let ((v (gensym "VALUE")))
+                           `(let ((,v ,value))
+                              (setf (aref (the (array ,',type) ,array) ,@subscripts)
+                                    ,v)))))
+                  when (row-major-aref-record-p record) collect
+                    `(define-inline ,name (array index)
+                       (declare (type (array ,type) array))
+                       (row-major-aref array index))
+                  when (setf-row-major-aref-record-p record) collect
+                    `(define-inline ,name (value array index)
+                       (declare (type ,type value)
+                                (type (array ,type) array))
+                       (setf (row-major-aref array index) value))))))
+  (define-reffers))

contrib/sb-simd/code/define-rounders.lisp

@@ -0,0 +1,75 @@
+(in-package #:sb-simd-sse4.1)
+
+(define-inline f32.4-round (x)
+  (f32.4-%round x #b00))
+
+(define-inline f32.4-floor (x)
+  (f32.4-%round x #b01))
+
+(define-inline f32.4-ceiling (x)
+  (f32.4-%round x #b10))
+
+(define-inline f32.4-truncate (x)
+  (f32.4-%round x #b11))
+
+(define-inline f64.2-round (x)
+  (f64.2-%round x #b00))
+
+(define-inline f64.2-floor (x)
+  (f64.2-%round x #b01))
+
+(define-inline f64.2-ceiling (x)
+  (f64.2-%round x #b10))
+
+(define-inline f64.2-truncate (x)
+  (f64.2-%round x #b11))
+
+(in-package #:sb-simd-avx)
+
+(define-inline f32.4-round (x)
+  (f32.4-%round x #b00))
+
+(define-inline f32.4-floor (x)
+  (f32.4-%round x #b01))
+
+(define-inline f32.4-ceiling (x)
+  (f32.4-%round x #b10))
+
+(define-inline f32.4-truncate (x)
+  (f32.4-%round x #b11))
+
+(define-inline f64.2-round (x)
+  (f64.2-%round x #b00))
+
+(define-inline f64.2-floor (x)
+  (f64.2-%round x #b01))
+
+(define-inline f64.2-ceiling (x)
+  (f64.2-%round x #b10))
+
+(define-inline f64.2-truncate (x)
+  (f64.2-%round x #b11))
+
+(define-inline f32.8-round (x)
+  (f32.8-%round x #b00))
+
+(define-inline f32.8-floor (x)
+  (f32.8-%round x #b01))
+
+(define-inline f32.8-ceiling (x)
+  (f32.8-%round x #b10))
+
+(define-inline f32.8-truncate (x)
+  (f32.8-%round x #b11))
+
+(define-inline f64.4-round (x)
+  (f64.4-%round x #b00))
+
+(define-inline f64.4-floor (x)
+  (f64.4-%round x #b01))
+
+(define-inline f64.4-ceiling (x)
+  (f64.4-%round x #b10))
+
+(define-inline f64.4-truncate (x)
+  (f64.4-%round x #b11))

contrib/sb-simd/code/define-scalar-casts.lisp

@@ -0,0 +1,78 @@
+(in-package #:sb-simd-internals)
+
+;;; For each value record we define a function of the same name that will
+;;; either suitably convert its argument to that value record's type, or
+;;; signal an error.
+
+(macrolet
+    ((define-scalar-cast (scalar-cast-record-name)
+       (with-accessors ((name scalar-cast-record-name))
+           (find-function-record scalar-cast-record-name)
+         (let ((err (mksym (symbol-package name) "CANNOT-CONVERT-TO-" name)))
+           `(progn
+              (define-notinline ,err (x)
+                (error "Cannot convert ~S to ~S." x ',name))
+              (sb-c:defknown ,name (t) (values ,name &optional)
+                  (sb-c:foldable)
+                :overwrite-fndb-silently t)
+              (sb-c:deftransform ,name ((x) (,name) *)
+                'x)
+              ,@(case name
+                  (sb-simd:f32
+                   `((sb-c:deftransform ,name ((x) (double-float) *)
+                       '(coerce x 'single-float))))
+                  (sb-simd:f64
+                   `((sb-c:deftransform ,name ((x) (single-float) *)
+                       '(coerce x 'double-float))))
+                  (sb-simd-sse:f32
+                   `((sb-c:deftransform ,name ((x) (double-float) *)
+                       '(sb-kernel:%single-float x))
+                     (sb-c:deftransform ,name ((x) ((signed-byte 64)) *)
+                       '(sb-simd-sse::f32-from-s64 x))))
+                  (sb-simd-sse2:f64
+                   `((sb-c:deftransform ,name ((x) (single-float) *)
+                       '(sb-simd-sse2::f64-from-f32 x))
+                     (sb-c:deftransform ,name ((x) ((signed-byte 64)) *)
+                       '(sb-simd-sse2::f64-from-s64 x))))
+                  (sb-simd-avx:f32
+                   `((sb-c:deftransform ,name ((x) (double-float) *)
+                       '(sb-simd-avx::f32-from-f64 x))
+                     (sb-c:deftransform ,name ((x) ((signed-byte 64)) *)
+                       '(sb-simd-avx::f32-from-s64 x))))
+                  (sb-simd-avx:f64
+                   `((sb-c:deftransform ,name ((x) (single-float) *)
+                       '(sb-simd-avx::f64-from-f32 x))
+                     (sb-c:deftransform ,name ((x) ((signed-byte 64)) *)
+                       '(sb-simd-avx::f64-from-s64 x)))))
+              (defun ,name (x)
+                (typecase x
+                  (,name x)
+                  ,@(case name
+                      (sb-simd:f32
+                       `((double-float (coerce x 'single-float))
+                         (real (coerce x ',name))))
+                      (sb-simd:f64
+                       `((sb-simd-sse2:f32 (coerce x 'double-float))
+                         (real (coerce x ',name))))
+                      (sb-simd-sse:f32
+                       `((double-float (sb-kernel:%single-float x))
+                         (sb-simd-sse:s64 (sb-simd-sse::%f32-from-s64 x))
+                         (real (coerce x ',name))))
+                      (sb-simd-sse2:f64
+                       `((sb-simd-sse2:f32 (sb-simd-sse2::%f64-from-f32 x))
+                         (sb-simd-sse2:s64 (sb-simd-sse2::%f64-from-s64 x))
+                         (real (coerce x ',name))))
+                      (sb-simd-avx:f32
+                       `((sb-simd-avx:f64 (sb-simd-avx::%f32-from-f64 x))
+                         (sb-simd-avx:s64 (sb-simd-avx::%f32-from-s64 x))
+                         (real (coerce x ',name))))
+                      (sb-simd-avx:f64
+                       `((sb-simd-avx:f32 (sb-simd-avx::%f64-from-f32 x))
+                         (sb-simd-avx:s64 (sb-simd-avx::%f64-from-s64 x))
+                         (real (coerce x ',name)))))
+                  (otherwise (,err x))))))))
+     (define-scalar-casts ()
+       `(progn
+          ,@(loop for scalar-cast-record in (filter-function-records #'scalar-cast-record-p)
+                  collect `(define-scalar-cast ,(function-record-name scalar-cast-record))))))
+  (define-scalar-casts))

contrib/sb-simd/code/define-simd-casts.lisp

@@ -0,0 +1,92 @@
+(in-package #:sb-simd-internals)
+
+;;; For each SIMD data type X.Y, define two functions:
+;;;
+;;; 1. A function named X.Y that ensures that an object is either of the
+;;;    type X.Y, or a scalar that can be broadcast to the type X.Y.
+;;;
+;;; 2. A function named X.Y! that reinterprets the bits of another SIMD
+;;;    pack or suitable scalar as an X.Y.  If the supplied argument has
+;;;    more bits than the target data type, the excess bits are discarded.
+;;;    If the supplied argument has less bits than the target data types,
+;;;    the remaining bits are set to zero.
+
+;;; The pXXX SIMD types are special - we define their 'cast' function
+;;; manually.
+(define-inline sb-simd-sse:p128 (x) (the sb-simd-sse:p128 x))
+(define-inline sb-simd-avx:p128 (x) (the sb-simd-avx:p128 x))
+(define-inline sb-simd-avx:p256 (x) (the sb-simd-avx:p256 x))
+
+(macrolet
+    (;; We cannot call known functions directly in the definition of a
+     ;; cast, only their VOPs.  The reason is that each known function
+     ;; definition uses casts to automatically upgrade its arguments, and
+     ;; we don't want to end up with a circular dependency.
+     (call-vop (instruction-record-name &rest arguments)
+       (with-accessors ((instruction-set instruction-record-instruction-set)
+                        (vop instruction-record-vop))
+           (find-function-record instruction-record-name)
+         (if (instruction-set-available-p instruction-set)
+             `(,vop ,@arguments)
+             `(progn
+                (missing-instruction
+                 (load-time-value
+                  (find-function-record ',instruction-record-name)))
+                (touch ,@arguments)))))
+     (define-simd-cast (simd-cast-record-name)
+       (with-accessors ((name simd-cast-record-name)
+                        (instruction-set simd-cast-record-instruction-set)
+                        (broadcast-record simd-cast-record-broadcast))
+           (find-function-record simd-cast-record-name)
+         (let* ((broadcast (function-record-name broadcast-record))
+                (simd-record (function-record-result-record broadcast-record))
+                (real-record (simd-record-scalar-record simd-record))
+                (simd-type (value-record-name simd-record))
+                (real-type (value-record-name real-record))
+                (package (instruction-set-package instruction-set))
+                (err (mksym package "CANNOT-CONVERT-TO-" name)))
+           `(progn
+              (define-notinline ,err (x)
+                (error "Cannot convert ~S to ~S." x ',name))
+              (sb-c:defknown ,name (t) (values ,name &optional)
+                  (sb-c:foldable)
+                :overwrite-fndb-silently t)
+              (sb-c:deftransform ,name ((x) (,simd-type) *)
+                'x)
+              (sb-c:deftransform ,name ((x) (real) *)
+                '(,broadcast (,real-type x)))
+              (defun ,name (x)
+                (typecase x
+                  (,simd-type x)
+                  (real (call-vop ,broadcast (,real-type x)))
+                  (otherwise (,err x))))))))
+     (define-reinterpret-cast (reinterpret-cast-record)
+       (with-accessors ((name reinterpret-cast-record-name)
+                        (instruction-set reinterpret-cast-record-instruction-set)
+                        (reinterpreters reinterpret-cast-record-reinterpreters))
+           (find-function-record reinterpret-cast-record)
+         (let* ((package (instruction-set-package instruction-set))
+                (err (mksym package "CANNOT-REINTERPRET-AS-" name)))
+           `(progn
+              (define-notinline ,err (x)
+                (error "Cannot reinterpret ~S as ~S." x ',name))
+              (defun ,name (x)
+                (typecase x
+                  ,@(loop for reinterpreter in reinterpreters
+                          for argument-record = (first (function-record-required-argument-records reinterpreter))
+                          collect
+                          `(,(value-record-name argument-record)
+                            (call-vop ,(function-record-name reinterpreter) x)))
+                  (otherwise (,err x))))))))
+     (define-simd-casts ()
+       `(progn
+          ,@(loop for simd-cast-record in (filter-function-records #'simd-cast-record-p)
+                  collect
+                  `(define-simd-cast ,(simd-cast-record-name simd-cast-record)))))
+     (define-reinterpret-casts ()
+       `(progn
+          ,@(loop for reinterpret-cast-record in (filter-function-records #'reinterpret-cast-record-p)
+                  collect
+                  `(define-reinterpret-cast ,(reinterpret-cast-record-name reinterpret-cast-record))))))
+  (define-simd-casts)
+  (define-reinterpret-casts))

contrib/sb-simd/code/define-types.lisp

@@ -0,0 +1,10 @@
+(in-package #:sb-simd-internals)
+
+(macrolet
+    ((define-types ()
+       `(progn
+          ,@(loop for value-record being the hash-values of *value-records*
+                  collect
+                  `(deftype ,(value-record-name value-record) ()
+                     ',(value-record-type value-record))))))
+  (define-types))

contrib/sb-simd/code/define-unequals.lisp

@@ -0,0 +1,46 @@
+(in-package #:sb-simd-internals)
+
+(macrolet
+    ((define-unequal (unequal-record-name)
+       (with-accessors ((name unequal-record-name)
+                        (neq unequal-record-neq)
+                        (and unequal-record-and)
+                        (truth unequal-record-truth))
+           (find-function-record unequal-record-name)
+         (with-accessors ((neq instruction-record-name)
+                          (result-records instruction-record-result-records)
+                          (argument-records instruction-record-argument-records)) neq
+           (destructuring-bind ((result-record) (argument-record other-argument-record))
+               (list result-records argument-records)
+             (assert (eq argument-record other-argument-record))
+             (let ((and (function-record-name and))
+                   (result-type (value-record-name result-record))
+                   (argument-type (value-record-name argument-record)))
+               `(progn
+                  (defun ,name (arg &rest more-args)
+                    (let ((args (list* (,argument-type arg) (mapcar #',argument-type more-args)))
+                          (result (,result-type ,truth)))
+                      (declare (,result-type result))
+                      (loop for (a . rest) on args do
+                        (loop for b in rest do
+                          (setf result (,and result (,neq a b)))))
+                      result))
+                  (define-compiler-macro ,name (arg &rest more-args)
+                    (if (null more-args)
+                        `(progn (,',argument-type ,arg) (,',result-type ,',truth))
+                        (let ((bindings
+                                (loop for arg in (list* arg more-args)
+                                      collect
+                                      (list (gensym "ARG") (list ',argument-type arg)))))
+                          `(let ,bindings
+                             (,',and
+                              ,@(loop for ((a nil) . rest) on bindings
+                                      append
+                                      (loop for (b nil) in rest
+                                            collect `(,',neq ,a ,b))))))))))))))
+     (define-unequals ()
+       `(progn
+          ,@(loop for unequal-record in (filter-function-records #'unequal-record-p)
+                  collect
+                  `(define-unequal ,(unequal-record-name unequal-record))))))
+  (define-unequals))

contrib/sb-simd/code/define-vop-functions.lisp

@@ -0,0 +1,68 @@
+(in-package #:sb-simd-internals)
+
+;;; For constant folding, SBCL needs functions of the same name as the VOP.
+;;; In this file, we define these functions.  Because some VOPs cannot
+;;; translate the full range of arguments supported by such a function,
+;;; e.g., because one argument is expected to be a constant, we also need
+;;; some macrology to have each function dispatch only to calls that can be
+;;; translated.
+
+(defmacro with-primitive-arguments (alist &body body)
+  ;; Each entry in ALIST is of the form (VARIABLE VALUE-RECORD-NAME).
+  (if (null alist)
+      `(progn ,@body)
+      `(with-primitive-argument ,(first alist)
+         (with-primitive-arguments ,(rest alist)
+           ,@body))))
+
+(defmacro with-primitive-argument ((symbol value-record) &body body)
+  (with-accessors ((primitive-type value-record-primitive-type)
+                   (simd-p simd-record-p))
+      (find-value-record value-record)
+    (etypecase primitive-type
+      ;; Case 1: A symbol denoting a primitive type.
+      (symbol
+       (let ((alias (find primitive-type sb-c::*backend-primitive-type-aliases* :key #'car)))
+         ;; ALIAS is either null or a list of the form (:OR PRIMITIVE-TYPE*)
+         (if (or (not simd-p) (not alias))
+             `(progn ,@body)
+             `(etypecase ,symbol
+                ,@(loop for pt in (rest (rest alias))
+                        for type = (sb-c::primitive-type-specifier (sb-c:primitive-type-or-lose pt))
+                        collect `(,type ,@body))))))
+      ;; Case 2: A list of the form (:CONSTANT TYPE), where TYPE is either
+      ;; of the form (SIGNED-BYTE N) or (UNSIGNED-BYTE N) for some positive
+      ;; integer N.
+      ((cons (eql :constant) (cons type-specifier null))
+       (multiple-value-bind (low high)
+           (integer-type-specifier-inclusive-bounds (second primitive-type))
+         `(ecase ,symbol
+            ,@(loop for value from low to high
+                    collect `(,value (symbol-macrolet ((,symbol ,value)) ,@body)))))))))
+
+(macrolet
+    ((define-vop-function (name)
+       (with-accessors ((name instruction-record-name)
+                        (vop instruction-record-vop)
+                        (argument-records instruction-record-argument-records)
+                        (encoding instruction-record-encoding)
+                        (instruction-set instruction-record-instruction-set))
+           (find-function-record name)
+         (let* ((argument-record-names (mapcar #'record-name argument-records))
+                (argument-symbols (prefixed-symbols "ARGUMENT-" (length argument-records))))
+           (unless (or (eq encoding :fake-vop)
+                       (not (instruction-set-available-p instruction-set)))
+             `(defun ,vop (,@argument-symbols)
+                (declare
+                 ,@(loop for argument-symbol in argument-symbols
+                         for argument-record in argument-records
+                         collect `(type ,(value-record-name argument-record) ,argument-symbol)))
+                (with-primitive-arguments ,(mapcar #'list argument-symbols argument-record-names)
+                  (,vop ,@argument-symbols)))))))
+
+     (define-vop-functions ()
+       `(progn
+          ,@(loop for instruction-record in (filter-function-records #'instruction-record-p)
+                  for name = (instruction-record-name instruction-record)
+                  collect `(define-vop-function ,name)))))
+  (define-vop-functions))

contrib/sb-simd/code/define-vref-vops.lisp

@@ -0,0 +1,97 @@
+(in-package #:sb-vm)
+
+;;; Both load- and store VOPs are augmented with an auxiliary last argument
+;;; that is a constant addend for the address calculation.  This addend is
+;;; zero by default, but we can sometimes transform the code for the index
+;;; calculation such that we have a nonzero addend.  We also generate two
+;;; variants of the VOP - one for the general case, and one for the case
+;;; where the index is a compile-time constant.
+
+(macrolet
+    ((define-vref-vop (vref-record-name)
+       (with-accessors ((name sb-simd-internals:vref-record-name)
+                        (vop sb-simd-internals:vref-record-vop)
+                        (vop-c sb-simd-internals:vref-record-vop-c)
+                        (mnemonic sb-simd-internals:vref-record-mnemonic)
+                        (value-record sb-simd-internals:vref-record-value-record)
+                        (vector-record sb-simd-internals:vref-record-vector-record)
+                        (store sb-simd-internals:store-record-p))
+           (sb-simd-internals:find-function-record vref-record-name)
+         (let* ((vector-type (sb-simd-internals:value-record-type vector-record))
+                (vector-primitive-type (sb-simd-internals:value-record-primitive-type vector-record))
+                (value-scs (sb-simd-internals:value-record-scs value-record))
+                (value-type (sb-simd-internals:value-record-type value-record))
+                (value-primitive-type (sb-simd-internals:value-record-primitive-type value-record))
+                (scalar-record
+                  (etypecase value-record
+                    (sb-simd-internals:simd-record (sb-simd-internals:simd-record-scalar-record value-record))
+                    (sb-simd-internals:value-record value-record)))
+                (bits-per-element (sb-simd-internals:value-record-bits scalar-record))
+                (bytes-per-element (ceiling bits-per-element 8))
+                (displacement
+                  (multiple-value-bind (lo hi)
+                      (displacement-bounds other-pointer-lowtag bits-per-element vector-data-offset)
+                    `(integer ,lo ,hi))))
+           (multiple-value-bind (index-scs scale)
+               (if (>= bytes-per-element (ash 1 n-fixnum-tag-bits))
+                   (values '(any-reg signed-reg unsigned-reg) `(index-scale ,bytes-per-element index))
+                   (values '(signed-reg unsigned-reg) bytes-per-element))
+             `(progn
+                (defknown ,vop (,@(when store `(,value-type)) ,vector-type index ,displacement)
+                    (values ,value-type &optional)
+                    (always-translatable)
+                  :overwrite-fndb-silently t)
+                (define-vop (,vop)
+                  (:translate ,vop)
+                  (:policy :fast-safe)
+                  (:args
+                   ,@(when store `((value :scs ,value-scs :target result)))
+                   (vector :scs (descriptor-reg))
+                   (index :scs ,index-scs))
+                  (:info addend)
+                  (:arg-types
+                   ,@(when store `(,value-primitive-type))
+                   ,vector-primitive-type
+                   positive-fixnum
+                   (:constant ,displacement))
+                  (:results (result :scs ,value-scs))
+                  (:result-types ,value-primitive-type)
+                  (:generator
+                   2
+                   ,@(let ((ea `(ea (+ (* vector-data-offset n-word-bytes)
+                                       (* addend ,bytes-per-element)
+                                       (- other-pointer-lowtag))
+                                    vector index ,scale)))
+                       (if store
+                           `((inst ,mnemonic ,ea value)
+                             (move result value))
+                           `((inst ,mnemonic result ,ea))))))
+                (define-vop (,vop-c)
+                  (:translate ,vop)
+                  (:policy :fast-safe)
+                  (:args ,@(when store `((value :scs ,value-scs :target result)))
+                         (vector :scs (descriptor-reg)))
+                  (:info index addend)
+                  (:arg-types ,@(when store `(,value-primitive-type))
+                              ,vector-primitive-type
+                              (:constant low-index)
+                              (:constant ,displacement))
+                  (:results (result :scs ,value-scs))
+                  (:result-types ,value-primitive-type)
+                  (:generator
+                   1
+                   ,@(let ((ea `(ea (+ (* vector-data-offset n-word-bytes)
+                                       (* ,bytes-per-element (+ index addend))
+                                       (- other-pointer-lowtag))
+                                    vector)))
+                       (if store
+                           `((inst ,mnemonic ,ea value)
+                             (move result value))
+                           `((inst ,mnemonic result ,ea)))))))))))
+     (define-vref-vops ()
+       `(progn
+          ,@(loop for vref-record
+                    in (sb-simd-internals:filter-available-function-records
+                        #'sb-simd-internals:vref-record-p)
+                  collect `(define-vref-vop ,(sb-simd-internals:vref-record-name vref-record))))))
+  (define-vref-vops))

contrib/sb-simd/code/define-vrefs.lisp

@@ -0,0 +1,44 @@
+(in-package #:sb-simd-internals)
+
+(macrolet
+    ((define-vref (name kind)
+       (with-accessors ((name vref-record-name)
+                        (instruction-set vref-record-instruction-set)
+                        (value-record vref-record-value-record)
+                        (vector-record vref-record-vector-record)
+                        (vop vref-record-vop))
+           (find-function-record name)
+         (let* ((simd-width (value-record-simd-width value-record))
+                (element-type
+                  (second
+                   (value-record-type vector-record))))
+           (ecase kind
+             (:load
+              `(define-inline ,name (array index)
+                 (declare (type (array ,element-type) array)
+                          (index index))
+                 (multiple-value-bind (vector index)
+                     (sb-kernel:%data-vector-and-index
+                      array
+                      (sb-kernel:check-bound array (- (array-total-size array) ,(1- simd-width)) index))
+                   (declare (type (simple-array ,element-type (*)) vector))
+                   (,vop vector index 0))))
+             (:store
+              `(define-inline ,name (value array index)
+                 (declare (type (array ,element-type) array)
+                          (index index))
+                 (multiple-value-bind (vector index)
+                     (sb-kernel:%data-vector-and-index
+                      array
+                      (sb-kernel:check-bound array (- (array-total-size array) ,(1- simd-width)) index))
+                   (declare (type (simple-array ,element-type (*)) vector))
+                   (,vop (,(value-record-name value-record) value) vector index 0))))))))
+     (define-vrefs ()
+       `(progn
+          ,@(loop for load-record in (filter-function-records #'load-record-p)
+                  for name = (load-record-name load-record)
+                  collect `(define-vref ,name :load))
+          ,@(loop for store-record in (filter-function-records #'store-record-p)
+                  for name = (store-record-name store-record)
+                  collect `(define-vref ,name :store)))))
+  (define-vrefs))

contrib/sb-simd/code/instruction-set-case.lisp

@@ -0,0 +1,106 @@
+(in-package #:sb-simd-internals)
+
+;;; Lisp supports an image-based workflow, so it is entirely possible that
+;;; a program is transferred from one machine to another one that supports
+;;; different instructions.  For example, a video game developer might
+;;; create a game on a machine supporting x86 instructions up to AVX2,
+;;; create an image-based executable, and ship it to a customer whose
+;;; machine supports only x86 instructions up to SSE4.1.
+;;;
+;;; This file defines a macro for conditionally executing code based on the
+;;; available instructions.  Special care was taken to make this
+;;; conditional selection as efficient as possible.  On x86, it should
+;;; compile to a jump table whose index is computed only at load time and
+;;; whenever the image is restarted.
+
+;;; For each instruction-set-case macro appearing in source code, we
+;;; register an IDISPATCH structure that tracks the available clauses, and
+;;; the index of the currently active clause.  Whenever the Lisp image is
+;;; restarted, we traverse all IDISPATCH structures and recompute the index
+;;; of the currently active clause.
+(defparameter *idispatch-table*
+  (make-hash-table :weakness :key :synchronized t))
+
+(defstruct (idispatch
+            (:constructor %make-idispatch)
+            (:predicate idispatchp))
+  ;; The clauses supplied to the instruction-set-case macro.
+  (clauses nil :type list :read-only t)
+  ;; A list of lists of instruction sets - one for each clause.
+  (isets nil :type list :read-only t)
+  ;; The index of the currently active clause.
+  (index 0 :type (unsigned-byte 8)))
+
+(defun make-idispatch (isets clauses)
+  (let ((idispatch (%make-idispatch
+                    :clauses clauses
+                    :isets isets)))
+    (update-idispatch-index idispatch)
+    (setf (gethash idispatch *idispatch-table*) t)
+    idispatch))
+
+(defun update-idispatch-index (idispatch)
+  (loop for index from 0
+        for instruction-sets in (idispatch-isets idispatch)
+        when (every #'instruction-set-available-p instruction-sets) do
+          (setf (idispatch-index idispatch) index)
+          (return-from update-idispatch-index)
+        finally
+           (warn "None of the idispatch clauses in ~@
+                  ~{~S~%~}is available on this machine."
+                 (idispatch-clauses idispatch))
+           (setf (idispatch-index idispatch)
+                 (length (idispatch-isets idispatch)))))
+
+(defun update-idispatch-indices ()
+  (loop for idispatch being the hash-keys of *idispatch-table* do
+    (update-idispatch-index idispatch)))
+
+(pushnew 'update-idispatch-indices sb-ext:*init-hooks*)
+
+(defmacro instruction-set-case (&body clauses)
+  "Execute the first clause whose instruction sets are available at run
+time, or signal an error if no clause could be run.
+
+Each clause has to start with an instruction set name, or a list of
+instruction set names, followed by a list of statements.
+
+Example:
+
+ (instruction-set-case
+   (:sse2 (foo))
+   (:avx (bar))
+   ((:avx2 :fma) (baz)))
+"
+  (sb-int:with-unique-names (idispatch)
+    (multiple-value-bind (isets bodies)
+        (parse-instruction-set-case-clauses clauses)
+      `(let ((,idispatch (load-time-value (make-idispatch ',isets ',clauses))))
+         (declare (idispatch ,idispatch))
+         (case (idispatch-index ,idispatch)
+           ,@(loop for iset in isets
+                   for body in bodies
+                   for index from 0
+                   collect `(,index ,@body))
+           (,(length clauses)
+            (idispatch-no-applicable-clause ,idispatch)))))))
+
+(defun parse-instruction-set-case-clauses (clauses)
+  (flet ((clause-iset (clause)
+           (unless (consp clause)
+             (error "Not a valid instruction-set-case clause: ~S"
+                    clause))
+           (let ((head (first clause)))
+             (if (listp head)
+                 (mapcar #'find-instruction-set head)
+                 (list (find-instruction-set head)))))
+         (clause-body (clause)
+           (rest clause)))
+    (values
+     (mapcar #'clause-iset clauses)
+     (mapcar #'clause-body clauses))))
+
+(defun idispatch-no-applicable-clause (idispatch)
+  (error "None of the idispatch clauses in ~@
+          ~{~S~%~}is available on this machine."
+         (idispatch-clauses idispatch)))

contrib/sb-simd/code/instruction-set.lisp

@@ -0,0 +1,152 @@
+(in-package #:sb-simd-internals)
+
+;;; An instruction set is a description for a set of data types and
+;;; functions in a particular package.
+
+(defclass instruction-set (printable)
+  (;; The instruction set's name.
+   (%name
+    :type keyword
+    :initarg :name
+    :initform (required-argument :name)
+    :reader instruction-set-name)
+   ;; The package that holds the instruction set's symbols.
+   (%package
+    :type package
+    :initarg :package
+    :initform (required-argument :package)
+    :reader instruction-set-package)
+   ;; A thunk, returning whether the instruction set is currently available.
+   ;; Such a run time check is needed in the case where an executable is
+   ;; created on one machine and run on another machine.  In that case, some
+   ;; of the instructions sets available on the former might not be available
+   ;; on the latter.
+   (%test
+    :type function
+    :initarg :test :initform (required-argument :test)
+    :reader instruction-set-test)
+   ;; A list of instruction sets included by this one.
+   (%includes
+    :type list
+    :initarg :includes
+    :initform '()
+    :reader instruction-set-includes)
+   ;; A hash table, mapping from function records of scalar functions to
+   ;; lists of function records of SIMD functions.
+   (%vectorizer-table
+    :type hash-table
+    :initform (make-hash-table :test #'eq)
+    :reader instruction-set-vectorizer-table)))
+
+(defmethod printable-slot-plist append ((instruction-set instruction-set))
+  (list :name (instruction-set-name instruction-set)
+        :package (instruction-set-package instruction-set)))
+
+(defun instruction-set-p (x)
+  (typep x 'instruction-set))
+
+(defun instruction-set-available-p (instruction-set)
+  (funcall (instruction-set-test instruction-set)))
+
+;;; Returns a list containing the name of the supplied instruction set, and
+;;; the names of all instruction sets that are directly or indirectly
+;;; included by it.
+(defun included-instruction-sets (instruction-set)
+  (let ((result '()))
+    (labels ((scan (instruction-set)
+               (with-accessors ((includes instruction-set-includes)) instruction-set
+                 (unless (member instruction-set result)
+                   (push instruction-set result)
+                   (mapcar #'scan includes)))))
+      (scan instruction-set)
+      (nreverse result))))
+
+(defun register-vectorizer (X-record X.Y-record)
+  (assert (scalar-function-record-p X-record))
+  (assert (simd-function-record-p X.Y-record))
+  (with-accessors ((vectorizer-table instruction-set-vectorizer-table))
+      (function-record-instruction-set X.Y-record)
+    (pushnew X.Y-record (gethash X-record vectorizer-table '()))))
+
+(defun instruction-set-vectorizers (instruction-set X-record)
+  (loop for instruction-set in (included-instruction-sets instruction-set)
+        append
+        (gethash X-record (instruction-set-vectorizer-table instruction-set) '())))
+
+;;; A hash table, mapping from instruction set names or packages to
+;;; instruction sets.
+(defparameter *instruction-sets* (make-hash-table :test #'eq))
+
+(defun find-instruction-set (designator &optional (errorp t))
+  (or (gethash designator *instruction-sets*)
+      (when errorp
+        (typecase designator
+          (symbol (error "There is no instruction set with the name ~S." designator))
+          (package (error "There is not instruction set with the package ~S" designator))
+          (otherwise (error "Not a valid instruction set designator: ~S" designator))))))
+
+(defmethod make-load-form ((instruction-set instruction-set) &optional env)
+  (declare (ignore env))
+  `(find-instruction-set ',(instruction-set-name instruction-set)))
+
+(defmethod shared-initialize :after
+    ((instruction-set instruction-set) slot-names &key &allow-other-keys)
+  (setf (gethash (instruction-set-name instruction-set) *instruction-sets*)
+        instruction-set)
+  (setf (gethash (instruction-set-package instruction-set) *instruction-sets*)
+        instruction-set)
+  instruction-set)
+
+;;; The currently active instruction set.
+(defvar *instruction-set*)
+
+;;; Defining Instruction Sets
+
+(defparameter *instruction-set-options*
+  '(:include :test :scalars :simd-packs :simd-casts :reinterpret-casts
+    :instructions :loads :stores :reffers
+    :associatives :reducers :comparisons :unequals :ifs))
+
+(defgeneric decode-record-definition (record-name expr))
+
+(defmacro define-instruction-set (name &body options)
+  ;; Ensure that only valid options are supplied.
+  (dolist (option options)
+    (unless (and (listp option)
+                 (member (first option) *instruction-set-options*))
+      (error "Not a valid instruction set option:~% ~S" option)))
+  (flet ((decode (keyword decoder)
+           (loop for (key . exprs) in options
+                 when (eq key keyword)
+                   append (mapcar decoder exprs)))
+         (decode-include (expr)
+           `(find-instruction-set ',expr))
+         (record-decoder (record-name)
+           (lambda (x)
+             (decode-record-definition record-name x))))
+    ;; The macro expansion of an instruction set is a very large expression
+    ;; that is evaluated exactly once, so compiling it would be a waste of
+    ;; resources.  Instead, we use SBCL's built-in interpreter.
+    `(let ((sb-ext:*evaluator-mode* :interpret))
+       (eval
+        '(let ((*instruction-set*
+                (make-instance 'instruction-set
+                 :name ',name
+                 :package ,(if (eq name :sb-simd)
+                               (find-package "SB-SIMD")
+                               (find-package (concatenate 'string "SB-SIMD-" (string name))))
+                 :test (lambda () (and ,@(decode :test #'identity)))
+                 :includes (list ,@(decode :include #'decode-include)))))
+          ,@(decode :scalars (record-decoder 'value-record))
+          ,@(decode :simd-packs (record-decoder 'simd-record))
+          ,@(decode :instructions (record-decoder 'instruction-record))
+          ,@(decode :loads (record-decoder 'load-record))
+          ,@(decode :stores (record-decoder 'store-record))
+          ,@(decode :reffers (record-decoder 'reffer-record))
+          ,@(decode :associatives (record-decoder 'associative-record))
+          ,@(decode :reducers (record-decoder 'reducer-record))
+          ,@(decode :comparisons (record-decoder 'comparison-record))
+          ,@(decode :unequals (record-decoder 'unequal-record))
+          ,@(decode :ifs (record-decoder 'if-record))
+          ,@(decode :simd-casts (record-decoder 'simd-cast-record))
+          ,@(decode :reinterpret-casts (record-decoder 'reinterpret-cast-record)))))))

contrib/sb-simd/code/instruction-sets/avx.lisp

@@ -0,0 +1,911 @@
+(in-package #:sb-simd-avx)
+
+(define-instruction-set :avx
+  (:test (avx-supported-p))
+  (:include :x86-64)
+  (:scalars
+   (f32 32 single-float #:single-float (#:single-reg))
+   (f64 64 double-float #:double-float (#:double-reg)))
+  (:simd-packs
+   (p128   nil 128 #:simd-pack (#:int-sse-reg #:double-sse-reg #:single-sse-reg))
+   (f32.4  f32 128 #:simd-pack-single (#:single-sse-reg))
+   (f64.2  f64 128 #:simd-pack-double (#:double-sse-reg))
+   (u8.16  u8  128 #:simd-pack-ub8  (#:int-sse-reg))
+   (u16.8  u16 128 #:simd-pack-ub16 (#:int-sse-reg))
+   (u32.4  u32 128 #:simd-pack-ub32 (#:int-sse-reg))
+   (u64.2  u64 128 #:simd-pack-ub64 (#:int-sse-reg))
+   (s8.16  s8  128 #:simd-pack-sb8  (#:int-sse-reg))
+   (s16.8  s16 128 #:simd-pack-sb16 (#:int-sse-reg))
+   (s32.4  s32 128 #:simd-pack-sb32 (#:int-sse-reg))
+   (s64.2  s64 128 #:simd-pack-sb64 (#:int-sse-reg))
+   (p256   nil 256 #:simd-pack-256 (#:int-avx2-reg #:double-avx2-reg #:single-avx2-reg))
+   (f32.8  f32 256 #:simd-pack-256-single (#:single-avx2-reg))
+   (f64.4  f64 256 #:simd-pack-256-double (#:double-avx2-reg))
+   (u8.32  u8  256 #:simd-pack-256-ub8  (#:int-avx2-reg))
+   (u16.16 u16 256 #:simd-pack-256-ub16 (#:int-avx2-reg))
+   (u32.8  u32 256 #:simd-pack-256-ub32 (#:int-avx2-reg))
+   (u64.4  u64 256 #:simd-pack-256-ub64 (#:int-avx2-reg))
+   (s8.32  s8  256 #:simd-pack-256-sb8  (#:int-avx2-reg))
+   (s16.16 s16 256 #:simd-pack-256-sb16 (#:int-avx2-reg))
+   (s32.8  s32 256 #:simd-pack-256-sb32 (#:int-avx2-reg))
+   (s64.4  s64 256 #:simd-pack-256-sb64 (#:int-avx2-reg)))
+  (:simd-casts
+   (f32.4 f32.4-broadcast)
+   (f64.2 f64.2-broadcast)
+   (f32.8 f32.8-broadcast)
+   (f64.4 f64.4-broadcast)
+   (u8.16 u8.16-broadcast)
+   (u16.8 u16.8-broadcast)
+   (u32.4 u32.4-broadcast)
+   (u64.2 u64.2-broadcast)
+   (u8.32 u8.32-broadcast)
+   (u16.16 u16.16-broadcast)
+   (u32.8 u32.8-broadcast)
+   (u64.4 u64.4-broadcast)
+   (s8.16 s8.16-broadcast)
+   (s16.8 s16.8-broadcast)
+   (s32.4 s32.4-broadcast)
+   (s64.2 s64.2-broadcast)
+   (s8.32 s8.32-broadcast)
+   (s16.16 s16.16-broadcast)
+   (s32.8 s32.8-broadcast)
+   (s64.4 s64.4-broadcast))
+  (:reinterpret-casts
+   (f32! f32!-from-p128 f32!-from-p256)
+   (f64! f64!-from-p128 f64!-from-p256)
+   (u8!  u8!-from-p128  u8!-from-p256)
+   (u16! u16!-from-p128 u16!-from-p256)
+   (u32! u32!-from-p128 u32!-from-p256)
+   (u64! u64!-from-p128 u64!-from-p256)
+   (f32.4! f32.4!-from-f32 f32.4!-from-p256)
+   (f64.2! f64.2!-from-f64 f64.2!-from-p128)
+   (f32.8! f32.8!-from-f32 f32.8!-from-p128 f32.8!-from-p256)
+   (f64.4! f64.4!-from-f64 f64.4!-from-p128 f64.4!-from-p256)
+   (u8.16! u8.16!-from-u8 u8.16!-from-p128 u8.16!-from-p256)
+   (u16.8! u16.8!-from-u16 u16.8!-from-p128 u16.8!-from-p256)
+   (u32.4! u32.4!-from-u32 u32.4!-from-p128 u32.4!-from-p256)
+   (u64.2! u64.2!-from-u64 u64.2!-from-p128 u64.2!-from-p256)
+   (u8.32! u8.32!-from-u8 u8.32!-from-p128 u8.32!-from-p256)
+   (u16.16! u16.16!-from-u16 u16.16!-from-p128 u16.16!-from-p256)
+   (u32.8! u32.8!-from-u32 u32.8!-from-p128 u32.8!-from-p256)
+   (u64.4! u64.4!-from-u64 u64.4!-from-p128 u64.4!-from-p256)
+   (s8.16! s8.16!-from-s8 s8.16!-from-p128 s8.16!-from-p256)
+   (s16.8! s16.8!-from-s16 s16.8!-from-p128 s16.8!-from-p256)
+   (s32.4! s32.4!-from-s32 s32.4!-from-p128 s32.4!-from-p256)
+   (s64.2! s64.2!-from-s64 s64.2!-from-p128 s64.2!-from-p256)
+   (s8.32! s8.32!-from-s8 s8.32!-from-p128 s8.32!-from-p256)
+   (s16.16! s16.16!-from-s16 s16.16!-from-p128 s16.16!-from-p256)
+   (s32.8! s32.8!-from-s32 s32.8!-from-p128 s32.8!-from-p256)
+   (s64.4! s64.4!-from-s64 s64.4!-from-p128 s64.4!-from-p256))
+  (:instructions
+   (vzeroupper          #:vzeroupper   ()      ()   :cost 1 :pure nil)
+   (vzeroall            #:vzeroall     ()      ()   :cost 1 :pure nil)
+   ;; f32
+   (f32-from-f64        #:vcvtsd2ss    (f32) (f64)     :cost 5)
+   (f32-from-s64        nil            (f32) (s64)     :cost 5 :encoding :custom)
+   (f32!-from-p128      nil            (f32) (p128)    :cost 1 :encoding :custom :always-translatable nil)
+   (f32!-from-p256      nil            (f32) (p256)    :cost 1 :encoding :custom :always-translatable nil)
+   (two-arg-f32-and     #:vandps       (f32) (f32 f32) :cost 1 :associative t)
+   (two-arg-f32-or      #:vorps        (f32) (f32 f32) :cost 1 :associative t)
+   (two-arg-f32-xor     #:vxorps       (f32) (f32 f32) :cost 1 :associative t)
+   (two-arg-f32-max     #:vmaxss       (f32) (f32 f32) :cost 1 :associative t)
+   (two-arg-f32-min     #:vminss       (f32) (f32 f32) :cost 1 :associative t)
+   (two-arg-f32+        #:vaddss       (f32) (f32 f32) :cost 1 :associative t)
+   (two-arg-f32-        #:vsubss       (f32) (f32 f32) :cost 2)
+   (two-arg-f32*        #:vmulss       (f32) (f32 f32) :cost 2 :associative t)
+   (two-arg-f32/        #:vdivss       (f32) (f32 f32) :cost 8)
+   (two-arg-f32=        #:vcmpss       (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:eq) :associative t)
+   (two-arg-f32/=       #:vcmpss       (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:neq) :associative t)
+   (two-arg-f32<        #:vcmpss       (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:lt))
+   (two-arg-f32<=       #:vcmpss       (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:le))
+   (two-arg-f32>        #:vcmpss       (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:nle))
+   (two-arg-f32>=       #:vcmpss       (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:nlt))
+   (f32-andc1           #:vandnps      (f32) (f32 f32) :cost 1)
+   (f32-not             nil            (f32) (f32)     :cost 1 :encoding :fake-vop)
+   (f32-reciprocal      #:vrcpss       (f32) (f32)     :cost 5)
+   (f32-rsqrt           #:vrsqrtss     (f32) (f32)     :cost 5)
+   (f32-sqrt            #:vsqrtss      (f32) (f32)     :cost 15)
+   ;; f64
+   (f64-from-f32        #:vcvtss2sd    (f64) (f32)     :cost 5)
+   (f64-from-s64        nil            (f64) (s64)     :cost 5 :encoding :custom)
+   (f64!-from-p128      nil            (f64) (p128)    :cost 1 :encoding :custom :always-translatable nil)
+   (f64!-from-p256      nil            (f64) (p256)    :cost 1 :encoding :custom :always-translatable nil)
+   (two-arg-f64-and     #:vandpd       (f64) (f64 f64) :cost 1 :associative t)
+   (two-arg-f64-or      #:vorpd        (f64) (f64 f64) :cost 1 :associative t)
+   (two-arg-f64-xor     #:vxorpd       (f64) (f64 f64) :cost 1 :associative t)
+   (two-arg-f64-max     #:vmaxsd       (f64) (f64 f64) :cost 1 :associative t)
+   (two-arg-f64-min     #:vminsd       (f64) (f64 f64) :cost 1 :associative t)
+   (two-arg-f64+        #:vaddsd       (f64) (f64 f64) :cost 1 :associative t)
+   (two-arg-f64-        #:vsubsd       (f64) (f64 f64) :cost 2)
+   (two-arg-f64*        #:vmulsd       (f64) (f64 f64) :cost 2 :associative t)
+   (two-arg-f64/        #:vdivsd       (f64) (f64 f64) :cost 8)
+   (two-arg-f64=        #:vcmpsd       (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:eq) :associative t)
+   (two-arg-f64/=       #:vcmpsd       (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:neq) :associative t)
+   (two-arg-f64<        #:vcmpsd       (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:lt))
+   (two-arg-f64<=       #:vcmpsd       (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:le))
+   (two-arg-f64>        #:vcmpsd       (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:nle))
+   (two-arg-f64>=       #:vcmpsd       (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:nlt))
+   (f64-andc1           #:vandnpd      (f64) (f64 f64) :cost 1)
+   (f64-not             nil            (f64) (f64)     :cost 1 :encoding :fake-vop)
+   (f64-sqrt            #:vsqrtsd      (f64) (f64)     :cost 15)
+   ;; scalar reinterpret casts
+   ( u8!-from-p128      nil            (u8)  (p128) :cost 1 :encoding :fake-vop)
+   ( u8!-from-p256      nil            (u8)  (p256) :cost 1 :encoding :fake-vop)
+   (u16!-from-p128      nil            (u16) (p128) :cost 1 :encoding :fake-vop)
+   (u16!-from-p256      nil            (u16) (p256) :cost 1 :encoding :fake-vop)
+   (u32!-from-p128      nil            (u32) (p128) :cost 1 :encoding :fake-vop)
+   (u32!-from-p256      nil            (u32) (p256) :cost 1 :encoding :fake-vop)
+   (u64!-from-p128      #:movq         (u64) (p128) :cost 1 :always-translatable nil)
+   (u64!-from-p256      #:movq         (u64) (p256) :cost 1 :always-translatable nil)
+   ;; f32.4
+   (f32.4-from-s32.4    #:vcvtdq2ps    (f32.4) (s32.4)       :cost 5)
+   (f32.4!-from-f32     #:vmovups      (f32.4) (f32)         :cost 1 :encoding :move)
+   (f32.4!-from-p128    #:vmovups      (f32.4) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (f32.4!-from-p256    #:vextractf128 (f32.4) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-f32.4          nil            (f32.4) (f32 f32 f32 f32) :cost 1 :encoding :fake-vop)
+   (f32.4-values        nil            (f32 f32 f32 f32) (f32.4) :cost 1 :encoding :fake-vop)
+   (f32.4-broadcast     #:vbroadcastss (f32.4) (f32)         :cost 1)
+   (f32.4-from-f64.4    #:vcvtpd2ps    (f32.4) (f64.4)       :cost 5) ;; wrong code is generated VCVTPD2PS XMM0, XMM0
+   (f32.4-blend         #:vblendvps    (f32.4) (f32.4 f32.4 u32.4) :cost 1)
+   (f32.4-blendc        #:vblendps     (f32.4) (f32.4 f32.4 imm4) :cost 1)
+   (two-arg-f32.4-and   #:vandps       (f32.4) (f32.4 f32.4) :cost 1 :associative t)
+   (two-arg-f32.4-or    #:vorps        (f32.4) (f32.4 f32.4) :cost 1 :associative t)
+   (two-arg-f32.4-xor   #:vxorps       (f32.4) (f32.4 f32.4) :cost 1 :associative t)
+   (two-arg-f32.4-max   #:vmaxps       (f32.4) (f32.4 f32.4) :cost 3 :associative t)
+   (two-arg-f32.4-min   #:vminps       (f32.4) (f32.4 f32.4) :cost 3 :associative t)
+   (f32.4-andc1         #:vandnps      (f32.4) (f32.4 f32.4) :cost 1)
+   (f32.4-not           nil            (f32.4) (f32.4)       :cost 1 :encoding :fake-vop)
+   (two-arg-f32.4+      #:vaddps       (f32.4) (f32.4 f32.4) :cost 2 :associative t)
+   (two-arg-f32.4-      #:vsubps       (f32.4) (f32.4 f32.4) :cost 2)
+   (two-arg-f32.4*      #:vmulps       (f32.4) (f32.4 f32.4) :cost 2 :associative t)
+   (two-arg-f32.4/      #:vdivps       (f32.4) (f32.4 f32.4) :cost 8)
+   (two-arg-f32.4=      #:vcmpps       (u32.4) (f32.4 f32.4) :cost 4 :prefix '(:eq) :associative t)
+   (two-arg-f32.4/=     #:vcmpps       (u32.4) (f32.4 f32.4) :cost 4 :prefix '(:neq) :associative t)
+   (two-arg-f32.4<      #:vcmpps       (u32.4) (f32.4 f32.4) :cost 4 :prefix '(:lt))
+   (two-arg-f32.4<=     #:vcmpps       (u32.4) (f32.4 f32.4) :cost 4 :prefix '(:le))
+   (two-arg-f32.4>      #:vcmpps       (u32.4) (f32.4 f32.4) :cost 4 :prefix '(:gt))
+   (two-arg-f32.4>=     #:vcmpps       (u32.4) (f32.4 f32.4) :cost 4 :prefix '(:ge))
+   (f32.4-horizontal-and nil           (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal-or  nil           (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal-xor nil           (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal-max nil           (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal-min nil           (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal+    nil           (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal*    nil           (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-dupeven       #:vmovsldup    (f32.4) (f32.4)       :cost 1)
+   (f32.4-dupodd        #:vmovshdup    (f32.4) (f32.4)       :cost 1)
+   (f32.4-addsub        #:vaddsubps    (f32.4) (f32.4 f32.4) :cost 3)
+   (f32.4-hadd          #:vhaddps      (f32.4) (f32.4 f32.4) :cost 6)
+   (f32.4-hsub          #:vhsubps      (f32.4) (f32.4 f32.4) :cost 6)
+   (f32.4-reciprocal    #:vrcpps       (f32.4) (f32.4)       :cost 5)
+   (f32.4-rsqrt         #:vrsqrtps     (f32.4) (f32.4)       :cost 5)
+   (f32.4-sqrt          #:vsqrtps      (f32.4) (f32.4)       :cost 15)
+   (f32.4-unpackhi      #:vunpckhps    (f32.4) (f32.4 f32.4) :cost 1)
+   (f32.4-unpacklo      #:vunpcklps    (f32.4) (f32.4 f32.4) :cost 1)
+   (f32.4-movemask      #:vmovmskps    (u4)    (f32.4)       :cost 1)
+   (f32.4-%round        #:vroundps     (f32.4) (f32.4 imm3)  :cost 2)
+   (f32.4-permute       #:vpermilps    (f32.4) (f32.4 imm8)  :cost 1)
+   (f32.4-shuffle       #:vshufps      (f32.4) (f32.4 f32.4 imm8) :cost 1)
+   (f32.4-movemask      #:vmovmskps    (u4)    (f32.4)       :cost 1)
+   ;; f64.2
+   (f64.2!-from-f64     #:vmovupd      (f64.2) (f64)         :cost 1 :encoding :move)
+   (f64.2!-from-p128    #:vmovupd      (f64.2) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (f64.2!-from-p256    #:vextractf128 (f64.2) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-f64.2          nil            (f64.2) (f64 f64)     :cost 1 :encoding :fake-vop)
+   (f64.2-values        nil            (f64 f64) (f64.2)     :cost 1 :encoding :fake-vop)
+   (f64.2-broadcast     #:vmovddup     (f64.2) (f64)         :cost 1)
+   (f64.2-blend         #:vblendvpd    (f64.2) (f64.2 f64.2 u64.2) :cost 1)
+   (f64.2-blendc        #:vblendpd     (f64.2) (f64.2 f64.2 imm2) :cost 1)
+   (two-arg-f64.2-and   #:vandpd       (f64.2) (f64.2 f64.2) :cost 1 :associative t)
+   (two-arg-f64.2-or    #:vorpd        (f64.2) (f64.2 f64.2) :cost 1 :associative t)
+   (two-arg-f64.2-xor   #:vxorpd       (f64.2) (f64.2 f64.2) :cost 1 :associative t)
+   (two-arg-f64.2-max   #:vmaxpd       (f64.2) (f64.2 f64.2) :cost 3 :associative t)
+   (two-arg-f64.2-min   #:vminpd       (f64.2) (f64.2 f64.2) :cost 3 :associative t)
+   (two-arg-f64.2+      #:vaddpd       (f64.2) (f64.2 f64.2) :cost 2 :associative t)
+   (f64.2-andc1         #:vandnpd      (f64.2) (f64.2 f64.2) :cost 1)
+   (f64.2-not           nil            (f64.2) (f64.2)       :cost 1 :encoding :fake-vop)
+   (two-arg-f64.2-      #:vsubpd       (f64.2) (f64.2 f64.2) :cost 2)
+   (two-arg-f64.2*      #:vmulpd       (f64.2) (f64.2 f64.2) :cost 2 :associative t)
+   (two-arg-f64.2/      #:vdivpd       (f64.2) (f64.2 f64.2) :cost 8)
+   (two-arg-f64.2=      #:vcmppd       (u64.2) (f64.2 f64.2) :cost 4 :prefix '(:eq) :associative t)
+   (two-arg-f64.2/=     #:vcmppd       (u64.2) (f64.2 f64.2) :cost 4 :prefix '(:neq) :associative t)
+   (two-arg-f64.2<      #:vcmppd       (u64.2) (f64.2 f64.2) :cost 4 :prefix '(:lt))
+   (two-arg-f64.2<=     #:vcmppd       (u64.2) (f64.2 f64.2) :cost 4 :prefix '(:le))
+   (two-arg-f64.2>      #:vcmppd       (u64.2) (f64.2 f64.2) :cost 4 :prefix '(:gt))
+   (two-arg-f64.2>=     #:vcmppd       (u64.2) (f64.2 f64.2) :cost 4 :prefix '(:ge))
+   (f64.2-horizontal-and nil           (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal-or  nil           (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal-xor nil           (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal-max nil           (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal-min nil           (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal+    nil           (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal*    nil           (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-addsub        #:vaddsubpd    (f64.2) (f64.2 f64.2) :cost 3)
+   (f64.2-hadd          #:vhaddpd      (f64.2) (f64.2 f64.2) :cost 6)
+   (f64.2-hsub          #:vhsubpd      (f64.2) (f64.2 f64.2) :cost 6)
+   (f64.2-sqrt          #:vsqrtpd      (f64.2) (f64.2)       :cost 20)
+   (f64.2-unpackhi      #:vunpckhpd    (f64.2) (f64.2 f64.2) :cost 1)
+   (f64.2-unpacklo      #:vunpcklpd    (f64.2) (f64.2 f64.2) :cost 1)
+   (f64.2-movemask      #:vmovmskpd    (u2)    (f64.2)       :cost 1)
+   (f64.2-%round        #:vroundpd     (f64.2) (f64.2 imm3)  :cost 2)
+   (f64.2-permute       #:vpermilpd    (f64.2) (f64.2 imm2)  :cost 1)
+   (f64.2-shuffle       #:vshufpd      (f64.2) (f64.2 f64.2 imm1) :cost 1)
+   (f64.2-movemask      #:vmovmskpd    (u2)    (f64.2)       :cost 1)
+   ;; f32.8
+   (f32.8-from-s32.8    #:vcvtdq2ps    (f32.8) (s32.8)      :cost 5)
+   (f32.8!-from-f32     #:vmovups      (f32.8) (f32)        :cost 1 :encoding :move)
+   (f32.8!-from-p128    #:vmovups      (f32.8) (p128)       :cost 1 :encoding :move :always-translatable nil)
+   (f32.8!-from-p256    #:vmovups      (f32.8) (p256)       :cost 1 :encoding :move :always-translatable nil)
+   (make-f32.8          nil            (f32.8) (f32 f32 f32 f32 f32 f32 f32 f32) :cost 1 :encoding :fake-vop)
+   (f32.8-values        nil            (f32 f32 f32 f32 f32 f32 f32 f32) (f32.8) :cost 1 :encoding :fake-vop)
+   (f32.8-broadcast     #:vbroadcastss (f32.8) (f32)         :cost 1)
+   (f32.8-from-u32.8    #:vcvtdq2ps    (f32.8) (u32.8)       :cost 5)
+   (f32.8-blend         #:vblendvps    (f32.8) (f32.8 f32.8 u32.8) :cost 1)
+   (f32.8-blendc        #:vblendps     (f32.8) (f32.8 f32.8 imm8) :cost 1)
+   (two-arg-f32.8-and   #:vandps       (f32.8) (f32.8 f32.8) :cost 1 :associative t)
+   (two-arg-f32.8-or    #:vorps        (f32.8) (f32.8 f32.8) :cost 1 :associative t)
+   (two-arg-f32.8-xor   #:vxorps       (f32.8) (f32.8 f32.8) :cost 1 :associative t)
+   (two-arg-f32.8-max   #:vmaxps       (f32.8) (f32.8 f32.8) :cost 3 :associative t)
+   (two-arg-f32.8-min   #:vminps       (f32.8) (f32.8 f32.8) :cost 3 :associative t)
+   (f32.8-andc1         #:vandnps      (f32.8) (f32.8 f32.8) :cost 1)
+   (f32.8-not           nil            (f32.8) (f32.8)       :cost 1 :encoding :fake-vop)
+   (two-arg-f32.8+      #:vaddps       (f32.8) (f32.8 f32.8) :cost 2 :associative t)
+   (two-arg-f32.8-      #:vsubps       (f32.8) (f32.8 f32.8) :cost 2)
+   (two-arg-f32.8*      #:vmulps       (f32.8) (f32.8 f32.8) :cost 2 :associative t)
+   (two-arg-f32.8/      #:vdivps       (f32.8) (f32.8 f32.8) :cost 8)
+   (two-arg-f32.8=      #:vcmpps       (u32.8) (f32.8 f32.8) :cost 4 :prefix '(:eq) :associative t)
+   (two-arg-f32.8/=     #:vcmpps       (u32.8) (f32.8 f32.8) :cost 4 :prefix '(:neq) :associative t)
+   (two-arg-f32.8<      #:vcmpps       (u32.8) (f32.8 f32.8) :cost 4 :prefix '(:lt))
+   (two-arg-f32.8<=     #:vcmpps       (u32.8) (f32.8 f32.8) :cost 4 :prefix '(:le))
+   (two-arg-f32.8>      #:vcmpps       (u32.8) (f32.8 f32.8) :cost 4 :prefix '(:gt))
+   (two-arg-f32.8>=     #:vcmpps       (u32.8) (f32.8 f32.8) :cost 4 :prefix '(:ge))
+   (f32.8-horizontal-and nil           (f32)   (f32.8)       :cost 5 :encoding :fake-vop)
+   (f32.8-horizontal-or  nil           (f32)   (f32.8)       :cost 5 :encoding :fake-vop)
+   (f32.8-horizontal-xor nil           (f32)   (f32.8)       :cost 5 :encoding :fake-vop)
+   (f32.8-horizontal-max nil           (f32)   (f32.8)       :cost 5 :encoding :fake-vop)
+   (f32.8-horizontal-min nil           (f32)   (f32.8)       :cost 5 :encoding :fake-vop)
+   (f32.8-horizontal+    nil           (f32)   (f32.8)       :cost 5 :encoding :fake-vop)
+   (f32.8-horizontal*    nil           (f32)   (f32.8)       :cost 5 :encoding :fake-vop)
+   (f32.8-dupeven       #:vmovsldup    (f32.8) (f32.8)       :cost 1)
+   (f32.8-dupodd        #:vmovshdup    (f32.8) (f32.8)       :cost 1)
+   (f32.8-hadd          #:vhaddps      (f32.8) (f32.8 f32.8) :cost 6)
+   (f32.8-hsub          #:vhsubps      (f32.8) (f32.8 f32.8) :cost 6)
+   (f32.8-reciprocal    #:vrcpps       (f32.8) (f32.8)       :cost 5)
+   (f32.8-rsqrt         #:vrsqrtps     (f32.8) (f32.8)       :cost 5)
+   (f32.8-sqrt          #:vsqrtps      (f32.8) (f32.8)       :cost 15)
+   (f32.8-unpackhi      #:vunpckhps    (f32.8) (f32.8 f32.8) :cost 1)
+   (f32.8-unpacklo      #:vunpcklps    (f32.8) (f32.8 f32.8) :cost 1)
+   (f32.8-movemask      #:vmovmskps    (u8)    (f32.8)       :cost 1)
+   (f32.8-%round        #:vroundps     (f32.8) (f32.8 imm3)  :cost 2)
+   (f32.8-permute       #:vpermilps    (f32.8) (f32.8 imm8)  :cost 1)
+   (f32.8-permute128    #:vperm2f128   (f32.8) (f32.8 f32.8 imm8) :cost 1)
+   (f32.8-shuffle       #:vshufps      (f32.8) (f32.8 f32.8 imm8) :cost 1)
+   (f32.4-from-f32.8    #:vextractf128 (f32.4) (f32.8 imm1)  :cost 1)
+   (f32.8-insert-f32.4  #:vinsertf128  (f32.8) (f32.8 f32.4 imm1) :cost 1)
+   (f32.8-movemask      #:vmovmskps    (u8)    (f32.8)       :cost 1)
+   ;; f64.4
+   (f64.4!-from-f64     #:vmovupd      (f64.4) (f64)         :cost 1 :encoding :move)
+   (f64.4!-from-p128    #:vmovupd      (f64.4) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (f64.4!-from-p256    #:vmovupd      (f64.4) (p256)        :cost 1 :encoding :move :always-translatable nil)
+   (make-f64.4          nil            (f64.4) (f64 f64 f64 f64) :cost 1 :encoding :fake-vop)
+   (f64.4-values        nil            (f64 f64 f64 f64) (f64.4) :cost 1 :encoding :fake-vop)
+   (f64.4-broadcast     #:vbroadcastsd (f64.4) (f64)         :cost 1)
+   (f64.4-from-f32.4    #:vcvtps2pd    (f64.4) (f32.4)       :cost 5)
+   (f64.4-from-u32.4    #:vcvtdq2pd    (f64.4) (u32.4)       :cost 5)
+   (f64.4-from-s32.4    #:vcvtdq2pd    (f64.4) (s32.4)       :cost 5)
+   (f64.4-blend         #:vblendvpd    (f64.4) (f64.4 f64.4 u64.4) :cost 1)
+   (f64.4-blendc        #:vblendpd     (f64.4) (f64.4 f64.4 imm4) :cost 1)
+   (two-arg-f64.4-and   #:vandpd       (f64.4) (f64.4 f64.4) :cost 1 :associative t)
+   (two-arg-f64.4-or    #:vorpd        (f64.4) (f64.4 f64.4) :cost 1 :associative t)
+   (two-arg-f64.4-xor   #:vxorpd       (f64.4) (f64.4 f64.4) :cost 1 :associative t)
+   (two-arg-f64.4-max   #:vmaxpd       (f64.4) (f64.4 f64.4) :cost 3 :associative t)
+   (two-arg-f64.4-min   #:vminpd       (f64.4) (f64.4 f64.4) :cost 3 :associative t)
+   (two-arg-f64.4+      #:vaddpd       (f64.4) (f64.4 f64.4) :cost 2 :associative t)
+   (two-arg-f64.4-      #:vsubpd       (f64.4) (f64.4 f64.4) :cost 2)
+   (two-arg-f64.4*      #:vmulpd       (f64.4) (f64.4 f64.4) :cost 2 :associative t)
+   (two-arg-f64.4/      #:vdivpd       (f64.4) (f64.4 f64.4) :cost 8)
+   (two-arg-f64.4=      #:vcmppd       (u64.4) (f64.4 f64.4) :cost 4 :prefix '(:eq) :associative t)
+   (two-arg-f64.4/=     #:vcmppd       (u64.4) (f64.4 f64.4) :cost 4 :prefix '(:neq) :associative t)
+   (two-arg-f64.4<      #:vcmppd       (u64.4) (f64.4 f64.4) :cost 4 :prefix '(:lt))
+   (two-arg-f64.4<=     #:vcmppd       (u64.4) (f64.4 f64.4) :cost 4 :prefix '(:le))
+   (two-arg-f64.4>      #:vcmppd       (u64.4) (f64.4 f64.4) :cost 4 :prefix '(:gt))
+   (two-arg-f64.4>=     #:vcmppd       (u64.4) (f64.4 f64.4) :cost 4 :prefix '(:ge))
+   (f64.4-horizontal-and nil           (f64)   (f64.4)       :cost 3 :encoding :fake-vop)
+   (f64.4-horizontal-or  nil           (f64)   (f64.4)       :cost 3 :encoding :fake-vop)
+   (f64.4-horizontal-xor nil           (f64)   (f64.4)       :cost 3 :encoding :fake-vop)
+   (f64.4-horizontal-max nil           (f64)   (f64.4)       :cost 3 :encoding :fake-vop)
+   (f64.4-horizontal-min nil           (f64)   (f64.4)       :cost 3 :encoding :fake-vop)
+   (f64.4-horizontal+    nil           (f64)   (f64.4)       :cost 3 :encoding :fake-vop)
+   (f64.4-horizontal*    nil           (f64)   (f64.4)       :cost 3 :encoding :fake-vop)
+   (f64.4-andc1         #:vandnpd      (f64.4) (f64.4 f64.4) :cost 1)
+   (f64.4-not           nil            (f64.4) (f64.4)       :cost 1 :encoding :fake-vop)
+   (f64.4-dupeven       #:vmovddup     (f64.4) (f64.4)       :cost 1)
+   (f64.4-hadd          #:vhaddpd      (f64.4) (f64.4 f64.4) :cost 6)
+   (f64.4-hsub          #:vhsubpd      (f64.4) (f64.4 f64.4) :cost 6)
+   (f64.4-sqrt          #:vsqrtpd      (f64.4) (f64.4)       :cost 20)
+   (f64.4-unpackhi      #:vunpckhpd    (f64.4) (f64.4 f64.4) :cost 1)
+   (f64.4-unpacklo      #:vunpcklpd    (f64.4) (f64.4 f64.4) :cost 1)
+   (f64.4-movemask      #:vmovmskpd    (u4)    (f64.4)       :cost 1)
+   (f64.4-%round        #:vroundpd     (f64.4) (f64.4 imm3)  :cost 2)
+   (f64.4-permute       #:vpermilpd    (f64.4) (f64.4 imm4)  :cost 1)
+   (f64.4-permute128    #:vperm2f128   (f64.4) (f64.4 f64.4 imm8) :cost 1)
+   (f64.4-shuffle       #:vshufpd      (f64.4) (f64.4 f64.4 imm2) :cost 1)
+   (f64.4-reverse       #:vpermilpd    (f64.4) (f64.4)       :cost 2 :encoding :fake-vop)
+   (f64.2-from-f64.4    #:vextractf128 (f64.2) (f64.4 imm1)  :cost 1)
+   (f64.4-insert-f64.2  #:vinsertf128  (f64.4) (f64.4 f64.2 imm1) :cost 1)
+   (f64.4-set128        #:vinsertf128  (f64.4) (f64.2 f64.2 imm1) :cost 1)
+   (f64.4-movemask      #:vmovmskpd    (u4)    (f64.4)       :cost 1)
+   ;; u8.16
+   (u8.16!-from-u8      #:movq         (u8.16) (u8)          :cost 1)
+   (u8.16!-from-p128    #:vmovdqu      (u8.16) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (u8.16!-from-p256    #:vextractf128 (u8.16) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-u8.16          nil            (u8.16) (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) :cost 1 :encoding :fake-vop)
+   (u8.16-values        nil            (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) (u8.16) :cost 1 :encoding :fake-vop)
+   (u8.16-broadcast     nil            (u8.16) (u8)          :cost 1 :encoding :fake-vop)
+   (u8.16-blend         #:vpblendvb    (u8.16) (u8.16 u8.16 u8.16) :cost 1)
+   (two-arg-u8.16-and   #:vpand        (u8.16) (u8.16 u8.16) :cost 1 :associative t)
+   (two-arg-u8.16-or    #:vpor         (u8.16) (u8.16 u8.16) :cost 1 :associative t)
+   (two-arg-u8.16-xor   #:vpxor        (u8.16) (u8.16 u8.16) :cost 1 :associative t)
+   (u8.16-andc1         #:vpandn       (u8.16) (u8.16 u8.16) :cost 1)
+   (u8.16-not           nil            (u8.16) (u8.16)       :cost 1 :encoding :fake-vop)
+   (two-arg-u8.16+      #:vpaddb       (u8.16) (u8.16 u8.16) :cost 2 :associative t)
+   (two-arg-u8.16-      #:vpsubb       (u8.16) (u8.16 u8.16) :cost 2)
+   (two-arg-u8.16=      #:vpcmpeqb     (u8.16) (u8.16 u8.16) :cost 1 :associative t)
+   (two-arg-u8.16/=     nil            (u8.16) (u8.16 u8.16) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u8.16>~     #:vpcmpgtb     (u8.16) (u8.16 u8.16) :cost 1)
+   (two-arg-u8.16>      nil            (u8.16) (u8.16 u8.16) :cost 1 :encoding :fake-vop)
+   (two-arg-u8.16<      nil            (u8.16) (u8.16 u8.16) :cost 1 :encoding :fake-vop)
+   (two-arg-u8.16>=     nil            (u8.16) (u8.16 u8.16) :cost 2 :encoding :fake-vop)
+   (two-arg-u8.16<=     nil            (u8.16) (u8.16 u8.16) :cost 2 :encoding :fake-vop)
+   (u8.16-unpackhi      #:vpunpckhbw   (u8.16) (u8.16 u8.16) :cost 1)
+   (u8.16-unpacklo      #:vpunpcklbw   (u8.16) (u8.16 u8.16) :cost 1)
+   (u8.16-movemask      #:vpmovmskb    (u16)   (u8.16)       :cost 1)
+   (u8.16-shuffle       #:vpshufb      (u8.16) (u8.16 u8.16) :cost 1) ;; TODO missing imm
+   ;; u16.8
+   (u16.8!-from-u16     #:movq         (u16.8) (u16)         :cost 1)
+   (u16.8!-from-p128    #:vmovdqu      (u16.8) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (u16.8!-from-p256    #:vextractf128 (u16.8) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-u16.8          nil            (u16.8) (u16 u16 u16 u16 u16 u16 u16 u16) :cost 1 :encoding :fake-vop)
+   (u16.8-values        nil            (u16 u16 u16 u16 u16 u16 u16 u16) (u16.8) :cost 1 :encoding :fake-vop)
+   (u16.8-broadcast     nil            (u16.8) (u16)         :cost 1 :encoding :fake-vop)
+   (u16.8-blend         #:vpblendvb    (u16.8) (u16.8 u16.8 u16.8) :cost 1)
+   (two-arg-u16.8-and   #:vpand        (u16.8) (u16.8 u16.8) :cost 1 :associative t)
+   (two-arg-u16.8-or    #:vpor         (u16.8) (u16.8 u16.8) :cost 1 :associative t)
+   (two-arg-u16.8-xor   #:vpxor        (u16.8) (u16.8 u16.8) :cost 1 :associative t)
+   (u16.8-andc1         #:vpandn       (u16.8) (u16.8 u16.8) :cost 1)
+   (u16.8-not           nil            (u16.8) (u16.8)       :cost 1 :encoding :fake-vop)
+   (two-arg-u16.8+      #:vpaddw       (u16.8) (u16.8 u16.8) :cost 2 :associative t)
+   (two-arg-u16.8-      #:vpsubw       (u16.8) (u16.8 u16.8) :cost 2)
+   (two-arg-u16.8=      #:vpcmpeqw     (u16.8) (u16.8 u16.8) :cost 1 :associative t)
+   (two-arg-u16.8/=     nil            (u16.8) (u16.8 u16.8) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u16.8>~     #:vpcmpgtw     (u16.8) (u16.8 u16.8) :cost 1)
+   (two-arg-u16.8>      nil            (u16.8) (u16.8 u16.8) :cost 1 :encoding :fake-vop)
+   (two-arg-u16.8<      nil            (u16.8) (u16.8 u16.8) :cost 1 :encoding :fake-vop)
+   (two-arg-u16.8>=     nil            (u16.8) (u16.8 u16.8) :cost 2 :encoding :fake-vop)
+   (two-arg-u16.8<=     nil            (u16.8) (u16.8 u16.8) :cost 2 :encoding :fake-vop)
+   (u16.8-shiftl        #:vpsllw       (u16.8) (u16.8 u16.8) :cost 1)
+   (u16.8-shiftr        #:vpsrlw       (u16.8) (u16.8 u16.8) :cost 1)
+   (u16.8-unpackhi      #:vpunpckhwd   (u16.8) (u16.8 u16.8) :cost 1)
+   (u16.8-unpacklo      #:vpunpcklwd   (u16.8) (u16.8 u16.8) :cost 1)
+   (u16.8-movemask      nil            (u8)    (u16.8)       :cost 1 :encoding :fake-vop)
+   (u16.8-shufflehi     #:vpshufhw     (u16.8) (u16.8 imm8)  :cost 1)
+   (u16.8-shufflelo     #:vpshuflw     (u16.8) (u16.8 imm8)  :cost 1)
+   ;; u32.4
+   (u32.4!-from-u32     #:movq         (u32.4) (u32)         :cost 1)
+   (u32.4!-from-p128    #:vmovdqu      (u32.4) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (u32.4!-from-p256    #:vextractf128 (u32.4) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-u32.4          nil            (u32.4) (u32 u32 u32 u32) :cost 1 :encoding :fake-vop)
+   (u32.4-values        nil            (u32 u32 u32 u32) (u32.4) :cost 1 :encoding :fake-vop)
+   (u32.4-broadcast     nil            (u32.4) (u32)         :cost 1 :encoding :fake-vop)
+   (u32.4-blend         #:vpblendvb    (u32.4) (u32.4 u32.4 u32.4) :cost 1)
+   (two-arg-u32.4-and   #:vpand        (u32.4) (u32.4 u32.4) :cost 1 :associative t)
+   (two-arg-u32.4-or    #:vpor         (u32.4) (u32.4 u32.4) :cost 1 :associative t)
+   (two-arg-u32.4-xor   #:vpxor        (u32.4) (u32.4 u32.4) :cost 1 :associative t)
+   (u32.4-andc1         #:vpandn       (u32.4) (u32.4 u32.4) :cost 1)
+   (u32.4-not           nil            (u32.4) (u32.4)       :cost 1 :encoding :fake-vop)
+   (two-arg-u32.4+      #:vpaddd       (u32.4) (u32.4 u32.4) :cost 2 :associative t)
+   (two-arg-u32.4-      #:vpsubd       (u32.4) (u32.4 u32.4) :cost 2)
+   (two-arg-u32.4=      #:vpcmpeqd     (u32.4) (u32.4 u32.4) :cost 1 :associative t)
+   (two-arg-u32.4/=     nil            (u32.4) (u32.4 u32.4) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u32.4>~     #:vpcmpgtd     (u32.4) (u32.4 u32.4) :cost 1)
+   (two-arg-u32.4>      nil            (u32.4) (u32.4 u32.4) :cost 1 :encoding :fake-vop)
+   (two-arg-u32.4<      nil            (u32.4) (u32.4 u32.4) :cost 1 :encoding :fake-vop)
+   (two-arg-u32.4>=     nil            (u32.4) (u32.4 u32.4) :cost 2 :encoding :fake-vop)
+   (two-arg-u32.4<=     nil            (u32.4) (u32.4 u32.4) :cost 2 :encoding :fake-vop)
+   (u32.4-unpackhi      #:vpunpckhdq   (u32.4) (u32.4 u32.4) :cost 1)
+   (u32.4-unpacklo      #:vpunpckldq   (u32.4) (u32.4 u32.4) :cost 1)
+   (u32.4-movemask      #:vmovmskps    (u4)    (u32.4)       :cost 1)
+   (u32.4-permute       #:vpermilps    (u32.4) (u32.4 imm8)  :cost 1)
+   ;; u64.2
+   (u64.2!-from-u64     #:movq         (u64.2) (u64)         :cost 1)
+   (u64.2!-from-p128    #:vmovdqu      (u64.2) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (u64.2!-from-p256    #:vextractf128 (u64.2) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-u64.2          nil            (u64.2) (u64 u64)     :cost 1 :encoding :fake-vop)
+   (u64.2-values        nil            (u64 u64) (u64.2)     :cost 1 :encoding :fake-vop)
+   (u64.2-broadcast     nil            (u64.2) (u64)         :cost 1 :encoding :fake-vop)
+   (u64.2-from-u64      #:movq         (u64.2) (u64)         :cost 1)
+   (u64.2-blend         #:vpblendvb    (u64.2) (u64.2 u64.2 u64.2) :cost 1)
+   (two-arg-u64.2-and   #:vpand        (u64.2) (u64.2 u64.2) :cost 1 :associative t)
+   (two-arg-u64.2-or    #:vpor         (u64.2) (u64.2 u64.2) :cost 1 :associative t)
+   (two-arg-u64.2-xor   #:vpxor        (u64.2) (u64.2 u64.2) :cost 1 :associative t)
+   (u64.2-andc1         #:vpandn       (u64.2) (u64.2 u64.2) :cost 1)
+   (u64.2-not           nil            (u64.2) (u64.2)       :cost 1 :encoding :fake-vop)
+   (two-arg-u64.2+      #:vpaddq       (u64.2) (u64.2 u64.2) :cost 2 :associative t)
+   (two-arg-u64.2-      #:vpsubq       (u64.2) (u64.2 u64.2) :cost 2)
+   (two-arg-u64.2=      #:vpcmpeqq     (u64.2) (u64.2 u64.2) :cost 1 :associative t)
+   (two-arg-u64.2/=     nil            (u64.2) (u64.2 u64.2) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u64.2>~     #:vpcmpgtq     (u64.2) (u64.2 u64.2) :cost 1)
+   (two-arg-u64.2>      nil            (u64.2) (u64.2 u64.2) :cost 1 :encoding :fake-vop)
+   (two-arg-u64.2<      nil            (u64.2) (u64.2 u64.2) :cost 1 :encoding :fake-vop)
+   (two-arg-u64.2>=     nil            (u64.2) (u64.2 u64.2) :cost 2 :encoding :fake-vop)
+   (two-arg-u64.2<=     nil            (u64.2) (u64.2 u64.2) :cost 2 :encoding :fake-vop)
+   (u64.2-unpackhi      #:vpunpckhqdq  (u64.2) (u64.2 u64.2) :cost 1)
+   (u64.2-unpacklo      #:vpunpcklqdq  (u64.2) (u64.2 u64.2) :cost 1)
+   (u64.2-movemask      #:vmovmskpd    (u2)    (u64.2)       :cost 1)
+   (u64.2-permute       #:vpermilpd    (u64.2) (u64.2 imm8)  :cost 1)
+   ;; u8.32
+   (u8.32!-from-u8      #:movq         (u8.32) (u8) :cost 1)
+   (u8.32!-from-p128    #:vmovdqu      (u8.32) (p128) :cost 1 :encoding :move :always-translatable nil)
+   (u8.32!-from-p256    #:vmovdqu      (u8.32) (p256) :cost 1 :encoding :move :always-translatable nil)
+   (make-u8.32          nil            (u8.32) (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) :cost 1 :encoding :fake-vop)
+   (u8.32-values        nil            (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) (u8.32) :cost 1 :encoding :fake-vop)
+   (u8.32-broadcast     nil            (u8.32) (u8) :cost 1 :encoding :fake-vop)
+   (u8.16-from-u8.32    #:vextractf128 (u8.16) (u8.32 imm1) :cost 1)
+   (u8.32-insert-u8.16  #:vinsertf128  (u8.32) (u8.32 u8.16 imm1) :cost 1)
+   ;; u16.16
+   (u16.16!-from-u16    #:movq         (u16.16) (u16) :cost 1)
+   (u16.16!-from-p128   #:vmovdqu      (u16.16) (p128) :cost 1 :encoding :move :always-translatable nil)
+   (u16.16!-from-p256   #:vmovdqu      (u16.16) (p256) :cost 1 :encoding :move :always-translatable nil)
+   (make-u16.16         nil            (u16.16) (u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16) :cost 1 :encoding :fake-vop)
+   (u16.16-values       nil            (u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16) (u16.16) :cost 1 :encoding :fake-vop)
+   (u16.16-broadcast    nil            (u16.16) (u16) :cost 1 :encoding :fake-vop)
+   (u16.8-from-u16.16   #:vextractf128 (u16.8)  (u16.16 imm1) :cost 1)
+   (u16.16-insert-u16.8 #:vinsertf128  (u16.16) (u16.16 u16.8 imm1) :cost 1)
+   ;; u32.8
+   (u32.8!-from-u32     #:movq         (u32.8) (u32) :cost 1)
+   (u32.8!-from-p128    #:vmovdqu      (u32.8) (p128) :cost 1 :encoding :move :always-translatable nil)
+   (u32.8!-from-p256    #:vmovdqu      (u32.8) (p256) :cost 1 :encoding :move :always-translatable nil)
+   (make-u32.8          nil            (u32.8) (u32 u32 u32 u32 u32 u32 u32 u32) :cost 1 :encoding :fake-vop)
+   (two-arg-u32.8-and   #:vandps       (u32.8) (u32.8 u32.8) :cost 1 :associative t)
+   (u32.8-values        nil            (u32 u32 u32 u32 u32 u32 u32 u32) (u32.8) :cost 1 :encoding :fake-vop)
+   (u32.8-broadcast     nil            (u32.8) (u32) :cost 1 :encoding :fake-vop)
+   (u32.8-permute       #:vpermilps    (u32.8) (u32.4 imm8) :cost 1)
+   (u32.4-from-u32.8    #:vextractf128 (u32.4) (u32.8 imm1) :cost 1)
+   (u32.8-insert-u32.4  #:vinsertf128  (u32.8) (u32.8 u32.4 imm1) :cost 1)
+   ;; u64.4
+   (u64.4!-from-u64     #:movq         (u64.4) (u64) :cost 1)
+   (u64.4!-from-p128    #:vmovdqu      (u64.4) (p128) :cost 1 :encoding :move :always-translatable nil)
+   (u64.4!-from-p256    #:vmovdqu      (u64.4) (p256) :cost 1 :encoding :move :always-translatable nil)
+   (make-u64.4          nil            (u64.4) (u64 u64 u64 u64) :cost 1 :encoding :fake-vop)
+   (two-arg-u64.4-and   #:vandpd       (u64.4) (u64.4 u64.4) :cost 1 :associative t)
+   (u64.4-values        nil            (u64 u64 u64 u64) (u64.4) :cost 1 :encoding :fake-vop)
+   (u64.4-broadcast     nil            (u64.4) (u64) :cost 1 :encoding :fake-vop)
+   (u64.4-permute       #:vpermilpd    (u64.4) (u64.4 imm8) :cost 1)
+   (u64.2-from-u64.4    #:vextractf128 (u64.2) (u64.4 imm1) :cost 1)
+   (u64.4-insert-u64.2  #:vinsertf128  (u64.4) (u64.4 u64.2 imm1) :cost 1)
+   ;; s8.16
+   (s8.16!-from-s8      nil            (s8.16) (s8)          :cost 1 :encoding :fake-vop)
+   (s8.16!-from-p128    #:vmovdqu      (s8.16) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (s8.16!-from-p256    #:vextractf128 (s8.16) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-s8.16          nil            (s8.16) (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) :cost 1 :encoding :fake-vop)
+   (s8.16-values        nil            (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) (s8.16) :cost 1 :encoding :fake-vop)
+   (s8.16-broadcast     nil            (s8.16)    (s8)       :cost 1 :encoding :fake-vop)
+   (s8.16-blend         #:vpblendvb    (s8.16) (s8.16 s8.16 u8.16) :cost 1)
+   (two-arg-s8.16-and   #:vpand        (s8.16) (s8.16 s8.16) :cost 1 :associative t)
+   (two-arg-s8.16-or    #:vpor         (s8.16) (s8.16 s8.16) :cost 1 :associative t)
+   (two-arg-s8.16-xor   #:vpxor        (s8.16) (s8.16 s8.16) :cost 1 :associative t)
+   (s8.16-andc1         #:vpandn       (s8.16) (s8.16 s8.16) :cost 1)
+   (s8.16-not           nil            (s8.16) (s8.16)       :cost 1 :encoding :fake-vop)
+   (two-arg-s8.16+      #:vpaddb       (s8.16) (s8.16 s8.16) :cost 2 :associative t)
+   (two-arg-s8.16-      #:vpsubb       (s8.16) (s8.16 s8.16) :cost 2)
+   (two-arg-s8.16=      #:vpcmpeqb     (u8.16) (s8.16 s8.16) :cost 1 :associative t)
+   (two-arg-s8.16/=     nil            (u8.16) (s8.16 s8.16) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s8.16>      #:vpcmpgtb     (u8.16) (s8.16 s8.16) :cost 1)
+   (two-arg-s8.16<      nil            (u8.16) (s8.16 s8.16) :cost 1 :encoding :fake-vop)
+   (two-arg-s8.16>=     nil            (u8.16) (s8.16 s8.16) :cost 2 :encoding :fake-vop)
+   (two-arg-s8.16<=     nil            (u8.16) (s8.16 s8.16) :cost 2 :encoding :fake-vop)
+   (s8.16-unpackhi      #:vpunpckhbw   (s8.16) (s8.16 s8.16) :cost 1)
+   (s8.16-unpacklo      #:vpunpcklbw   (s8.16) (s8.16 s8.16) :cost 1)
+   (s8.16-movemask      #:vpmovmskb    (u16)   (s8.16)       :cost 1)
+   (s8.16-shuffle       #:vpshufb      (s8.16) (s8.16 u8.16) :cost 1)
+   ;; s16.8
+   (s16.8!-from-s16     nil            (s16.8) (s16)         :cost 1 :encoding :fake-vop)
+   (s16.8!-from-p128    #:vmovdqu      (s16.8) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (s16.8!-from-p256    #:vextractf128 (s16.8) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-s16.8          nil            (s16.8) (s16 s16 s16 s16 s16 s16 s16 s16) :cost 1 :encoding :fake-vop)
+   (s16.8-values        nil            (s16 s16 s16 s16 s16 s16 s16 s16) (s16.8) :cost 1 :encoding :fake-vop)
+   (s16.8-broadcast     nil            (s16.8) (s16)         :cost 1 :encoding :fake-vop)
+   (s16.8-blend         #:vpblendvb    (s16.8) (s16.8 s16.8 u16.8) :cost 1)
+   (two-arg-s16.8-and   #:vpand        (s16.8) (s16.8 s16.8) :cost 1 :associative t)
+   (two-arg-s16.8-or    #:vpor         (s16.8) (s16.8 s16.8) :cost 1 :associative t)
+   (two-arg-s16.8-xor   #:vpxor        (s16.8) (s16.8 s16.8) :cost 1 :associative t)
+   (s16.8-andc1         #:vpandn       (s16.8) (s16.8 s16.8) :cost 1)
+   (s16.8-not           nil            (s16.8) (s16.8)       :cost 1 :encoding :fake-vop)
+   (two-arg-s16.8+      #:vpaddw       (s16.8) (s16.8 s16.8) :cost 2 :associative t)
+   (two-arg-s16.8-      #:vpsubw       (s16.8) (s16.8 s16.8) :cost 2)
+   (two-arg-s16.8-mullo #:vpmullw      (s16.8) (s16.8 s16.8) :cost 2 :associative t)
+   (s16.8-shiftl        #:vpsllq       (s16.8) (s16.8 s16.8) :cost 1)
+   (s16.8-shiftr        #:vpsrlq       (s16.8) (s16.8 s16.8) :cost 1)
+   (two-arg-s16.8=      #:vpcmpeqw     (u16.8) (s16.8 s16.8) :cost 1 :associative t)
+   (two-arg-s16.8/=     nil            (u16.8) (s16.8 s16.8) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s16.8>      #:vpcmpgtw     (u16.8) (s16.8 s16.8) :cost 1)
+   (two-arg-s16.8<      nil            (u16.8) (s16.8 s16.8) :cost 1 :encoding :fake-vop)
+   (two-arg-s16.8>=     nil            (u16.8) (s16.8 s16.8) :cost 2 :encoding :fake-vop)
+   (two-arg-s16.8<=     nil            (u16.8) (s16.8 s16.8) :cost 2 :encoding :fake-vop)
+   (s16.8-shiftl        #:vpsllw       (s16.8) (s16.8 s16.8) :cost 1)
+   (s16.8-shiftr        #:vpsrlw       (s16.8) (s16.8 s16.8) :cost 1)
+   (s16.8-unpackhi      #:vpunpckhwd   (s16.8) (s16.8 s16.8) :cost 1)
+   (s16.8-unpacklo      #:vpunpcklwd   (s16.8) (s16.8 s16.8) :cost 1)
+   (s16.8-movemask      nil            (u8)    (s16.8)       :cost 1 :encoding :fake-vop)
+   (s16.8-shufflehi     #:vpshufhw     (s16.8) (s16.8 imm8)  :cost 1)
+   (s16.8-shufflelo     #:vpshuflw     (s16.8) (s16.8 imm8)  :cost 1)
+   (two-arg-s16.8-mullo #:vpmullw      (s16.8) (s16.8 s16.8) :cost 2 :associative t)
+   ;; s32.4
+   (s32.4!-from-s32     nil            (s32.4) (s32)         :cost 1 :encoding :fake-vop)
+   (s32.4!-from-p128    #:vmovdqu      (s32.4) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (s32.4!-from-p256    #:vextractf128 (s32.4) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-s32.4          nil            (s32.4) (s32 s32 s32 s32) :cost 1 :encoding :fake-vop)
+   (s32.4-values        nil            (s32 s32 s32 s32) (s32.4) :cost 1 :encoding :fake-vop)
+   (s32.4-broadcast     nil            (s32.4) (s32)         :cost 1 :encoding :fake-vop)
+   (s32.4-blend         #:vpblendvb    (s32.4) (s32.4 s32.4 u32.4) :cost 1)
+   (s32.4-from-f64.4    #:vcvtpd2dq    (s32.4) (f64.4)       :cost 5) ;; wrong code is generated VCVTPD2DQ XMM0, XMM0. 3rd and 4th elements are 0s
+   (s32.4-from-f32.4    #:vcvtps2dq    (s32.4) (f32.4)       :cost 5)
+   (two-arg-s32.4-and   #:vpand        (s32.4) (s32.4 s32.4) :cost 1 :associative t)
+   (two-arg-s32.4-or    #:vpor         (s32.4) (s32.4 s32.4) :cost 1 :associative t)
+   (two-arg-s32.4-xor   #:vpxor        (s32.4) (s32.4 s32.4) :cost 1 :associative t)
+   (s32.4-andc1         #:vpandn       (s32.4) (s32.4 s32.4) :cost 1)
+   (s32.4-not           nil            (s32.4) (s32.4)       :cost 1 :encoding :fake-vop)
+   (two-arg-s32.4+      #:vpaddd       (s32.4) (s32.4 s32.4) :cost 2 :associative t)
+   (two-arg-s32.4-      #:vpsubd       (s32.4) (s32.4 s32.4) :cost 2)
+   (two-arg-s32.4-mullo #:vpmulld      (s32.4) (s32.4 s32.4) :cost 2 :associative t)
+   (two-arg-s32.4=      #:vpcmpeqd     (u32.4) (s32.4 s32.4) :cost 1 :associative t)
+   (two-arg-s32.4/=     nil            (u32.4) (s32.4 s32.4) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s32.4>      #:vpcmpgtd     (u32.4) (s32.4 s32.4) :cost 1)
+   (two-arg-s32.4<      nil            (u32.4) (s32.4 s32.4) :cost 1 :encoding :fake-vop)
+   (two-arg-s32.4>=     nil            (u32.4) (s32.4 s32.4) :cost 2 :encoding :fake-vop)
+   (two-arg-s32.4<=     nil            (u32.4) (s32.4 s32.4) :cost 2 :encoding :fake-vop)
+   (s32.4-unpackhi      #:vpunpckhdq   (s32.4) (s32.4 s32.4) :cost 1)
+   (s32.4-unpacklo      #:vpunpckldq   (s32.4) (s32.4 s32.4) :cost 1)
+   (s32.4-movemask      #:vmovmskps    (u4)    (s32.4)       :cost 1)
+   (s32.4-permute       #:vpermilps    (s32.4) (s32.4 imm8)  :cost 1)
+   ;; s64.2
+   (s64.2!-from-s64     nil            (s64.2) (s64)         :cost 1 :encoding :fake-vop)
+   (s64.2!-from-p128    #:vmovdqu      (s64.2) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (s64.2!-from-p256    #:vextractf128 (s64.2) (p256)        :cost 1 :suffix '(0) :always-translatable nil)
+   (make-s64.2          nil            (s64.2) (s64 s64)     :cost 1 :encoding :fake-vop)
+   (s64.2-values        nil            (s64 s64) (s64.2)     :cost 1 :encoding :fake-vop)
+   (s64.2-broadcast     nil            (s64.2) (s64)         :cost 1 :encoding :fake-vop)
+   (s64.2-blend         #:vpblendvb    (s64.2) (s64.2 s64.2 u64.2) :cost 1)
+   (two-arg-s64.2-and   #:vpand        (s64.2) (s64.2 s64.2) :cost 1 :associative t)
+   (two-arg-s64.2-or    #:vpor         (s64.2) (s64.2 s64.2) :cost 1 :associative t)
+   (two-arg-s64.2-xor   #:vpxor        (s64.2) (s64.2 s64.2) :cost 1 :associative t)
+   (s64.2-andc1         #:vpandn       (s64.2) (s64.2 s64.2) :cost 1)
+   (s64.2-not           nil            (s64.2) (s64.2)       :cost 1 :encoding :fake-vop)
+   (two-arg-s64.2+      #:vpaddq       (s64.2) (s64.2 s64.2) :cost 2 :associative t)
+   (two-arg-s64.2-      #:vpsubq       (s64.2) (s64.2 s64.2) :cost 2)
+   (s64.2-shiftl        #:vpsllq       (s64.2) (s64.2 s64.2) :cost 1)
+   (s64.2-shiftr        #:vpsrlq       (s64.2) (s64.2 s64.2) :cost 1)
+   (two-arg-s64.2=      #:vpcmpeqq     (u64.2) (s64.2 s64.2) :cost 1 :associative t)
+   (two-arg-s64.2/=     nil            (u64.2) (s64.2 s64.2) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s64.2>      #:vpcmpgtq     (u64.2) (s64.2 s64.2) :cost 1)
+   (two-arg-s64.2<      nil            (u64.2) (s64.2 s64.2) :cost 1 :encoding :fake-vop)
+   (two-arg-s64.2>=     nil            (u64.2) (s64.2 s64.2) :cost 2 :encoding :fake-vop)
+   (two-arg-s64.2<=     nil            (u64.2) (s64.2 s64.2) :cost 2 :encoding :fake-vop)
+   (s64.2-unpackhi      #:vpunpckhqdq  (s64.2) (s64.2 s64.2) :cost 1)
+   (s64.2-unpacklo      #:vpunpcklqdq  (s64.2) (s64.2 s64.2) :cost 1)
+   (s64.2-movemask      #:vmovmskpd    (u2)    (s64.2)       :cost 1)
+   (s64.2-permute       #:vpermilpd    (s64.2) (s64.2 imm8)  :cost 1)
+   ;; s8.32
+   (s8.32!-from-s8      nil            (s8.32) (s8) :cost 1 :encoding :fake-vop)
+   (s8.32!-from-p128    #:vmovdqu      (s8.32) (p128) :cost 1 :encoding :move :always-translatable nil)
+   (s8.32!-from-p256    #:vmovdqu      (s8.32) (p256) :cost 1 :encoding :move :always-translatable nil)
+   (make-s8.32          nil            (s8.32) (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) :cost 1 :encoding :fake-vop)
+   (s8.32-values        nil            (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) (s8.32) :cost 1 :encoding :fake-vop)
+   (s8.32-broadcast     nil            (s8.32) (s8) :cost 1 :encoding :fake-vop)
+   (s8.16-from-s8.32    #:vextractf128 (s8.16) (s8.32 imm1) :cost 1)
+   (s8.32-insert-s8.16  #:vinsertf128  (s8.32) (s8.32 s8.16 imm1) :cost 1)
+   (s8.32-permute128    #:vperm2f128   (s8.32) (s8.32 s8.32 imm8) :cost 1)
+   ;; s16.16
+   (s16.16!-from-s16    nil            (s16.16) (s16) :cost 1 :encoding :fake-vop)
+   (s16.16!-from-p128   #:vmovdqu      (s16.16) (p128) :cost 1 :encoding :move :always-translatable nil)
+   (s16.16!-from-p256   #:vmovdqu      (s16.16) (p256) :cost 1 :encoding :move :always-translatable nil)
+   (make-s16.16         nil            (s16.16) (s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16) :cost 1 :encoding :fake-vop)
+   (s16.16-values       nil            (s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16) (s16.16) :cost 1 :encoding :fake-vop)
+   (s16.16-broadcast    nil            (s16.16) (s16) :cost 1 :encoding :fake-vop)
+   (s16.8-from-s16.16   #:vextractf128 (s16.8) (s16.16 imm1) :cost 1)
+   (s16.16-insert-s16.8 #:vinsertf128  (s16.16) (s16.16 s16.8 imm1) :cost 1)
+   (s16.16-permute128   #:vperm2f128   (s16.16) (s16.16 s16.16 imm8) :cost 1)
+   ;; s32.8
+   (s32.8!-from-s32     nil            (s32.8) (s32) :cost 1 :encoding :fake-vop)
+   (s32.8!-from-p128    #:vmovdqu      (s32.8) (p128) :cost 1 :encoding :move :always-translatable nil)
+   (s32.8!-from-p256    #:vmovdqu      (s32.8) (p256) :cost 1 :encoding :move :always-translatable nil)
+   (make-s32.8          nil            (s32.8) (s32 s32 s32 s32 s32 s32 s32 s32) :cost 1 :encoding :fake-vop)
+   (s32.8-values        nil            (s32 s32 s32 s32 s32 s32 s32 s32) (s32.8) :cost 1 :encoding :fake-vop)
+   (s32.8-broadcast     nil            (s32.8) (s32) :cost 1 :encoding :fake-vop)
+   (s32.8-from-f32.8    #:vcvtps2dq    (s32.8) (f32.8)       :cost 4)
+   (s32.8-permute       #:vpermilps    (s32.8) (s32.4 imm8) :cost 1)
+   (s32.4-from-s32.8    #:vextractf128 (s32.4) (s32.8 imm1) :cost 1)
+   (s32.8-insert-s32.4  #:vinsertf128  (s32.8) (s32.8 s32.4 imm1) :cost 1)
+   (s32.8-permute128    #:vperm2f128   (s32.8) (s32.8 s32.8 imm8) :cost 1)
+   ;; s64.4
+   (s64.4!-from-s64     nil            (s64.4) (s64) :cost 1 :encoding :fake-vop)
+   (s64.4!-from-p128    #:vmovdqu      (s64.4) (p128) :cost 1 :encoding :move :always-translatable nil)
+   (s64.4!-from-p256    #:vmovdqu      (s64.4) (p256) :cost 1 :encoding :move :always-translatable nil)
+   (make-s64.4          nil            (s64.4) (s64 s64 s64 s64) :cost 1 :encoding :fake-vop)
+   (s64.4-values        nil            (s64 s64 s64 s64) (s64.4) :cost 1 :encoding :fake-vop)
+   (s64.4-broadcast     nil            (s64.4) (s64) :cost 1 :encoding :fake-vop)
+   (s64.4-permute       #:vpermilpd    (s64.4) (s64.4 imm8) :cost 1)
+   (s64.2-from-s64.4    #:vextractf128 (s64.2) (s64.4 imm1) :cost 1)
+   (s64.4-insert-s64.2  #:vinsertf128  (s64.4) (s64.4 s64.2 imm1) :cost 1)
+   (s64.4-permute128    #:vperm2f128   (s64.4) (s64.4 s64.4 imm8) :cost 1))
+  (:loads
+   (u32.4-load-from-string #:vmovdqu u32.4 charvec char-array u32.4-string-ref u32.4-row-major-string-ref)
+   (u32.8-load-from-string #:vmovdqu u32.8 charvec char-array u32.8-string-ref u32.8-row-major-string-ref)
+   (f32-load    #:vmovss  f32    f32vec f32-array f32-aref   f32-row-major-aref)
+   (f64-load    #:vmovsd  f64    f64vec f64-array f64-aref   f64-row-major-aref)
+   (f32.4-load  #:vmovups f32.4  f32vec f32-array f32.4-aref f32.4-row-major-aref)
+   (f64.2-load  #:vmovupd f64.2  f64vec f64-array f64.2-aref f64.2-row-major-aref)
+   (f32.8-load  #:vmovups f32.8  f32vec f32-array f32.8-aref f32.8-row-major-aref)
+   (f64.4-load  #:vmovupd f64.4  f64vec f64-array f64.4-aref f64.4-row-major-aref)
+   (u8.16-load  #:vmovdqu u8.16   u8vec  u8-array u8.16-aref u8.16-row-major-aref)
+   (u16.8-load  #:vmovdqu u16.8  u16vec u16-array u16.8-aref u16.8-row-major-aref)
+   (u32.4-load  #:vmovdqu u32.4  u32vec u32-array u32.4-aref u32.4-row-major-aref)
+   (u64.2-load  #:vmovdqu u64.2  u64vec u64-array u64.2-aref u64.2-row-major-aref)
+   (u8.32-load  #:vmovdqu u8.32   u8vec  u8-array u8.32-aref u8.32-row-major-aref)
+   (u16.16-load #:vmovdqu u16.16 u16vec u16-array u16.16-aref u16.16-row-major-aref)
+   (u32.8-load  #:vmovdqu u32.8  u32vec u32-array u32.8-aref u32.8-row-major-aref)
+   (u64.4-load  #:vmovdqu u64.4  u64vec u64-array u64.4-aref u64.4-row-major-aref)
+   (s8.16-load  #:vmovdqu s8.16   s8vec  s8-array s8.16-aref s8.16-row-major-aref)
+   (s16.8-load  #:vmovdqu s16.8  s16vec s16-array s16.8-aref s16.8-row-major-aref)
+   (s32.4-load  #:vmovdqu s32.4  s32vec s32-array s32.4-aref s32.4-row-major-aref)
+   (s64.2-load  #:vmovdqu s64.2  s64vec s64-array s64.2-aref s64.2-row-major-aref)
+   (s8.32-load  #:vmovdqu s8.32   s8vec  s8-array s8.32-aref s8.32-row-major-aref)
+   (s16.16-load #:vmovdqu s16.16 s16vec s16-array s16.16-aref s16.16-row-major-aref)
+   (s32.8-load  #:vmovdqu s32.8  s32vec s32-array s32.8-aref s32.8-row-major-aref)
+   (s64.4-load  #:vmovdqu s64.4  s64vec s64-array s64.4-aref s64.4-row-major-aref))
+  (:stores
+   (u32.4-store-into-string #:vmovdqu u32.4 charvec char-array u32.4-string-ref u32.4-row-major-string-ref)
+   (u32.8-store-into-string #:vmovdqu u32.8 charvec char-array u32.8-string-ref u32.8-row-major-string-ref)
+   (f32-store   #:vmovss  f32    f32vec f32-array f32-aref   f32-row-major-aref)
+   (f64-store   #:vmovsd  f64    f64vec f64-array f64-aref   f64-row-major-aref)
+   (f32.4-store #:vmovups f32.4  f32vec f32-array f32.4-aref f32.4-row-major-aref)
+   (f64.2-store #:vmovupd f64.2  f64vec f64-array f64.2-aref f64.2-row-major-aref)
+   (f32.8-store #:vmovups f32.8  f32vec f32-array f32.8-aref f32.8-row-major-aref)
+   (f64.4-store #:vmovupd f64.4  f64vec f64-array f64.4-aref f64.4-row-major-aref)
+   (u8.16-store #:vmovdqu u8.16   u8vec  u8-array u8.16-aref u8.16-row-major-aref)
+   (u16.8-store #:vmovdqu u16.8  u16vec u16-array u16.8-aref u16.8-row-major-aref)
+   (u32.4-store #:vmovdqu u32.4  u32vec u32-array u32.4-aref u32.4-row-major-aref)
+   (u64.2-store #:vmovdqu u64.2  u64vec u64-array u64.2-aref u64.2-row-major-aref)
+   (s8.16-store #:vmovdqu s8.16   s8vec  s8-array s8.16-aref s8.16-row-major-aref)
+   (s16.8-store #:vmovdqu s16.8  s16vec s16-array s16.8-aref s16.8-row-major-aref)
+   (s32.4-store #:vmovdqu s32.4  s32vec s32-array s32.4-aref s32.4-row-major-aref)
+   (s64.2-store #:vmovdqu s64.2  s64vec s64-array s64.2-aref s64.2-row-major-aref)
+   (u8.32-store    #:vmovdqu  u8.32   u8vec  u8-array u8.32-aref  u8.32-row-major-aref)
+   (u16.16-store   #:vmovdqu  u16.16 u16vec u16-array u16.16-aref u16.16-row-major-aref)
+   (u32.8-store    #:vmovdqu  u32.8  u32vec u32-array u32.8-aref  u32.8-row-major-aref)
+   (u64.4-store    #:vmovdqu  u64.4  u64vec u64-array u64.4-aref  u64.4-row-major-aref)
+   (s8.32-store    #:vmovdqu  s8.32   s8vec  s8-array s8.32-aref  s8.32-row-major-aref)
+   (s16.16-store   #:vmovdqu  s16.16 s16vec s16-array s16.16-aref s16.16-row-major-aref)
+   (s32.8-store    #:vmovdqu  s32.8  s32vec s32-array s32.8-aref  s32.8-row-major-aref)
+   (s64.4-store    #:vmovdqu  s64.4  s64vec s64-array s64.4-aref  s64.4-row-major-aref)
+   (f32.4-ntstore  #:vmovntps f32.4  f32vec f32-array f32.4-non-temporal-aref  f32.4-non-temporal-row-major-aref)
+   (f64.2-ntstore  #:vmovntpd f64.2  f64vec f64-array f64.2-non-temporal-aref  f64.2-non-temporal-row-major-aref)
+   (f32.8-ntstore  #:vmovntps f32.8  f32vec f32-array f32.8-non-temporal-aref  f32.8-non-temporal-row-major-aref)
+   (f64.4-ntstore  #:vmovntpd f64.4  f64vec f64-array f64.4-non-temporal-aref  f64.4-non-temporal-row-major-aref)
+   (u8.16-ntstore  #:vmovntdq u8.16   u8vec  u8-array u8.16-non-temporal-aref  u8.16-non-temporal-row-major-aref)
+   (u16.8-ntstore  #:vmovntdq u16.8  u16vec u16-array u16.8-non-temporal-aref  u16.8-non-temporal-row-major-aref)
+   (u32.4-ntstore  #:vmovntdq u32.4  u32vec u32-array u32.4-non-temporal-aref  u32.4-non-temporal-row-major-aref)
+   (u64.2-ntstore  #:vmovntdq u64.2  u64vec u64-array u64.2-non-temporal-aref  u64.2-non-temporal-row-major-aref)
+   (s8.16-ntstore  #:vmovntdq s8.16   s8vec  s8-array s8.16-non-temporal-aref  s8.16-non-temporal-row-major-aref)
+   (s16.8-ntstore  #:vmovntdq s16.8  s16vec s16-array s16.8-non-temporal-aref  s16.8-non-temporal-row-major-aref)
+   (s32.4-ntstore  #:vmovntdq s32.4  s32vec s32-array s32.4-non-temporal-aref  s32.4-non-temporal-row-major-aref)
+   (s64.2-ntstore  #:vmovntdq s64.2  s64vec s64-array s64.2-non-temporal-aref  s64.2-non-temporal-row-major-aref)
+   (u8.32-ntstore  #:vmovntdq u8.32   u8vec  u8-array u8.32-non-temporal-aref  u8.32-non-temporal-row-major-aref)
+   (u16.16-ntstore #:vmovntdq u16.16 u16vec u16-array u16.16-non-temporal-aref u16.16-non-temporal-row-major-aref)
+   (u32.8-ntstore  #:vmovntdq u32.8  u32vec u32-array u32.8-non-temporal-aref  u32.8-non-temporal-row-major-aref)
+   (u64.4-ntstore  #:vmovntdq u64.4  u64vec u64-array u64.4-non-temporal-aref  u64.4-non-temporal-row-major-aref)
+   (s8.32-ntstore  #:vmovntdq s8.32   s8vec  s8-array s8.32-non-temporal-aref  s8.32-non-temporal-row-major-aref)
+   (s16.16-ntstore #:vmovntdq s16.16 s16vec s16-array s16.16-non-temporal-aref s16.16-non-temporal-row-major-aref)
+   (s32.8-ntstore  #:vmovntdq s32.8  s32vec s32-array s32.8-non-temporal-aref  s32.8-non-temporal-row-major-aref)
+   (s64.4-ntstore  #:vmovntdq s64.4  s64vec s64-array s64.4-non-temporal-aref  s64.4-non-temporal-row-major-aref))
+  (:associatives
+   (f32-and two-arg-f32-and +f32-true+)
+   (f32-or  two-arg-f32-or  +f32-false+)
+   (f32-xor two-arg-f32-xor +f32-false+)
+   (f32-max two-arg-f32-max nil)
+   (f32-min two-arg-f32-min nil)
+   (f32+    two-arg-f32+ 0f0)
+   (f32*    two-arg-f32* 1f0)
+   (f64-and two-arg-f64-and +f64-true+)
+   (f64-or  two-arg-f64-or  +f64-false+)
+   (f64-xor two-arg-f64-xor +f64-false+)
+   (f64-max two-arg-f64-max nil)
+   (f64-min two-arg-f64-min nil)
+   (f64+    two-arg-f64+ 0d0)
+   (f64*    two-arg-f64* 1d0)
+   (f32.4-and two-arg-f32.4-and +f32-true+)
+   (f32.4-or  two-arg-f32.4-or  +f32-false+)
+   (f32.4-xor two-arg-f32.4-xor +f32-false+)
+   (f32.4-max two-arg-f32.4-max nil)
+   (f32.4-min two-arg-f32.4-min nil)
+   (f32.4+    two-arg-f32.4+ 0f0)
+   (f32.4*    two-arg-f32.4* 1f0)
+   (f64.2-and two-arg-f64.2-and +f64-true+)
+   (f64.2-or  two-arg-f64.2-or  +f64-false+)
+   (f64.2-xor two-arg-f64.2-xor +f64-false+)
+   (f64.2-max two-arg-f64.2-max nil)
+   (f64.2-min two-arg-f64.2-min nil)
+   (f64.2+    two-arg-f64.2+ 0d0)
+   (f64.2*    two-arg-f64.2* 1d0)
+   (f32.8-and two-arg-f32.8-and +f32-true+)
+   (f32.8-or  two-arg-f32.8-or  +f32-false+)
+   (f32.8-xor two-arg-f32.8-xor +f32-false+)
+   (f32.8-max two-arg-f32.8-max nil)
+   (f32.8-min two-arg-f32.8-min nil)
+   (f32.8+    two-arg-f32.8+ 0f0)
+   (f32.8*    two-arg-f32.8* 1f0)
+   (f64.4-and two-arg-f64.4-and +f64-true+)
+   (f64.4-or  two-arg-f64.4-or  +f64-false+)
+   (f64.4-xor two-arg-f64.4-xor +f64-false+)
+   (f64.4-max two-arg-f64.4-max nil)
+   (f64.4-min two-arg-f64.4-min nil)
+   (f64.4+    two-arg-f64.4+ 0d0)
+   (f64.4*    two-arg-f64.4* 1d0)
+   (u8.16-and two-arg-u8.16-and +u8-true+)
+   (u8.16-or  two-arg-u8.16-or  +u8-false+)
+   (u8.16-xor two-arg-u8.16-xor +u8-false+)
+   (u8.16+    two-arg-u8.16+ 0)
+   (u16.8-and two-arg-u16.8-and +u16-true+)
+   (u16.8-or  two-arg-u16.8-or  +u16-false+)
+   (u16.8-xor two-arg-u16.8-xor +u16-false+)
+   (u16.8+    two-arg-u16.8+ 0)
+   (u32.4-and two-arg-u32.4-and +u32-true+)
+   (u32.4-or  two-arg-u32.4-or  +u32-false+)
+   (u32.4-xor two-arg-u32.4-xor +u32-false+)
+   (u32.4+    two-arg-u32.4+ 0)
+   (u64.2-and two-arg-u64.2-and +u64-true+)
+   (u64.2-or  two-arg-u64.2-or  +u64-false+)
+   (u64.2-xor two-arg-u64.2-xor +u64-false+)
+   (u64.2+    two-arg-u64.2+ 0)
+   (s8.16-and two-arg-s8.16-and +s8-true+)
+   (s8.16-or  two-arg-s8.16-or  +s8-false+)
+   (s8.16-xor two-arg-s8.16-xor +s8-false+)
+   (s8.16+    two-arg-s8.16+ 0)
+   (s16.8-and two-arg-s16.8-and +s16-true+)
+   (s16.8-or  two-arg-s16.8-or  +s16-false+)
+   (s16.8-xor two-arg-s16.8-xor +s16-false+)
+   (s16.8+    two-arg-s16.8+ 0)
+   (s16.8-mullo two-arg-s16.8-mullo 1)
+   (s32.4-and two-arg-s32.4-and +s32-true+)
+   (s32.4-or  two-arg-s32.4-or  +s32-false+)
+   (s32.4-xor two-arg-s32.4-xor +s32-false+)
+   (s32.4+    two-arg-s32.4+ 0)
+   (s32.4-mullo two-arg-s32.4-mullo 1)
+   (s64.2-and two-arg-s64.2-and +s64-true+)
+   (s64.2-or  two-arg-s64.2-or  +s64-false+)
+   (s64.2-xor two-arg-s64.2-xor +s64-false+)
+   (s64.2+    two-arg-s64.2+ 0)
+   ;; The next two functions are only required for implementing
+   ;; floating-point comparisons and unequals.
+   (u32.8-and two-arg-u32.8-and +u32-true+)
+   (u64.4-and two-arg-u64.4-and +u64-true+))
+  (:comparisons
+   (f32=  two-arg-f32=  u32-and +u32-true+)
+   (f32<  two-arg-f32<  u32-and +u32-true+)
+   (f32<= two-arg-f32<= u32-and +u32-true+)
+   (f32>  two-arg-f32>  u32-and +u32-true+)
+   (f32>= two-arg-f32>= u32-and +u32-true+)
+   (f64=  two-arg-f64=  u64-and +u64-true+)
+   (f64<  two-arg-f64<  u64-and +u64-true+)
+   (f64<= two-arg-f64<= u64-and +u64-true+)
+   (f64>  two-arg-f64>  u64-and +u64-true+)
+   (f64>= two-arg-f64>= u64-and +u64-true+)
+   (f32.4=  two-arg-f32.4=  u32.4-and +u32-true+)
+   (f32.4<  two-arg-f32.4<  u32.4-and +u32-true+)
+   (f32.4<= two-arg-f32.4<= u32.4-and +u32-true+)
+   (f32.4>  two-arg-f32.4>  u32.4-and +u32-true+)
+   (f32.4>= two-arg-f32.4>= u32.4-and +u32-true+)
+   (f64.2=  two-arg-f64.2=  u64.2-and +u64-true+)
+   (f64.2<  two-arg-f64.2<  u64.2-and +u64-true+)
+   (f64.2<= two-arg-f64.2<= u64.2-and +u64-true+)
+   (f64.2>  two-arg-f64.2>  u64.2-and +u64-true+)
+   (f64.2>= two-arg-f64.2>= u64.2-and +u64-true+)
+   (f32.8=  two-arg-f32.8=  u32.8-and +u32-true+)
+   (f32.8<  two-arg-f32.8<  u32.8-and +u32-true+)
+   (f32.8<= two-arg-f32.8<= u32.8-and +u32-true+)
+   (f32.8>  two-arg-f32.8>  u32.8-and +u32-true+)
+   (f32.8>= two-arg-f32.8>= u32.8-and +u32-true+)
+   (f64.4=  two-arg-f64.4=  u64.4-and +u64-true+)
+   (f64.4<  two-arg-f64.4<  u64.4-and +u64-true+)
+   (f64.4<= two-arg-f64.4<= u64.4-and +u64-true+)
+   (f64.4>  two-arg-f64.4>  u64.4-and +u64-true+)
+   (f64.4>= two-arg-f64.4>= u64.4-and +u64-true+)
+   (u8.16=  two-arg-u8.16=  u8.16-and +u8-true+)
+   (u8.16<  two-arg-u8.16<  u8.16-and +u8-true+)
+   (u8.16<= two-arg-u8.16<= u8.16-and +u8-true+)
+   (u8.16>  two-arg-u8.16>  u8.16-and +u8-true+)
+   (u8.16>= two-arg-u8.16>= u8.16-and +u8-true+)
+   (u16.8=  two-arg-u16.8=  u16.8-and +u16-true+)
+   (u16.8<  two-arg-u16.8<  u16.8-and +u16-true+)
+   (u16.8<= two-arg-u16.8<= u16.8-and +u16-true+)
+   (u16.8>  two-arg-u16.8>  u16.8-and +u16-true+)
+   (u16.8>= two-arg-u16.8>= u16.8-and +u16-true+)
+   (u32.4=  two-arg-u32.4=  u32.4-and +u32-true+)
+   (u32.4<  two-arg-u32.4<  u32.4-and +u32-true+)
+   (u32.4<= two-arg-u32.4<= u32.4-and +u32-true+)
+   (u32.4>  two-arg-u32.4>  u32.4-and +u32-true+)
+   (u32.4>= two-arg-u32.4>= u32.4-and +u32-true+)
+   (u64.2=  two-arg-u64.2=  u64.2-and +u64-true+)
+   (u64.2<  two-arg-u64.2<  u64.2-and +u64-true+)
+   (u64.2<= two-arg-u64.2<= u64.2-and +u64-true+)
+   (u64.2>  two-arg-u64.2>  u64.2-and +u64-true+)
+   (u64.2>= two-arg-u64.2>= u64.2-and +u64-true+)
+   (s8.16=  two-arg-s8.16=  u8.16-and +u8-true+)
+   (s8.16<  two-arg-s8.16<  u8.16-and +u8-true+)
+   (s8.16<= two-arg-s8.16<= u8.16-and +u8-true+)
+   (s8.16>  two-arg-s8.16>  u8.16-and +u8-true+)
+   (s8.16>= two-arg-s8.16>= u8.16-and +u8-true+)
+   (s16.8=  two-arg-s16.8=  u16.8-and +u16-true+)
+   (s16.8<  two-arg-s16.8<  u16.8-and +u16-true+)
+   (s16.8<= two-arg-s16.8<= u16.8-and +u16-true+)
+   (s16.8>  two-arg-s16.8>  u16.8-and +u16-true+)
+   (s16.8>= two-arg-s16.8>= u16.8-and +u16-true+)
+   (s32.4=  two-arg-s32.4=  u32.4-and +u32-true+)
+   (s32.4<  two-arg-s32.4<  u32.4-and +u32-true+)
+   (s32.4<= two-arg-s32.4<= u32.4-and +u32-true+)
+   (s32.4>  two-arg-s32.4>  u32.4-and +u32-true+)
+   (s32.4>= two-arg-s32.4>= u32.4-and +u32-true+)
+   (s64.2=  two-arg-s64.2=  u64.2-and +u64-true+)
+   (s64.2<  two-arg-s64.2<  u64.2-and +u64-true+)
+   (s64.2<= two-arg-s64.2<= u64.2-and +u64-true+)
+   (s64.2>  two-arg-s64.2>  u64.2-and +u64-true+)
+   (s64.2>= two-arg-s64.2>= u64.2-and +u64-true+))
+  (:ifs
+   (f32.4-if f32.4-blend)
+   (f32.8-if f32.8-blend)
+   (f64.2-if f64.2-blend)
+   (f64.4-if f64.4-blend)
+   (u8.16-if u8.16-blend)
+   (u16.8-if u16.8-blend)
+   (u32.4-if u32.4-blend)
+   (u64.2-if u64.2-blend)
+   (s8.16-if s8.16-blend)
+   (s16.8-if s16.8-blend)
+   (s32.4-if s32.4-blend)
+   (s64.2-if s64.2-blend))
+  (:reducers
+   (f32- two-arg-f32- 0f0)
+   (f32/ two-arg-f32/ 1f0)
+   (f64- two-arg-f64- 0d0)
+   (f64/ two-arg-f64/ 1d0)
+   (f32.4- two-arg-f32.4- 0f0)
+   (f32.4/ two-arg-f32.4/ 1f0)
+   (f64.2- two-arg-f64.2- 0d0)
+   (f64.2/ two-arg-f64.2/ 1d0)
+   (f32.8- two-arg-f32.8- 0f0)
+   (f32.8/ two-arg-f32.8/ 1f0)
+   (f64.4- two-arg-f64.4- 0d0)
+   (f64.4/ two-arg-f64.4/ 1d0)
+   (u8.16- two-arg-u8.16- 0)
+   (u16.8- two-arg-u16.8- 0)
+   (u32.4- two-arg-u32.4- 0)
+   (u64.2- two-arg-u64.2- 0)
+   (s8.16- two-arg-s8.16- 0)
+   (s16.8- two-arg-s16.8- 0)
+   (s32.4- two-arg-s32.4- 0)
+   (s64.2- two-arg-s64.2- 0))
+  (:unequals
+   (f32/= two-arg-f32/= u32-and +u32-true+)
+   (f64/= two-arg-f64/= u64-and +u64-true+)
+   (f32.4/= two-arg-f32.4/= u32.4-and +u32-true+)
+   (f64.2/= two-arg-f64.2/= u64.2-and +u64-true+)
+   (f32.8/= two-arg-f32.8/= u32.8-and +u32-true+)
+   (f64.4/= two-arg-f64.4/= u64.4-and +u64-true+)
+   (u8.16/= two-arg-u8.16/= u8.16-and +u8-true+)
+   (u16.8/= two-arg-u16.8/= u16.8-and +u16-true+)
+   (u32.4/= two-arg-u32.4/= u32.4-and +u32-true+)
+   (u64.2/= two-arg-u64.2/= u64.2-and +u64-true+)
+   (s8.16/= two-arg-s8.16/= u8.16-and +u8-true+)
+   (s16.8/= two-arg-s16.8/= u16.8-and +u16-true+)
+   (s32.4/= two-arg-s32.4/= u32.4-and +u32-true+)
+   (s64.2/= two-arg-s64.2/= u64.2-and +u64-true+)))

contrib/sb-simd/code/instruction-sets/avx2.lisp

@@ -0,0 +1,523 @@
+(in-package #:sb-simd-avx2)
+
+(define-instruction-set :avx2
+  (:include :avx)
+  (:test (avx2-supported-p))
+  (:simd-packs
+   ;; You may wonder why we define these SIMD packs twice both for AVX and
+   ;; AVX2.  The reason is that each of these instruction sets provides its
+   ;; own version of the constructor for integer SIMD packs.  And since the
+   ;; constructor has the same name as the pack, AVX2 cannot inherit them
+   ;; from AVX.  Luckily, defining SIMD records is cheap, so we can simply
+   ;; define them again with a different name (or rather, with the same
+   ;; name but in a different package).
+   (u8.32  u8  256 #:simd-pack-256-ub8  (#:int-avx2-reg))
+   (u16.16 u16 256 #:simd-pack-256-ub16 (#:int-avx2-reg))
+   (u32.8  u32 256 #:simd-pack-256-ub32 (#:int-avx2-reg))
+   (u64.4  u64 256 #:simd-pack-256-ub64 (#:int-avx2-reg))
+   (s8.32  s8  256 #:simd-pack-256-sb8  (#:int-avx2-reg))
+   (s16.16 s16 256 #:simd-pack-256-sb16 (#:int-avx2-reg))
+   (s32.8  s32 256 #:simd-pack-256-sb32 (#:int-avx2-reg))
+   (s64.4  s64 256 #:simd-pack-256-sb64 (#:int-avx2-reg)))
+  (:simd-casts
+   (u8.32 u8.32-broadcast)
+   (u16.16 u16.16-broadcast)
+   (u32.8 u32.8-broadcast)
+   (u64.4 u64.4-broadcast)
+   (s8.32 s8.32-broadcast)
+   (s16.16 s16.16-broadcast)
+   (s32.8 s32.8-broadcast)
+   (s64.4 s64.4-broadcast))
+  (:reinterpret-casts
+   (u8.16! sb-simd-avx::u8.16!-from-u8  sb-simd-avx::u8.16!-from-p128 u8.16!-from-p256)
+   (u16.8! sb-simd-avx::u16.8!-from-u16 sb-simd-avx::u16.8!-from-p128 u16.8!-from-p256)
+   (u32.4! sb-simd-avx::u32.4!-from-u32 sb-simd-avx::u32.4!-from-p128 u32.4!-from-p256)
+   (u64.2! sb-simd-avx::u64.2!-from-u64 sb-simd-avx::u64.2!-from-p128 u64.2!-from-p256)
+   (s8.16! sb-simd-avx::s8.16!-from-s8  sb-simd-avx::s8.16!-from-p128 s8.16!-from-p256)
+   (s16.8! sb-simd-avx::s16.8!-from-s16 sb-simd-avx::s16.8!-from-p128 s16.8!-from-p256)
+   (s32.4! sb-simd-avx::s32.4!-from-s32 sb-simd-avx::s32.4!-from-p128 s32.4!-from-p256)
+   (s64.2! sb-simd-avx::s64.2!-from-s64 sb-simd-avx::s64.2!-from-p128 s64.2!-from-p256))
+  (:instructions
+   ;; f64.4
+   (f64.4-permute4x64     #:vpermpd       (f64.4) (f64.4 imm8)    :cost 1)
+   (f64.4-reverse         nil             (f64.4) (f64.4)         :cost 2 :encoding :fake-vop)
+   ;; u8.16
+   (u8.16!-from-p256      #:vextracti128  (u8.16) (p256)          :cost 1 :suffix '(0) :always-translatable nil)
+   (u8.16-broadcast       nil             (u8.16) (u8)            :cost 1 :encoding :fake-vop)
+   (u8.16-broadcastvec    #:vpbroadcastb  (u8.16) (u8.16)         :cost 1)
+   ;; u16.8
+   (u16.8!-from-p256      #:vextracti128  (u16.8) (p256)          :cost 1 :suffix '(0) :always-translatable nil)
+   (u16.8-broadcast       nil             (u16.16) (u16)          :cost 1 :encoding :fake-vop)
+   (u16.8-broadcastvec    #:vpbroadcastw  (u16.16) (u16.8)        :cost 1)
+   ;; u32.4
+   (u32.4!-from-p256      #:vextracti128  (u32.4) (p256)          :cost 1 :suffix '(0) :always-translatable nil)
+   (u32.4-broadcast       nil             (u32.4) (u32)           :cost 1 :encoding :fake-vop)
+   (u32.4-broadcastvec    #:vpbroadcastd  (u32.4) (u32.4)         :cost 1)
+   (u32.4-shiftl          #:vpsllvd       (u32.4) (u32.4 u32.4)   :cost 1)
+   (u32.4-shiftr          #:vpsrlvd       (u32.4) (u32.4 u32.4)   :cost 1)
+   ;; u64.2
+   (u64.2!-from-p256      #:vextracti128  (u64.2) (p256)          :cost 1 :suffix '(0) :always-translatable nil)
+   (u64.2-broadcast       nil             (u64.2) (u64)           :cost 1 :encoding :fake-vop)
+   (u64.2-broadcastvec    #:vpbroadcastq  (u64.2) (u64.2)         :cost 1)
+   (u64.2-shiftl          #:vpsllvd       (u64.2) (u64.2 u64.2)   :cost 1)
+   (u64.2-shiftr          #:vpsrlvd       (u64.2) (u64.2 u64.2)   :cost 1)
+   ;; s8.16
+   (s8.16!-from-p256      #:vextracti128  (s8.16) (p256)          :cost 1 :suffix '(0) :always-translatable nil)
+   (s8.16-broadcast       nil             (s8.16) (s8)            :cost 1 :encoding :fake-vop)
+   (s8.16-broadcastvec    #:vpbroadcastb  (s8.16) (s8.16)         :cost 1)
+   ;; s16.8
+   (s16.8!-from-p256      #:vextracti128  (s16.8) (p256)          :cost 1 :suffix '(1) :always-translatable nil)
+   (s16.8-broadcast       nil             (s16.16) (s16)          :cost 1 :encoding :fake-vop)
+   (s16.8-broadcastvec    #:vpbroadcastw  (s16.16) (s16.8)        :cost 1)
+   ;; s32.4
+   (s32.4!-from-p256      #:vextracti128  (s32.4) (p256)          :cost 1 :suffix '(0) :always-translatable nil)
+   (s32.4-broadcast       nil             (s32.4) (s32)           :cost 1 :encoding :fake-vop)
+   (s32.4-broadcastvec    #:vpbroadcastd  (s32.4) (s32.4)         :cost 1)
+   (s32.4-shiftl          #:vpsllvq       (s32.4) (s32.4 s32.4)   :cost 1)
+   (s32.4-shiftr          #:vpsrlvq       (s32.4) (s32.4 s32.4)   :cost 1)
+   ;; s64.2
+   (s64.2!-from-p256      #:vextracti128  (s64.2) (p256)          :cost 1 :suffix '(0) :always-translatable nil)
+   (s64.2-broadcast       nil             (s64.2) (s64)           :cost 1 :encoding :fake-vop)
+   (s64.2-broadcastvec    #:vpbroadcastq  (s64.2) (s64.2)         :cost 1)
+   (s64.2-shiftl          #:vpsllvq       (s64.2) (s64.2 s64.2)   :cost 1)
+   (s64.2-shiftr          #:vpsrlvq       (s64.2) (s64.2 s64.2)   :cost 1)
+   ;; u8.32
+   (make-u8.32            nil             (u8.32) (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) :cost 1 :encoding :fake-vop)
+   (u8.32-values          nil             (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) (u8.32) :cost 1 :encoding :fake-vop)
+   (u8.32-broadcast       nil             (u8.32) (u8)            :cost 1 :encoding :fake-vop)
+   (u8.32-broadcastvec    #:vpbroadcastb  (u8.32) (u8.32)         :cost 1)
+   (u8.32-blend           #:vpblendvb     (u8.32) (u8.32 u8.32 u8.32) :cost 1)
+   (two-arg-u8.32-and     #:vpand         (u8.32) (u8.32 u8.32)   :cost 1 :associative t)
+   (two-arg-u8.32-or      #:vpor          (u8.32) (u8.32 u8.32)   :cost 1 :associative t)
+   (two-arg-u8.32-xor     #:vpxor         (u8.32) (u8.32 u8.32)   :cost 1 :associative t)
+   (u8.32-andc1           #:vpandn        (u8.32) (u8.32 u8.32)   :cost 1)
+   (u8.32-not             nil             (u8.32) (u8.32)         :cost 1 :encoding :fake-vop)
+   (two-arg-u8.32-max     #:vpmaxub       (u8.32) (u8.32 u8.32)   :cost 2 :associative t)
+   (two-arg-u8.32-min     #:vpminub       (u8.32) (u8.32 u8.32)   :cost 2 :associative t)
+   (two-arg-u8.32+        #:vpaddb        (u8.32) (u8.32 u8.32)   :cost 2 :associative t)
+   (two-arg-u8.32-        #:vpsubb        (u8.32) (u8.32 u8.32)   :cost 2)
+   (two-arg-u8.32=        #:vpcmpeqb      (u8.32) (u8.32 u8.32)   :cost 1 :associative t)
+   (two-arg-u8.32/=       nil             (u8.32) (u8.32 u8.32)   :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u8.32>~       #:vpcmpgtb      (u8.32) (u8.32 u8.32)   :cost 1)
+   (two-arg-u8.32>        nil             (u8.32) (u8.32 u8.32)   :cost 1 :encoding :fake-vop)
+   (two-arg-u8.32<        nil             (u8.32) (u8.32 u8.32)   :cost 1 :encoding :fake-vop)
+   (two-arg-u8.32>=       nil             (u8.32) (u8.32 u8.32)   :cost 2 :encoding :fake-vop)
+   (two-arg-u8.32<=       nil             (u8.32) (u8.32 u8.32)   :cost 2 :encoding :fake-vop)
+   (u8.32-avg             #:vpavgb        (u8.32) (u8.32 u8.32)   :cost 2)
+   (u8.32-packus          #:vpackuswb     (u8.32) (u16.16 u16.16) :cost 2)
+   (u8.32-unpackhi        #:vpunpckhbw    (u8.32) (u8.32 u8.32)   :cost 1)
+   (u8.32-unpacklo        #:vpunpcklbw    (u8.32) (u8.32 u8.32)   :cost 1)
+   (u8.32-movemask        #:vpmovmskb     (u32)   (u8.32)         :cost 1)
+   (u8.32-shuffle         #:vpshufb       (u8.32) (u8.32 u8.32)   :cost 1)
+   (u8.32-insert-u8.16    #:vinserti128   (u8.32) (u8.32 u8.16 imm8) :cost 1)
+   (u8.16-from-u8.32      #:vextracti128  (u8.16) (u8.32 imm1) :cost 1)
+   (u8.32-permute128      #:vperm2i128    (u8.32) (u8.32 u8.32 imm8) :cost 1)
+   ;; u16.16
+   (make-u16.16           nil            (u16.16) (u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16) :cost 1 :encoding :fake-vop)
+   (u16.16-values         nil            (u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16 u16) (u16.16) :cost 1 :encoding :fake-vop)
+   (u16.16-broadcast      nil            (u16.16) (u16)           :cost 1 :encoding :fake-vop)
+   (u16.16-broadcastvec   #:vpbroadcastw (u16.16) (u16.16)        :cost 1)
+   (u16.16-blend          #:vpblendvb    (u16.16) (u16.16 u16.16 u16.16) :cost 1)
+   (u16.16-from-u8.16     #:vpmovzxbw    (u16.16) (u8.16)         :cost 5)
+   (two-arg-u16.16-and    #:vpand        (u16.16) (u16.16 u16.16) :cost 1 :associative t)
+   (two-arg-u16.16-or     #:vpor         (u16.16) (u16.16 u16.16) :cost 1 :associative t)
+   (two-arg-u16.16-xor    #:vpxor        (u16.16) (u16.16 u16.16) :cost 1 :associative t)
+   (u16.16-andc1          #:vpandn       (u16.16) (u16.16 u16.16) :cost 1)
+   (u16.16-not            nil            (u16.16) (u16.16)        :cost 1 :encoding :fake-vop)
+   (two-arg-u16.16-max    #:vpmaxuw      (u16.16) (u16.16 u16.16) :cost 2 :associative t)
+   (two-arg-u16.16-min    #:vpminuw      (u16.16) (u16.16 u16.16) :cost 2 :associative t)
+   (two-arg-u16.16+       #:vpaddw       (u16.16) (u16.16 u16.16) :cost 2 :associative t)
+   (two-arg-u16.16-       #:vpsubw       (u16.16) (u16.16 u16.16) :cost 2)
+   (two-arg-u16.16=       #:vpcmpeqw     (u16.16) (u16.16 u16.16) :cost 1 :associative t)
+   (two-arg-u16.16/=      nil            (u16.16) (u16.16 u16.16) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u16.16>~      #:vpcmpgtw     (u16.16) (u16.16 u16.16) :cost 1)
+   (two-arg-u16.16>       nil            (u16.16) (u16.16 u16.16) :cost 1 :encoding :fake-vop)
+   (two-arg-u16.16<       nil            (u16.16) (u16.16 u16.16) :cost 1 :encoding :fake-vop)
+   (two-arg-u16.16>=      nil            (u16.16) (u16.16 u16.16) :cost 2 :encoding :fake-vop)
+   (two-arg-u16.16<=      nil            (u16.16) (u16.16 u16.16) :cost 2 :encoding :fake-vop)
+   (u16.16-shiftl         #:vpsllw       (u16.16) (u16.16 u16.8)  :cost 2)
+   (u16.16-shiftr         #:vpsrlw       (u16.16) (u16.16 u16.8)  :cost 2)
+   (u16.16-avg            #:vpavgw       (u16.16) (u16.16 u16.16) :cost 1)
+   (u16.16-packus         #:vpackusdw    (u16.16) (u32.8 u32.8)   :cost 1)
+   (u16.16-unpackhi       #:vpunpckhwd   (u16.16) (u16.16 u16.16) :cost 1)
+   (u16.16-unpacklo       #:vpunpcklwd   (u16.16) (u16.16 u16.16) :cost 1)
+   (u16.16-movemask       nil            (u16)    (u16.16)        :cost 1 :encoding :fake-vop)
+   (u16.16-shufflehi      #:vpshufhw     (u16.16) (u16.16 imm8)   :cost 1)
+   (u16.16-shufflelo      #:vpshuflw     (u16.16) (u16.16 imm8)   :cost 1)
+   (u16.8-from-u16.16     #:vextracti128 (u16.8)  (u16.16 imm1)   :cost 1)
+   (u16.16-insert-u16.8   #:vinserti128  (u16.16) (u16.16 u16.8 imm1) :cost 1)
+   (u16.16-permute128     #:vperm2i128   (u16.16) (u16.16 u16.16 imm8) :cost 1)
+   ;; u32.8
+   (make-u32.8            nil            (u32.8) (u32 u32 u32 u32 u32 u32 u32 u32) :cost 1 :encoding :fake-vop)
+   (u32.8-values          nil            (u32 u32 u32 u32 u32 u32 u32 u32) (u32.8) :cost 1 :encoding :fake-vop)
+   (u32.8-broadcast       nil            (u32.8) (u32)         :cost 1 :encoding :fake-vop)
+   (u32.8-broadcastvec    #:vpbroadcastd (u32.8) (u32.8)       :cost 1)
+   (u32.8-blend           #:vpblendvb    (u32.8) (u32.8 u32.8 u32.8) :cost 1)
+   (u32.8-from-u16.8      #:vpmovzxwd    (u32.8) (u16.8)       :cost 5)
+   (u32.8-from-u8.16      #:vpmovzxbd    (u32.8) (u8.16)       :cost 5)
+   (two-arg-u32.8-and     #:vpand        (u32.8) (u32.8 u32.8) :cost 1 :associative t)
+   (two-arg-u32.8-or      #:vpor         (u32.8) (u32.8 u32.8) :cost 1 :associative t)
+   (two-arg-u32.8-xor     #:vpxor        (u32.8) (u32.8 u32.8) :cost 1 :associative t)
+   (u32.8-andc1           #:vpandn       (u32.8) (u32.8 u32.8) :cost 1)
+   (u32.8-not             nil            (u32.8) (u32.8)       :cost 1 :encoding :fake-vop)
+   (two-arg-u32.8-max     #:vpmaxud      (u32.8) (u32.8 u32.8) :cost 2 :associative t)
+   (two-arg-u32.8-min     #:vpminud      (u32.8) (u32.8 u32.8) :cost 2 :associative t)
+   (two-arg-u32.8+        #:vpaddd       (u32.8) (u32.8 u32.8) :cost 2 :associative t)
+   (two-arg-u32.8-        #:vpsubd       (u32.8) (u32.8 u32.8) :cost 2)
+   (two-arg-u32.8=        #:vpcmpeqd     (u32.8) (u32.8 u32.8) :cost 1 :associative t)
+   (two-arg-u32.8/=       nil            (u32.8) (u32.8 u32.8) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u32.8>~       #:vpcmpgtd     (u32.8) (u32.8 u32.8) :cost 1)
+   (two-arg-u32.8>        nil            (u32.8) (u32.8 u32.8) :cost 1 :encoding :fake-vop)
+   (two-arg-u32.8<        nil            (u32.8) (u32.8 u32.8) :cost 1 :encoding :fake-vop)
+   (two-arg-u32.8>=       nil            (u32.8) (u32.8 u32.8) :cost 2 :encoding :fake-vop)
+   (two-arg-u32.8<=       nil            (u32.8) (u32.8 u32.8) :cost 2 :encoding :fake-vop)
+   (u32.8-shiftl          #:vpsllvd      (u32.8) (u32.8 u32.8) :cost 1)
+   (u32.8-shiftr          #:vpsrlvd      (u32.8) (u32.8 u32.8) :cost 1)
+   (u32.8-unpackhi        #:vpunpckhdq   (u32.8) (u32.8 u32.8) :cost 1)
+   (u32.8-unpacklo        #:vpunpckldq   (u32.8) (u32.8 u32.8) :cost 1)
+   (u32.8-movemask        #:vmovmskps    (u8)    (u32.8)       :cost 1)
+   (u32.4-from-u32.8      #:vextracti128 (u32.4) (u32.8 imm1)  :cost 1)
+   (u32.8-insert-u32.4    #:vinserti128  (u32.8) (u32.8 u32.4 imm1) :cost 1)
+   (u32.8-permute128      #:vperm2i128   (u32.8) (u32.8 u32.8 imm8) :cost 1)
+   ;; u64.4
+   (make-u64.4            nil            (u64.4) (u64 u64 u64 u64) :cost 1 :encoding :fake-vop)
+   (u64.4-values          nil            (u64 u64 u64 u64) (u64.4) :cost 1 :encoding :fake-vop)
+   (u64.4-broadcast       nil            (u64.4) (u64)         :cost 1 :encoding :fake-vop)
+   (u64.4-broadcastvec    #:vpbroadcastq (u64.4) (u64.4)       :cost 1)
+   (u64.4-blend           #:vpblendvb    (u64.4) (u64.4 u64.4 u64.4) :cost 1)
+   (u64.4-from-u16.8      #:vpmovzxwq    (u64.4) (u16.8)       :cost 5)
+   (u64.4-from-u32.4      #:vpmovzxdq    (u64.4) (u32.4)       :cost 5)
+   (u64.4-from-u8.16      #:vpmovzxbq    (u64.4) (u8.16)       :cost 5)
+   (two-arg-u64.4-and     #:vpand        (u64.4) (u64.4 u64.4) :cost 1 :associative t)
+   (two-arg-u64.4-or      #:vpor         (u64.4) (u64.4 u64.4) :cost 1 :associative t)
+   (two-arg-u64.4-xor     #:vpxor        (u64.4) (u64.4 u64.4) :cost 1 :associative t)
+   (u64.4-andc1           #:vpandn       (u64.4) (u64.4 u64.4) :cost 1)
+   (u64.4-not             nil            (u64.4) (u64.4)       :cost 1 :encoding :fake-vop)
+   (two-arg-u64.4+        #:vpaddq       (u64.4) (u64.4 u64.4) :cost 2 :associative t)
+   (two-arg-u64.4-        #:vpsubq       (u64.4) (u64.4 u64.4) :cost 2)
+   (two-arg-u64.4-mul     #:vpmuludq     (u64.4) (u64.4 u64.4) :cost 1 :associative t)
+   (two-arg-u64.4=        #:vpcmpeqq     (u64.4) (u64.4 u64.4) :cost 1 :associative t)
+   (two-arg-u64.4/=       nil            (u64.4) (u64.4 u64.4) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u64.4>~       #:vpcmpgtq     (u64.4) (u64.4 u64.4) :cost 1)
+   (two-arg-u64.4>        nil            (u64.4) (u64.4 u64.4) :cost 1 :encoding :fake-vop)
+   (two-arg-u64.4<        nil            (u64.4) (u64.4 u64.4) :cost 1 :encoding :fake-vop)
+   (two-arg-u64.4>=       nil            (u64.4) (u64.4 u64.4) :cost 2 :encoding :fake-vop)
+   (two-arg-u64.4<=       nil            (u64.4) (u64.4 u64.4) :cost 2 :encoding :fake-vop)
+   (u64.4-shiftl          #:vpsllvq      (u64.4) (u64.4 u64.4) :cost 1)
+   (u64.4-shiftr          #:vpsrlvq      (u64.4) (u64.4 u64.4) :cost 1)
+   (u64.4-unpackhi        #:vpunpckhqdq  (u64.4) (u64.4 u64.4) :cost 1)
+   (u64.4-unpacklo        #:vpunpcklqdq  (u64.4) (u64.4 u64.4) :cost 1)
+   (u64.4-movemask        #:vmovmskpd    (u4)    (u64.4)       :cost 1)
+   (u64.2-from-u64.4      #:vextracti128 (u64.2) (u64.4 imm1)  :cost 1)
+   (u64.4-insert-u64.2    #:vinserti128  (u64.4) (u64.4 u64.2 imm1) :cost 1)
+   (u64.4-permute128      #:vperm2i128   (u64.4) (u64.4 u64.4 imm8) :cost 1)
+   ;; s8.32
+   (make-s8.32            nil            (s8.32) (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) :cost 1 :encoding :fake-vop)
+   (s8.32-values          nil            (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) (s8.32) :cost 1 :encoding :fake-vop)
+   (s8.32-broadcast       nil            (s8.32) (s8)             :cost 1 :encoding :fake-vop)
+   (s8.32-broadcastvec    #:vpbroadcastb (s8.32) (s8.32)          :cost 1)
+   (s8.32-blend           #:vpblendvb    (s8.32) (s8.32 s8.32 u8.32) :cost 1)
+   (two-arg-s8.32-and     #:vpand        (s8.32) (s8.32 s8.32)    :cost 1 :associative t)
+   (two-arg-s8.32-or      #:vpor         (s8.32) (s8.32 s8.32)    :cost 1 :associative t)
+   (two-arg-s8.32-xor     #:vpxor        (s8.32) (s8.32 s8.32)    :cost 1 :associative t)
+   (s8.32-andc1           #:vpandn       (s8.32) (s8.32 s8.32)    :cost 1)
+   (s8.32-not             nil            (s8.32) (s8.32)          :cost 1 :encoding :fake-vop)
+   (two-arg-s8.32-max     #:vpmaxsb      (s8.32) (s8.32 s8.32)    :cost 2 :associative t)
+   (two-arg-s8.32-min     #:vpminsb      (s8.32) (s8.32 s8.32)    :cost 2 :associative t)
+   (two-arg-s8.32+        #:vpaddb       (s8.32) (s8.32 s8.32)    :cost 2 :associative t)
+   (two-arg-s8.32-        #:vpsubb       (s8.32) (s8.32 s8.32)    :cost 2)
+   (two-arg-s8.32=        #:vpcmpeqb     (u8.32) (s8.32 s8.32)    :cost 1 :associative t)
+   (two-arg-s8.32/=       nil            (u8.32) (s8.32 s8.32)    :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s8.32>        #:vpcmpgtb     (u8.32) (s8.32 s8.32)    :cost 1)
+   (two-arg-s8.32<        nil            (u8.32) (s8.32 s8.32)    :cost 1 :encoding :fake-vop)
+   (two-arg-s8.32>=       nil            (u8.32) (s8.32 s8.32)    :cost 2 :encoding :fake-vop)
+   (two-arg-s8.32<=       nil            (u8.32) (s8.32 s8.32)    :cost 2 :encoding :fake-vop)
+   (s8.32-abs             #:vpabsb       (s8.32) (s8.32)          :cost 2)
+   (s8.32-packs           #:vpacksswb    (s8.32) (s16.16 s16.16)  :cost 2)
+   (s8.32-unpackhi        #:vpunpckhbw   (s8.32) (s8.32 s8.32)    :cost 1)
+   (s8.32-unpacklo        #:vpunpcklbw   (s8.32) (s8.32 s8.32)    :cost 1)
+   (s8.32-movemask        #:vpmovmskb    (u32)   (s8.32)          :cost 1)
+   (s8.32-shuffle         #:vpshufb      (s8.32) (s8.32 u8.32)    :cost 1)
+   (s8.32-sign            #:vpsignb      (s8.32) (s8.32 s8.32)    :cost 1)
+   (s8.32-broadcastvec    #:vpbroadcastb (s8.32) (s8.32)          :cost 1)
+   (s8.16-from-s8.32      #:vextracti128 (s8.16) (s8.32 imm1)     :cost 1)
+   (s8.32-insert-s8.16    #:vinserti128  (s8.32) (s8.32 s8.16 imm1) :cost 1)
+   (s8.32-permute128      #:vperm2i128   (s8.32) (s8.32 s8.32 imm8) :cost 1)
+   ;; s16.16
+   (make-s16.16           nil            (s16.16) (s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16) :cost 1 :encoding :fake-vop)
+   (s16.16-values         nil            (s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16 s16) (s16.16) :cost 1 :encoding :fake-vop)
+   (s16.16-broadcast      nil            (s16.16) (s16)           :cost 1 :encoding :fake-vop)
+   (s16.16-broadcastvec   #:vpbroadcastw (s16.16) (s16.16)        :cost 1)
+   (s16.16-blend          #:vpblendvb    (s16.16) (s16.16 s16.16 u16.16) :cost 1)
+   (s16.16-from-s8.16     #:vpmovsxbw    (s16.16) (s8.16)         :cost 5)
+   (s16.16-from-u8.16     #:vpmovsxbw    (s16.16) (u8.16)         :cost 5)
+   (two-arg-s16.16-and    #:vpand        (s16.16) (s16.16 s16.16) :cost 1 :associative t)
+   (two-arg-s16.16-or     #:vpor         (s16.16) (s16.16 s16.16) :cost 1 :associative t)
+   (two-arg-s16.16-xor    #:vpxor        (s16.16) (s16.16 s16.16) :cost 1 :associative t)
+   (s16.16-andc1          #:vpandn       (s16.16) (s16.16 s16.16) :cost 1)
+   (s16.16-not            nil            (s16.16) (s16.16)        :cost 1 :encoding :fake-vop)
+   (two-arg-s16.16-max    #:vpmaxsw      (s16.16) (s16.16 s16.16) :cost 2 :associative t)
+   (two-arg-s16.16-min    #:vpminsw      (s16.16) (s16.16 s16.16) :cost 2 :associative t)
+   (two-arg-s16.16+       #:vpaddw       (s16.16) (s16.16 s16.16) :cost 2 :associative t)
+   (two-arg-s16.16-       #:vpsubw       (s16.16) (s16.16 s16.16) :cost 2)
+   (two-arg-s16.16-mulhi  #:vpmulhw      (s16.16) (s16.16 s16.16) :cost 1 :associative t)
+   (two-arg-s16.16-mullo  #:vpmullw      (s16.16) (s16.16 s16.16) :cost 1 :associative t)
+   (two-arg-s16.16-mulhrs #:vpmulhrsw    (s16.16) (s16.16 s16.16) :cost 1 :associative t)
+   (two-arg-s16.16=       #:vpcmpeqw     (u16.16) (s16.16 s16.16) :cost 1 :associative t)
+   (two-arg-s16.16/=      nil            (u16.16) (s16.16 s16.16) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s16.16>       #:vpcmpgtw     (u16.16) (s16.16 s16.16) :cost 1)
+   (two-arg-s16.16<       nil            (u16.16) (s16.16 s16.16) :cost 1 :encoding :fake-vop)
+   (two-arg-s16.16>=      nil            (u16.16) (s16.16 s16.16) :cost 2 :encoding :fake-vop)
+   (two-arg-s16.16<=      nil            (u16.16) (s16.16 s16.16) :cost 2 :encoding :fake-vop)
+   (s16.16-abs            #:vpabsw       (s16.16) (s16.16)        :cost 2)
+   (s16.16-hadd           #:vphaddw      (s16.16) (s16.16 s16.16) :cost 2)
+   (s16.16-hadds          #:vphaddsw     (s16.16) (s16.16 s16.16) :cost 2)
+   (s16.16-madd           #:vpmaddwd     (s16.16) (s16.16 s16.16) :cost 2)
+   (s16.16-maddubs        #:vpmaddubsw   (s16.16) (u8.32 s8.32)   :cost 2)
+   (s16.16-hsub           #:vphsubw      (s16.16) (s16.16 s16.16) :cost 2)
+   (s16.16-hsubs          #:vphsubsw     (s16.16) (s16.16 s16.16) :cost 2)
+   (s16.16-packs          #:vpackssdw    (s16.16) (s32.8 s32.8)   :cost 1)
+   (s16.16-unpackhi       #:vpunpckhwd   (s16.16) (s16.16 s16.16) :cost 1)
+   (s16.16-unpacklo       #:vpunpcklwd   (s16.16) (s16.16 s16.16) :cost 1)
+   (s16.16-movemask       nil            (u16)    (s16.16)        :cost 1 :encoding :fake-vop)
+   (s16.16-shufflehi      #:vpshufhw     (s16.16) (s16.16 imm8)   :cost 1)
+   (s16.16-shufflelo      #:vpshuflw     (s16.16) (s16.16 imm8)   :cost 1)
+   (s16.16-shiftl         #:vpsllw       (s16.16) (s16.16 s16.8)  :cost 2)
+   (s16.16-shiftr         #:vpsrlw       (s16.16) (s16.16 s16.8)  :cost 2)
+   (s16.16-sign           #:vpsignw      (s16.16) (s16.16 s16.16) :cost 2)
+   (s16.8-from-s16.16     #:vextracti128 (s16.8) (s16.16 imm1)    :cost 1)
+   (s16.16-insert-s16.8   #:vinserti128  (s16.16) (s16.16 s16.8 imm1) :cost 1)
+   (s16.16-permute128     #:vperm2i128   (s16.16) (s16.16 s16.16 imm8) :cost 1)
+   ;; s32.8
+   (make-s32.8            nil            (s32.8) (s32 s32 s32 s32 s32 s32 s32 s32) :cost 1 :encoding :fake-vop)
+   (s32.8-values          nil            (s32 s32 s32 s32 s32 s32 s32 s32) (s32.8) :cost 1 :encoding :fake-vop)
+   (s32.8-broadcast       nil            (s32.8) (s32)         :cost 1 :encoding :fake-vop)
+   (s32.8-broadcastvec    #:vpbroadcastd (s32.8) (s32.8)       :cost 1)
+   (s32.8-blend           #:vpblendvb    (s32.8) (s32.8 s32.8 u32.8) :cost 1)
+   (s32.8-from-s16.8      #:vpmovsxwd    (s32.8) (s16.8)       :cost 5)
+   (s32.8-from-u16.8      #:vpmovsxwd    (s32.8) (u16.8)       :cost 5)
+   (s32.8-from-s8.16      #:vpmovsxbd    (s32.8) (s8.16)       :cost 5)
+   (s32.8-from-u8.16      #:vpmovsxbd    (s32.8) (u8.16)       :cost 5)
+   (two-arg-s32.8-and     #:vpand        (s32.8) (s32.8 s32.8) :cost 1 :associative t)
+   (two-arg-s32.8-or      #:vpor         (s32.8) (s32.8 s32.8) :cost 1 :associative t)
+   (two-arg-s32.8-xor     #:vpxor        (s32.8) (s32.8 s32.8) :cost 1 :associative t)
+   (s32.8-andc1           #:vpandn       (s32.8) (s32.8 s32.8) :cost 1)
+   (s32.8-not             nil            (s32.8) (s32.8)       :cost 1 :encoding :fake-vop)
+   (two-arg-s32.8-max     #:vpmaxsd      (s32.8) (s32.8 s32.8) :cost 2 :associative t)
+   (two-arg-s32.8-min     #:vpminsd      (s32.8) (s32.8 s32.8) :cost 2 :associative t)
+   (two-arg-s32.8+        #:vpaddd       (s32.8) (s32.8 s32.8) :cost 2 :associative t)
+   (two-arg-s32.8-        #:vpsubd       (s32.8) (s32.8 s32.8) :cost 2)
+   (two-arg-s32.8-mullo   #:vpmulld      (s32.8) (s32.8 s32.8) :cost 2 :associative t)
+   (two-arg-s32.8=        #:vpcmpeqd     (u32.8) (s32.8 s32.8) :cost 1 :associative t)
+   (two-arg-s32.8/=       nil            (u32.8) (s32.8 s32.8) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s32.8>        #:vpcmpgtd     (u32.8) (s32.8 s32.8) :cost 1)
+   (two-arg-s32.8<        nil            (u32.8) (s32.8 s32.8) :cost 1 :encoding :fake-vop)
+   (two-arg-s32.8>=       nil            (u32.8) (s32.8 s32.8) :cost 2 :encoding :fake-vop)
+   (two-arg-s32.8<=       nil            (u32.8) (s32.8 s32.8) :cost 2 :encoding :fake-vop)
+   (s32.8-abs             #:vpabsd       (s32.8) (s32.8)       :cost 2)
+   (s32.8-hadd            #:vphaddd      (s32.8) (s32.8 s32.8) :cost 1)
+   (s32.8-hsub            #:vphsubd      (s32.8) (s32.8 s32.8) :cost 1)
+   (s32.8-shiftl          #:vpsllvd      (s32.8) (s32.8 s32.8) :cost 1)
+   (s32.8-shiftr          #:vpsrlvd      (s32.8) (s32.8 s32.8) :cost 1)
+   (s32.8-unpackhi        #:vpunpckhdq   (s32.8) (s32.8 s32.8) :cost 1)
+   (s32.8-unpacklo        #:vpunpckldq   (s32.8) (s32.8 s32.8) :cost 1)
+   (s32.8-movemask        #:vmovmskps    (u8)    (s32.8)       :cost 1)
+   (s32.8-sign            #:vpsignd      (s32.8) (s32.8 s32.8) :cost 1)
+   (s32.4-from-s32.8      #:vextracti128 (s32.4) (s32.8 imm1)  :cost 1)
+   (s32.8-insert-s32.4    #:vinserti128  (s32.8) (s32.8 s32.4 imm1) :cost 1)
+   (s32.8-permute128      #:vperm2i128   (s32.8) (s32.8 s32.8 imm8) :cost 1)
+   ;; s64.4
+   (make-s64.4            nil            (s64.4) (s64 s64 s64 s64) :cost 1 :encoding :fake-vop)
+   (s64.4-values          nil            (s64 s64 s64 s64) (s64.4) :cost 1 :encoding :fake-vop)
+   (s64.4-broadcast       nil            (s64.4) (s64)         :cost 1 :encoding :fake-vop)
+   (s64.4-broadcastvec    #:vpbroadcastq (s64.4) (s64.4)       :cost 1)
+   (s64.4-blend           #:vpblendvb    (s64.4) (s64.4 s64.4 u64.4) :cost 1)
+   (s64.4-from-s16.8      #:vpmovsxwq    (s64.4) (s16.8)       :cost 5)
+   (s64.4-from-u16.8      #:vpmovsxwq    (s64.4) (u16.8)       :cost 5)
+   (s64.4-from-s32.4      #:vpmovsxdq    (s64.4) (s32.4)       :cost 5)
+   (s64.4-from-u32.4      #:vpmovsxdq    (s64.4) (u32.4)       :cost 5)
+   (s64.4-from-s8.16      #:vpmovsxbq    (s64.4) (s8.16)       :cost 5)
+   (s64.4-from-u8.16      #:vpmovsxbq    (s64.4) (u8.16)       :cost 5)
+   (two-arg-s64.4-and     #:vpand        (s64.4) (s64.4 s64.4) :cost 1 :associative t)
+   (two-arg-s64.4-or      #:vpor         (s64.4) (s64.4 s64.4) :cost 1 :associative t)
+   (two-arg-s64.4-xor     #:vpxor        (s64.4) (s64.4 s64.4) :cost 1 :associative t)
+   (s64.4-andc1           #:vandnpd      (s64.4) (s64.4 s64.4) :cost 1)
+   (s64.4-not             nil            (s64.4) (s64.4)       :cost 1 :encoding :fake-vop)
+   (two-arg-s64.4+        #:vpaddq       (s64.4) (s64.4 s64.4) :cost 1 :associative t)
+   (two-arg-s64.4-        #:vpsubq       (s64.4) (s64.4 s64.4) :cost 1)
+   (two-arg-s64.4-mul     #:vpmuldq      (s64.4) (s64.4 s64.4) :cost 1 :associative t)
+   (two-arg-s64.4=        #:vpcmpeqq     (u64.4) (s64.4 s64.4) :cost 1 :associative t)
+   (two-arg-s64.4/=       nil            (u64.4) (s64.4 s64.4) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s64.4>        #:vpcmpgtq     (u64.4) (s64.4 s64.4) :cost 1)
+   (two-arg-s64.4<        nil            (u64.4) (s64.4 s64.4) :cost 1 :encoding :fake-vop)
+   (two-arg-s64.4>=       nil            (u64.4) (s64.4 s64.4) :cost 2 :encoding :fake-vop)
+   (two-arg-s64.4<=       nil            (u64.4) (s64.4 s64.4) :cost 2 :encoding :fake-vop)
+   (s64.4-shiftl          #:vpsllvq      (s64.4) (s64.4 s64.4) :cost 1)
+   (s64.4-shiftr          #:vpsrlvq      (s64.4) (s64.4 s64.4) :cost 1)
+   (s64.4-unpackhi        #:vpunpckhqdq  (s64.4) (s64.4 s64.4) :cost 1)
+   (s64.4-unpacklo        #:vpunpcklqdq  (s64.4) (s64.4 s64.4) :cost 1)
+   (s64.4-movemask        #:vmovmskpd    (u4)    (s64.4)       :cost 1)
+   (s64.2-from-s64.4      #:vextracti128 (s64.2) (s64.4 imm1)  :cost 1)
+   (s64.4-insert-s64.2    #:vinserti128  (s64.4) (s64.4 s64.2 imm1) :cost 1)
+   (s64.4-permute128      #:vperm2i128   (s64.4) (s64.4 s64.4 imm8) :cost 1))
+  (:loads
+   (f32.4-ntload  #:vmovntdqa f32.4  f32vec f32-array f32.4-non-temporal-aref f32.4-non-temporal-row-major-aref)
+   (f64.2-ntload  #:vmovntdqa f64.2  f64vec f64-array f64.2-non-temporal-aref f64.2-non-temporal-row-major-aref)
+   (f32.8-ntload  #:vmovntdqa f32.8  f32vec f32-array f32.8-non-temporal-aref f32.8-non-temporal-row-major-aref)
+   (f64.4-ntload  #:vmovntdqa f64.4  f64vec f64-array f64.4-non-temporal-aref f64.4-non-temporal-row-major-aref)
+   (u8.16-ntload  #:vmovntdqa u8.16   u8vec  u8-array u8.16-non-temporal-aref u8.16-non-temporal-row-major-aref)
+   (u16.8-ntload  #:vmovntdqa u16.8  u16vec u16-array u16.8-non-temporal-aref u16.8-non-temporal-row-major-aref)
+   (u32.4-ntload  #:vmovntdqa u32.4  u32vec u32-array u32.4-non-temporal-aref u32.4-non-temporal-row-major-aref)
+   (u64.2-ntload  #:vmovntdqa u64.2  u64vec u64-array u64.2-non-temporal-aref u64.2-non-temporal-row-major-aref)
+   (s8.16-ntload  #:vmovntdqa s8.16   s8vec  s8-array s8.16-non-temporal-aref s8.16-non-temporal-row-major-aref)
+   (s16.8-ntload  #:vmovntdqa s16.8  s16vec s16-array s16.8-non-temporal-aref s16.8-non-temporal-row-major-aref)
+   (s32.4-ntload  #:vmovntdqa s32.4  s32vec s32-array s32.4-non-temporal-aref s32.4-non-temporal-row-major-aref)
+   (s64.2-ntload  #:vmovntdqa s64.2  s64vec s64-array s64.2-non-temporal-aref s64.2-non-temporal-row-major-aref)
+   (u8.32-ntload  #:vmovntdqa u8.32   u8vec  u8-array u8.32-non-temporal-aref  u8.32-non-temporal-row-major-aref)
+   (u16.16-ntload #:vmovntdqa u16.16 u16vec u16-array u16.16-non-temporal-aref u16.16-non-temporal-row-major-aref)
+   (u32.8-ntload  #:vmovntdqa u32.8  u32vec u32-array u32.8-non-temporal-aref  u32.8-non-temporal-row-major-aref)
+   (u64.4-ntload  #:vmovntdqa u64.4  u64vec u64-array u64.4-non-temporal-aref  u64.4-non-temporal-row-major-aref)
+   (s8.32-ntload  #:vmovntdqa s8.32   s8vec  s8-array s8.32-non-temporal-aref  s8.32-non-temporal-row-major-aref)
+   (s16.16-ntload #:vmovntdqa s16.16 s16vec s16-array s16.16-non-temporal-aref s16.16-non-temporal-row-major-aref)
+   (s32.8-ntload  #:vmovntdqa s32.8  s32vec s32-array s32.8-non-temporal-aref  s32.8-non-temporal-row-major-aref)
+   (s64.4-ntload  #:vmovntdqa s64.4  s64vec s64-array s64.4-non-temporal-aref  s64.4-non-temporal-row-major-aref))
+  (:stores
+   (f32.4-ntstore  #:vmovntps f32.4  f32vec f32-array f32.4-non-temporal-aref  f32.4-non-temporal-row-major-aref)
+   (f64.2-ntstore  #:vmovntpd f64.2  f64vec f64-array f64.2-non-temporal-aref  f64.2-non-temporal-row-major-aref)
+   (f32.8-ntstore  #:vmovntps f32.8  f32vec f32-array f32.8-non-temporal-aref  f32.8-non-temporal-row-major-aref)
+   (f64.4-ntstore  #:vmovntpd f64.4  f64vec f64-array f64.4-non-temporal-aref  f64.4-non-temporal-row-major-aref)
+   (u8.16-ntstore  #:vmovntdq u8.16   u8vec  u8-array u8.16-non-temporal-aref  u8.16-non-temporal-row-major-aref)
+   (u16.8-ntstore  #:vmovntdq u16.8  u16vec u16-array u16.8-non-temporal-aref  u16.8-non-temporal-row-major-aref)
+   (u32.4-ntstore  #:vmovntdq u32.4  u32vec u32-array u32.4-non-temporal-aref  u32.4-non-temporal-row-major-aref)
+   (u64.2-ntstore  #:vmovntdq u64.2  u64vec u64-array u64.2-non-temporal-aref  u64.2-non-temporal-row-major-aref)
+   (s8.16-ntstore  #:vmovntdq s8.16   s8vec  s8-array s8.16-non-temporal-aref  s8.16-non-temporal-row-major-aref)
+   (s16.8-ntstore  #:vmovntdq s16.8  s16vec s16-array s16.8-non-temporal-aref  s16.8-non-temporal-row-major-aref)
+   (s32.4-ntstore  #:vmovntdq s32.4  s32vec s32-array s32.4-non-temporal-aref  s32.4-non-temporal-row-major-aref)
+   (s64.2-ntstore  #:vmovntdq s64.2  s64vec s64-array s64.2-non-temporal-aref  s64.2-non-temporal-row-major-aref)
+   (u8.32-ntstore  #:vmovntdq u8.32   u8vec  u8-array u8.32-non-temporal-aref  u8.32-non-temporal-row-major-aref)
+   (u16.16-ntstore #:vmovntdq u16.16 u16vec u16-array u16.16-non-temporal-aref u16.16-non-temporal-row-major-aref)
+   (u32.8-ntstore  #:vmovntdq u32.8  u32vec u32-array u32.8-non-temporal-aref  u32.8-non-temporal-row-major-aref)
+   (u64.4-ntstore  #:vmovntdq u64.4  u64vec u64-array u64.4-non-temporal-aref  u64.4-non-temporal-row-major-aref)
+   (s8.32-ntstore  #:vmovntdq s8.32   s8vec  s8-array s8.32-non-temporal-aref  s8.32-non-temporal-row-major-aref)
+   (s16.16-ntstore #:vmovntdq s16.16 s16vec s16-array s16.16-non-temporal-aref s16.16-non-temporal-row-major-aref)
+   (s32.8-ntstore  #:vmovntdq s32.8  s32vec s32-array s32.8-non-temporal-aref  s32.8-non-temporal-row-major-aref)
+   (s64.4-ntstore  #:vmovntdq s64.4  s64vec s64-array s64.4-non-temporal-aref  s64.4-non-temporal-row-major-aref))
+  (:associatives
+   (u8.32-and two-arg-u8.32-and +u8-true+)
+   (u8.32-or  two-arg-u8.32-or  +u8-false+)
+   (u8.32-xor two-arg-u8.32-xor +u8-false+)
+   (u8.32-max two-arg-u8.32-max nil)
+   (u8.32-min two-arg-u8.32-min nil)
+   (u8.32+    two-arg-u8.32+ 0)
+   (u16.16-and two-arg-u16.16-and +u16-true+)
+   (u16.16-or  two-arg-u16.16-or  +u16-false+)
+   (u16.16-xor two-arg-u16.16-xor +u16-false+)
+   (u16.16-max two-arg-u16.16-max nil)
+   (u16.16-min two-arg-u16.16-min nil)
+   (u16.16+    two-arg-u16.16+ 0)
+   (u32.8-and two-arg-u32.8-and +u32-true+)
+   (u32.8-or  two-arg-u32.8-or  +u32-false+)
+   (u32.8-xor two-arg-u32.8-xor +u32-false+)
+   (u32.8-max two-arg-u32.8-max nil)
+   (u32.8-min two-arg-u32.8-min nil)
+   (u32.8+    two-arg-u32.8+ 0)
+   (u64.4-and two-arg-u64.4-and +u64-true+)
+   (u64.4-or  two-arg-u64.4-or  +u64-false+)
+   (u64.4-xor two-arg-u64.4-xor +u64-false+)
+   (u64.4+    two-arg-u64.4+ 0)
+   (u64.4-mul two-arg-u64.4-mul 1)
+   (s8.32-and two-arg-s8.32-and +s8-true+)
+   (s8.32-or  two-arg-s8.32-or  +s8-false+)
+   (s8.32-xor two-arg-s8.32-xor +s8-false+)
+   (s8.32-max two-arg-s8.32-max nil)
+   (s8.32-min two-arg-s8.32-min nil)
+   (s8.32+    two-arg-s8.32+ 0)
+   (s16.16-and two-arg-s16.16-and +s16-true+)
+   (s16.16-or  two-arg-s16.16-or  +s16-false+)
+   (s16.16-xor two-arg-s16.16-xor +s16-false+)
+   (s16.16-max two-arg-s16.16-max nil)
+   (s16.16-min two-arg-s16.16-min nil)
+   (s16.16+    two-arg-s16.16+ 0)
+   (s16.16-mulhi two-arg-s16.16-mulhi 1)
+   (s16.16-mullo two-arg-s16.16-mullo 1)
+   (s16.16-mulhrs two-arg-s16.16-mulhrs 1)
+   (s32.8-and two-arg-s32.8-and +s32-true+)
+   (s32.8-or  two-arg-s32.8-or  +s32-false+)
+   (s32.8-xor two-arg-s32.8-xor +s32-false+)
+   (s32.8-max two-arg-s32.8-max nil)
+   (s32.8-min two-arg-s32.8-min nil)
+   (s32.8+    two-arg-s32.8+ 0)
+   (s32.8-mullo two-arg-s32.8-mullo 1)
+   (s64.4-and two-arg-s64.4-and +s64-true+)
+   (s64.4-or  two-arg-s64.4-or  +s64-false+)
+   (s64.4-xor two-arg-s64.4-xor +s64-false+)
+   (s64.4+    two-arg-s64.4+ 0)
+   (s64.4-mul two-arg-s64.4-mul 1))
+  (:comparisons
+   (u8.32=  two-arg-u8.32=  u8.32-and +u8-true+)
+   (u8.32<  two-arg-u8.32<  u8.32-and +u8-true+)
+   (u8.32<= two-arg-u8.32<= u8.32-and +u8-true+)
+   (u8.32>  two-arg-u8.32>  u8.32-and +u8-true+)
+   (u8.32>= two-arg-u8.32>= u8.32-and +u8-true+)
+   (u16.16=  two-arg-u16.16=  u16.16-and +u16-true+)
+   (u16.16<  two-arg-u16.16<  u16.16-and +u16-true+)
+   (u16.16<= two-arg-u16.16<= u16.16-and +u16-true+)
+   (u16.16>  two-arg-u16.16>  u16.16-and +u16-true+)
+   (u16.16>= two-arg-u16.16>= u16.16-and +u16-true+)
+   (u32.8=  two-arg-u32.8=  u32.8-and +u32-true+)
+   (u32.8<  two-arg-u32.8<  u32.8-and +u32-true+)
+   (u32.8<= two-arg-u32.8<= u32.8-and +u32-true+)
+   (u32.8>  two-arg-u32.8>  u32.8-and +u32-true+)
+   (u32.8>= two-arg-u32.8>= u32.8-and +u32-true+)
+   (u64.4=  two-arg-u64.4=  u64.4-and +u64-true+)
+   (u64.4<  two-arg-u64.4<  u64.4-and +u64-true+)
+   (u64.4<= two-arg-u64.4<= u64.4-and +u64-true+)
+   (u64.4>  two-arg-u64.4>  u64.4-and +u64-true+)
+   (u64.4>= two-arg-u64.4>= u64.4-and +u64-true+)
+   (s8.32=  two-arg-s8.32=  u8.32-and +u8-true+)
+   (s8.32<  two-arg-s8.32<  u8.32-and +u8-true+)
+   (s8.32<= two-arg-s8.32<= u8.32-and +u8-true+)
+   (s8.32>  two-arg-s8.32>  u8.32-and +u8-true+)
+   (s8.32>= two-arg-s8.32>= u8.32-and +u8-true+)
+   (s16.16=  two-arg-s16.16=  u16.16-and +u16-true+)
+   (s16.16<  two-arg-s16.16<  u16.16-and +u16-true+)
+   (s16.16<= two-arg-s16.16<= u16.16-and +u16-true+)
+   (s16.16>  two-arg-s16.16>  u16.16-and +u16-true+)
+   (s16.16>= two-arg-s16.16>= u16.16-and +u16-true+)
+   (s32.8=  two-arg-s32.8=  u32.8-and +u32-true+)
+   (s32.8<  two-arg-s32.8<  u32.8-and +u32-true+)
+   (s32.8<= two-arg-s32.8<= u32.8-and +u32-true+)
+   (s32.8>  two-arg-s32.8>  u32.8-and +u32-true+)
+   (s32.8>= two-arg-s32.8>= u32.8-and +u32-true+)
+   (s64.4=  two-arg-s64.4=  u64.4-and +u64-true+)
+   (s64.4<  two-arg-s64.4<  u64.4-and +u64-true+)
+   (s64.4<= two-arg-s64.4<= u64.4-and +u64-true+)
+   (s64.4>  two-arg-s64.4>  u64.4-and +u64-true+)
+   (s64.4>= two-arg-s64.4>= u64.4-and +u64-true+))
+  (:ifs
+   (u8.32-if u8.32-blend)
+   (u16.16-if u16.16-blend)
+   (u32.8-if u32.8-blend)
+   (u64.4-if u64.4-blend)
+   (s8.32-if s8.32-blend)
+   (s16.16-if s16.16-blend)
+   (s32.8-if s32.8-blend)
+   (s64.4-if s64.4-blend))
+  (:reducers
+   (u8.32- two-arg-u8.32- 0)
+   (u16.16- two-arg-u16.16- 0)
+   (u32.8- two-arg-u32.8- 0)
+   (u64.4- two-arg-u64.4- 0)
+   (s8.32- two-arg-s8.32- 0)
+   (s16.16- two-arg-s16.16- 0)
+   (s32.8- two-arg-s32.8- 0)
+   (s64.4- two-arg-s64.4- 0))
+  (:unequals
+   (u8.32/=  two-arg-u8.32/=  u8.32-and  +u8-true+)
+   (u16.16/= two-arg-u16.16/= u16.16-and +u16-true+)
+   (u32.8/=  two-arg-u32.8/=  u32.8-and  +u32-true+)
+   (u64.4/=  two-arg-u64.4/=  u64.4-and  +u64-true+)
+   (s8.32/=  two-arg-s8.32/=  u8.32-and  +u8-true+)
+   (s16.16/= two-arg-s16.16/= u16.16-and +u16-true+)
+   (s32.8/=  two-arg-s32.8/=  u32.8-and  +u32-true+)
+   (s64.4/=  two-arg-s64.4/=  u64.4-and  +u64-true+)))

contrib/sb-simd/code/instruction-sets/fma.lisp

@@ -0,0 +1,38 @@
+(in-package #:sb-simd-fma)
+
+(define-instruction-set :fma
+  (:test (fma-supported-p))
+  (:include :avx2)
+  (:instructions
+   ;; f32
+   (f32-fmadd       #:vfmadd213ss      (f32) (f32 f32 f32) :cost 1 :encoding :fma)
+   (f32-fnmadd      #:vfnmadd213ss     (f32) (f32 f32 f32) :cost 1 :encoding :fma)
+   (f32-fmsub       #:vfmsub213ss      (f32) (f32 f32 f32) :cost 1 :encoding :fma)
+   ;; f64
+   (f64-fmadd       #:vfmadd213sd     (f64) (f64 f64 f64) :cost 1 :encoding :fma)
+   (f64-fnmadd      #:vfnmadd213sd    (f64) (f64 f64 f64) :cost 1 :encoding :fma)
+   (f64-fmsub       #:vfmsub213sd     (f64) (f64 f64 f64) :cost 1 :encoding :fma)
+   ;; f32.4
+   (f32.4-fmadd     #:vfmadd213ps     (f32.4) (f32.4 f32.4 f32.4) :cost 1 :encoding :fma)
+   (f32.4-fnmadd    #:vfnmadd213ps    (f32.4) (f32.4 f32.4 f32.4) :cost 1 :encoding :fma)
+   (f32.4-fmsub     #:vfmsub213ps     (f32.4) (f32.4 f32.4 f32.4) :cost 1 :encoding :fma)
+   (f32.4-fmaddsub  #:vfmaddsub213ps  (f32.4) (f32.4 f32.4 f32.4) :cost 1 :encoding :fma)
+   (f32.4-fmsubadd  #:vfmsubadd213ps  (f32.4) (f32.4 f32.4 f32.4) :cost 1 :encoding :fma)
+   ;; f32.8
+   (f32.8-fmadd     #:vfmadd213ps     (f32.8) (f32.8 f32.8 f32.8) :cost 1 :encoding :fma)
+   (f32.8-fnmadd    #:vfnmadd213ps    (f32.8) (f32.8 f32.8 f32.8) :cost 1 :encoding :fma)
+   (f32.8-fmsub     #:vfmsub213ps     (f32.8) (f32.8 f32.8 f32.8) :cost 1 :encoding :fma)
+   (f32.8-fmaddsub  #:vfmaddsub213ps  (f32.8) (f32.8 f32.8 f32.8) :cost 1 :encoding :fma)
+   (f32.8-fmsubadd  #:vfmsubadd213ps  (f32.8) (f32.8 f32.8 f32.8) :cost 1 :encoding :fma)
+   ;; f64.2
+   (f64.2-fmadd     #:vfmadd213pd     (f64.2) (f64.2 f64.2 f64.2) :cost 1 :encoding :fma)
+   (f64.2-fnmadd    #:vfnmadd213pd    (f64.2) (f64.2 f64.2 f64.2) :cost 1 :encoding :fma)
+   (f64.2-fmsub     #:vfmsub213pd     (f64.2) (f64.2 f64.2 f64.2) :cost 1 :encoding :fma)
+   (f64.2-fmaddsub  #:vfmaddsub213pd  (f64.2) (f64.2 f64.2 f64.2) :cost 1 :encoding :fma)
+   (f64.2-fmsubadd  #:vfmsubadd213pd  (f64.2) (f64.2 f64.2 f64.2) :cost 1 :encoding :fma)
+   ;; f64.4
+   (f64.4-fmadd     #:vfmadd213pd     (f64.4) (f64.4 f64.4 f64.4) :cost 1 :encoding :fma)
+   (f64.4-fnmadd    #:vfnmadd213pd    (f64.4) (f64.4 f64.4 f64.4) :cost 1 :encoding :fma)
+   (f64.4-fmsub     #:vfmsub213pd     (f64.4) (f64.4 f64.4 f64.4) :cost 1 :encoding :fma)
+   (f64.4-fmaddsub  #:vfmaddsub213pd  (f64.4) (f64.4 f64.4 f64.4) :cost 1 :encoding :fma)
+   (f64.4-fmsubadd  #:vfmsubadd213pd  (f64.4) (f64.4 f64.4 f64.4) :cost 1 :encoding :fma)))

contrib/sb-simd/code/instruction-sets/sb-simd.lisp

@@ -0,0 +1,391 @@
+(in-package #:sb-simd)
+
+(define-instruction-set :sb-simd
+  (:scalars
+   (any     64  t                  #:t)
+   ;; Numbers
+   (index   64  sb-simd-internals::index #:signed-num (#:signed-reg))
+   (u1      1   (unsigned-byte  1) #:unsigned-num (#:unsigned-reg))
+   (u2      2   (unsigned-byte  2) #:unsigned-num (#:unsigned-reg))
+   (u4      4   (unsigned-byte  4) #:unsigned-num (#:unsigned-reg))
+   (u8      8   (unsigned-byte  8) #:unsigned-num (#:unsigned-reg))
+   (u16     16  (unsigned-byte 16) #:unsigned-num (#:unsigned-reg))
+   (u32     32  (unsigned-byte 32) #:unsigned-num (#:unsigned-reg))
+   (u64     64  (unsigned-byte 64) #:unsigned-num (#:unsigned-reg))
+   (s8      8   (signed-byte  8)   #:signed-num   (#:signed-reg))
+   (s16     16  (signed-byte 16)   #:signed-num   (#:signed-reg))
+   (s32     32  (signed-byte 32)   #:signed-num   (#:signed-reg))
+   (s64     64  (signed-byte 64)   #:signed-num   (#:signed-reg))
+   (f32     32  single-float       #:single-float (#:single-reg))
+   (f64     64  double-float       #:double-float (#:double-reg))
+   ;; Vectors
+   (charvec 64  (simple-array character (*))          #:simple-character-string)
+   ( u8vec  64  (simple-array (unsigned-byte 8) (*))  #:simple-array-unsigned-byte-8)
+   (u16vec  64  (simple-array (unsigned-byte 16) (*)) #:simple-array-unsigned-byte-16)
+   (u32vec  64  (simple-array (unsigned-byte 32) (*)) #:simple-array-unsigned-byte-32)
+   (u64vec  64  (simple-array (unsigned-byte 64) (*)) #:simple-array-unsigned-byte-64)
+   ( s8vec  64  (simple-array (signed-byte 8) (*))    #:simple-array-signed-byte-8)
+   (s16vec  64  (simple-array (signed-byte 16) (*))   #:simple-array-signed-byte-16)
+   (s32vec  64  (simple-array (signed-byte 32) (*))   #:simple-array-signed-byte-32)
+   (s64vec  64  (simple-array (signed-byte 64) (*))   #:simple-array-signed-byte-64)
+   (f32vec  64  (simple-array single-float (*))       #:simple-array-single-float)
+   (f64vec  64  (simple-array double-float (*))       #:simple-array-double-float)
+   ;; Arrays
+   (char-array 64  (array character))
+   ( u8-array  64  (array (unsigned-byte 8)))
+   (u16-array  64  (array (unsigned-byte 16)))
+   (u32-array  64  (array (unsigned-byte 32)))
+   (u64-array  64  (array (unsigned-byte 64)))
+   ( s8-array  64  (array (signed-byte 8)))
+   (s16-array  64  (array (signed-byte 16)))
+   (s32-array  64  (array (signed-byte 32)))
+   (s64-array  64  (array (signed-byte 64)))
+   (f32-array  64  (array single-float))
+   (f64-array  64  (array double-float)))
+  (:instructions
+   ;; ub64 packers and unpackers
+   (u64-from-f32  nil (u64) (f32 f32)                 :encoding :fake-vop)
+   (u64-from-f64  nil (u64) (f64)                     :encoding :fake-vop)
+   (u64-from-u8s  nil (u64) (u8 u8 u8 u8 u8 u8 u8 u8) :encoding :fake-vop)
+   (u64-from-u16s nil (u64) (u16 u16 u16 u16)         :encoding :fake-vop)
+   (u64-from-u32s nil (u64) (u32 u32)                 :encoding :fake-vop)
+   (u64-from-s8s  nil (u64) (s8 s8 s8 s8 s8 s8 s8 s8) :encoding :fake-vop)
+   (u64-from-s16s nil (u64) (s16 s16 s16 s16)         :encoding :fake-vop)
+   (u64-from-s32s nil (u64) (s32 s32)                 :encoding :fake-vop)
+   (u64-from-s64  nil (u64) (s64)                     :encoding :fake-vop)
+   ( u8s-from-u64 nil (u8 u8 u8 u8 u8 u8 u8 u8) (u64) :encoding :fake-vop)
+   (u16s-from-u64 nil (u16 u16 u16 u16) (u64)         :encoding :fake-vop)
+   (u32s-from-u64 nil (u32 u32) (u64)                 :encoding :fake-vop)
+   ( s8s-from-u64 nil (s8 s8 s8 s8 s8 s8 s8 s8) (u64) :encoding :fake-vop)
+   (s16s-from-u64 nil (s16 s16 s16 s16) (u64)         :encoding :fake-vop)
+   (s32s-from-u64 nil (s32 s32) (u64)                 :encoding :fake-vop)
+   ( s64-from-u64 nil (s64) (u64)                     :encoding :fake-vop)
+   ;; f32
+   (f32-if          nil (f32) (u32 f32 f32) :encoding :fake-vop)
+   (two-arg-f32-and nil (f32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32-or  nil (f32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32-xor nil (f32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32-max nil (f32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32-min nil (f32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32+    nil (f32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32-    nil (f32) (f32 f32) :encoding :fake-vop)
+   (two-arg-f32*    nil (f32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32/    nil (f32) (f32 f32) :encoding :fake-vop)
+   (two-arg-f32=    nil (u32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32/=   nil (u32) (f32 f32) :encoding :fake-vop :associative t)
+   (two-arg-f32<    nil (u32) (f32 f32) :encoding :fake-vop)
+   (two-arg-f32<=   nil (u32) (f32 f32) :encoding :fake-vop)
+   (two-arg-f32>    nil (u32) (f32 f32) :encoding :fake-vop)
+   (two-arg-f32>=   nil (u32) (f32 f32) :encoding :fake-vop)
+   (f32-andc1       nil (f32) (f32 f32) :encoding :fake-vop)
+   (f32-not         nil (f32) (f32) :encoding :fake-vop)
+   (f32-reciprocal  nil (f32) (f32) :encoding :fake-vop)
+   (f32-rsqrt       nil (f32) (f32) :encoding :fake-vop)
+   (f32-sqrt        nil (f32) (f32) :encoding :fake-vop)
+   ;; f64
+   (f64-if          nil (f64) (u64 f64 f64) :encoding :fake-vop)
+   (two-arg-f64-and nil (f64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64-or  nil (f64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64-xor nil (f64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64-max nil (f64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64-min nil (f64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64+    nil (f64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64-    nil (f64) (f64 f64) :encoding :fake-vop)
+   (two-arg-f64*    nil (f64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64/    nil (f64) (f64 f64) :encoding :fake-vop)
+   (two-arg-f64=    nil (u64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64/=   nil (u64) (f64 f64) :encoding :fake-vop :associative t)
+   (two-arg-f64<    nil (u64) (f64 f64) :encoding :fake-vop)
+   (two-arg-f64<=   nil (u64) (f64 f64) :encoding :fake-vop)
+   (two-arg-f64>    nil (u64) (f64 f64) :encoding :fake-vop)
+   (two-arg-f64>=   nil (u64) (f64 f64) :encoding :fake-vop)
+   (f64-andc1       nil (f64) (f64 f64) :encoding :fake-vop)
+   (f64-not         nil (f64) (f64) :encoding :fake-vop)
+   (f64-reciprocal  nil (f64) (f64) :encoding :fake-vop)
+   (f64-rsqrt       nil (f64) (f64) :encoding :fake-vop)
+   (f64-sqrt        nil (f64) (f64) :encoding :fake-vop)
+   ;; u8
+   (u8-if          nil (u8) (u8 u8 u8) :encoding :fake-vop)
+   (two-arg-u8-and nil (u8) (u8 u8) :encoding :fake-vop :associative t)
+   (two-arg-u8-or  nil (u8) (u8 u8) :encoding :fake-vop :associative t)
+   (two-arg-u8-xor nil (u8) (u8 u8) :encoding :fake-vop :associative t)
+   (two-arg-u8-max nil (u8) (u8 u8) :encoding :fake-vop :associative t)
+   (two-arg-u8-min nil (u8) (u8 u8) :encoding :fake-vop :associative t)
+   (two-arg-u8+    nil (u8) (u8 u8) :encoding :fake-vop :associative t)
+   (two-arg-u8-    nil (u8) (u8 u8) :encoding :fake-vop)
+   (two-arg-u8=    nil (u8) (u8 u8) :encoding :fake-vop :associative t)
+   (two-arg-u8/=   nil (u8) (u8 u8) :encoding :fake-vop :associative t)
+   (two-arg-u8<    nil (u8) (u8 u8) :encoding :fake-vop)
+   (two-arg-u8<=   nil (u8) (u8 u8) :encoding :fake-vop)
+   (two-arg-u8>    nil (u8) (u8 u8) :encoding :fake-vop)
+   (two-arg-u8>=   nil (u8) (u8 u8) :encoding :fake-vop)
+   (u8-andc1       nil (u8) (u8 u8) :encoding :fake-vop)
+   (u8-not         nil (u8) (u8)    :encoding :fake-vop)
+   ;; u16
+   (u16-if          nil (u16) (u16 u16 u16) :encoding :fake-vop)
+   (two-arg-u16-and nil (u16) (u16 u16) :encoding :fake-vop :associative t)
+   (two-arg-u16-or  nil (u16) (u16 u16) :encoding :fake-vop :associative t)
+   (two-arg-u16-xor nil (u16) (u16 u16) :encoding :fake-vop :associative t)
+   (two-arg-u16-max nil (u16) (u16 u16) :encoding :fake-vop :associative t)
+   (two-arg-u16-min nil (u16) (u16 u16) :encoding :fake-vop :associative t)
+   (two-arg-u16+    nil (u16) (u16 u16) :encoding :fake-vop :associative t)
+   (two-arg-u16-    nil (u16) (u16 u16) :encoding :fake-vop)
+   (two-arg-u16=    nil (u16) (u16 u16) :encoding :fake-vop :associative t)
+   (two-arg-u16/=   nil (u16) (u16 u16) :encoding :fake-vop :associative t)
+   (two-arg-u16<    nil (u16) (u16 u16) :encoding :fake-vop)
+   (two-arg-u16<=   nil (u16) (u16 u16) :encoding :fake-vop)
+   (two-arg-u16>    nil (u16) (u16 u16) :encoding :fake-vop)
+   (two-arg-u16>=   nil (u16) (u16 u16) :encoding :fake-vop)
+   (u16-andc1       nil (u16) (u16 u16) :encoding :fake-vop)
+   (u16-not         nil (u16) (u16)     :encoding :fake-vop)
+   ;; u32
+   (u32-if          nil (u32) (u32 u32 u32) :encoding :fake-vop)
+   (two-arg-u32-and nil (u32) (u32 u32) :encoding :fake-vop :associative t)
+   (two-arg-u32-or  nil (u32) (u32 u32) :encoding :fake-vop :associative t)
+   (two-arg-u32-xor nil (u32) (u32 u32) :encoding :fake-vop :associative t)
+   (two-arg-u32-max nil (u32) (u32 u32) :encoding :fake-vop :associative t)
+   (two-arg-u32-min nil (u32) (u32 u32) :encoding :fake-vop :associative t)
+   (two-arg-u32+    nil (u32) (u32 u32) :encoding :fake-vop :associative t)
+   (two-arg-u32-    nil (u32) (u32 u32) :encoding :fake-vop)
+   (two-arg-u32=    nil (u32) (u32 u32) :encoding :fake-vop :associative t)
+   (two-arg-u32/=   nil (u32) (u32 u32) :encoding :fake-vop :associative t)
+   (two-arg-u32<    nil (u32) (u32 u32) :encoding :fake-vop)
+   (two-arg-u32<=   nil (u32) (u32 u32) :encoding :fake-vop)
+   (two-arg-u32>    nil (u32) (u32 u32) :encoding :fake-vop)
+   (two-arg-u32>=   nil (u32) (u32 u32) :encoding :fake-vop)
+   (u32-andc1       nil (u32) (u32 u32) :encoding :fake-vop)
+   (u32-not         nil (u32) (u32)     :encoding :fake-vop)
+   ;; u64
+   (u64-if          nil (u64) (u64 u64 u64) :encoding :fake-vop)
+   (two-arg-u64-and nil (u64) (u64 u64) :encoding :fake-vop :associative t)
+   (two-arg-u64-or  nil (u64) (u64 u64) :encoding :fake-vop :associative t)
+   (two-arg-u64-xor nil (u64) (u64 u64) :encoding :fake-vop :associative t)
+   (two-arg-u64-max nil (u64) (u64 u64) :encoding :fake-vop :associative t)
+   (two-arg-u64-min nil (u64) (u64 u64) :encoding :fake-vop :associative t)
+   (two-arg-u64+    nil (u64) (u64 u64) :encoding :fake-vop :associative t)
+   (two-arg-u64-    nil (u64) (u64 u64) :encoding :fake-vop)
+   (two-arg-u64=    nil (u64) (u64 u64) :encoding :fake-vop :associative t)
+   (two-arg-u64/=   nil (u64) (u64 u64) :encoding :fake-vop :associative t)
+   (two-arg-u64<    nil (u64) (u64 u64) :encoding :fake-vop)
+   (two-arg-u64<=   nil (u64) (u64 u64) :encoding :fake-vop)
+   (two-arg-u64>    nil (u64) (u64 u64) :encoding :fake-vop)
+   (two-arg-u64>=   nil (u64) (u64 u64) :encoding :fake-vop)
+   (u64-andc1       nil (u64) (u64 u64) :encoding :fake-vop)
+   (u64-not         nil (u64) (u64)     :encoding :fake-vop)
+   ;; s8
+   (s8-if          nil (s8) (u8 s8 s8) :encoding :fake-vop)
+   (two-arg-s8-and nil (s8) (s8 s8) :encoding :fake-vop :associative t)
+   (two-arg-s8-or  nil (s8) (s8 s8) :encoding :fake-vop :associative t)
+   (two-arg-s8-xor nil (s8) (s8 s8) :encoding :fake-vop :associative t)
+   (two-arg-s8-max nil (s8) (s8 s8) :encoding :fake-vop :associative t)
+   (two-arg-s8-min nil (s8) (s8 s8) :encoding :fake-vop :associative t)
+   (two-arg-s8+    nil (s8) (s8 s8) :encoding :fake-vop :associative t)
+   (two-arg-s8-    nil (s8) (s8 s8) :encoding :fake-vop)
+   (two-arg-s8=    nil (u8) (s8 s8) :encoding :fake-vop :associative t)
+   (two-arg-s8/=   nil (u8) (s8 s8) :encoding :fake-vop :associative t)
+   (two-arg-s8<    nil (u8) (s8 s8) :encoding :fake-vop)
+   (two-arg-s8<=   nil (u8) (s8 s8) :encoding :fake-vop)
+   (two-arg-s8>    nil (u8) (s8 s8) :encoding :fake-vop)
+   (two-arg-s8>=   nil (u8) (s8 s8) :encoding :fake-vop)
+   (s8-andc1       nil (s8) (s8 s8) :encoding :fake-vop)
+   (s8-not         nil (s8) (s8)    :encoding :fake-vop)
+   ;; s16
+   (s16-if          nil (s16) (u16 s16 s16) :encoding :fake-vop)
+   (two-arg-s16-and nil (s16) (s16 s16) :encoding :fake-vop :associative t)
+   (two-arg-s16-or  nil (s16) (s16 s16) :encoding :fake-vop :associative t)
+   (two-arg-s16-xor nil (s16) (s16 s16) :encoding :fake-vop :associative t)
+   (two-arg-s16-max nil (s16) (s16 s16) :encoding :fake-vop :associative t)
+   (two-arg-s16-min nil (s16) (s16 s16) :encoding :fake-vop :associative t)
+   (two-arg-s16+    nil (s16) (s16 s16) :encoding :fake-vop :associative t)
+   (two-arg-s16-    nil (s16) (s16 s16) :encoding :fake-vop)
+   (two-arg-s16=    nil (u16) (s16 s16) :encoding :fake-vop :associative t)
+   (two-arg-s16/=   nil (u16) (s16 s16) :encoding :fake-vop :associative t)
+   (two-arg-s16<    nil (u16) (s16 s16) :encoding :fake-vop)
+   (two-arg-s16<=   nil (u16) (s16 s16) :encoding :fake-vop)
+   (two-arg-s16>    nil (u16) (s16 s16) :encoding :fake-vop)
+   (two-arg-s16>=   nil (u16) (s16 s16) :encoding :fake-vop)
+   (s16-andc1       nil (s16) (s16 s16) :encoding :fake-vop)
+   (s16-not         nil (s16) (s16)     :encoding :fake-vop)
+   ;; s32
+   (s32-if          nil (s32) (u32 s32 s32) :encoding :fake-vop)
+   (two-arg-s32-and nil (s32) (s32 s32) :encoding :fake-vop :associative t)
+   (two-arg-s32-or  nil (s32) (s32 s32) :encoding :fake-vop :associative t)
+   (two-arg-s32-xor nil (s32) (s32 s32) :encoding :fake-vop :associative t)
+   (two-arg-s32-max nil (s32) (s32 s32) :encoding :fake-vop :associative t)
+   (two-arg-s32-min nil (s32) (s32 s32) :encoding :fake-vop :associative t)
+   (two-arg-s32+    nil (s32) (s32 s32) :encoding :fake-vop :associative t)
+   (two-arg-s32-    nil (s32) (s32 s32) :encoding :fake-vop)
+   (two-arg-s32=    nil (u32) (s32 s32) :encoding :fake-vop :associative t)
+   (two-arg-s32/=   nil (u32) (s32 s32) :encoding :fake-vop :associative t)
+   (two-arg-s32<    nil (u32) (s32 s32) :encoding :fake-vop)
+   (two-arg-s32<=   nil (u32) (s32 s32) :encoding :fake-vop)
+   (two-arg-s32>    nil (u32) (s32 s32) :encoding :fake-vop)
+   (two-arg-s32>=   nil (u32) (s32 s32) :encoding :fake-vop)
+   (s32-andc1       nil (s32) (s32 s32) :encoding :fake-vop)
+   (s32-not         nil (s32) (s32)     :encoding :fake-vop)
+   ;; s64
+   (s64-if          nil (s64) (u64 s64 s64) :encoding :fake-vop)
+   (two-arg-s64-and nil (s64) (s64 s64) :encoding :fake-vop :associative t)
+   (two-arg-s64-or  nil (s64) (s64 s64) :encoding :fake-vop :associative t)
+   (two-arg-s64-xor nil (s64) (s64 s64) :encoding :fake-vop :associative t)
+   (two-arg-s64-max nil (s64) (s64 s64) :encoding :fake-vop :associative t)
+   (two-arg-s64-min nil (s64) (s64 s64) :encoding :fake-vop :associative t)
+   (two-arg-s64+    nil (s64) (s64 s64) :encoding :fake-vop :associative t)
+   (two-arg-s64-    nil (s64) (s64 s64) :encoding :fake-vop)
+   (two-arg-s64=    nil (u64) (s64 s64) :encoding :fake-vop :associative t)
+   (two-arg-s64/=   nil (u64) (s64 s64) :encoding :fake-vop :associative t)
+   (two-arg-s64<    nil (u64) (s64 s64) :encoding :fake-vop)
+   (two-arg-s64<=   nil (u64) (s64 s64) :encoding :fake-vop)
+   (two-arg-s64>    nil (u64) (s64 s64) :encoding :fake-vop)
+   (two-arg-s64>=   nil (u64) (s64 s64) :encoding :fake-vop)
+   (s64-andc1       nil (s64) (s64 s64) :encoding :fake-vop)
+   (s64-not         nil (s64) (s64)     :encoding :fake-vop))
+  (:reffers
+   (f32 f32-array f32-aref f32-row-major-aref)
+   (f64 f64-array f64-aref f64-row-major-aref)
+   ( u8  u8-array  u8-aref  u8-row-major-aref)
+   (u16 u16-array u16-aref u16-row-major-aref)
+   (u32 u32-array u32-aref u32-row-major-aref)
+   (u64 u64-array u64-aref u64-row-major-aref)
+   ( s8  s8-array  s8-aref  s8-row-major-aref)
+   (s16 s16-array s16-aref s16-row-major-aref)
+   (s32 s32-array s32-aref s32-row-major-aref)
+   (s64 s64-array s64-aref s64-row-major-aref))
+  (:associatives
+   (f32-and two-arg-f32-and +f32-true+)
+   (f32-or  two-arg-f32-or  +f32-false+)
+   (f32-xor two-arg-f32-xor +f32-false+)
+   (f32-max two-arg-f32-max nil)
+   (f32-min two-arg-f32-min nil)
+   (f32+    two-arg-f32+ 0f0)
+   (f32*    two-arg-f32* 1f0)
+   (f64-and two-arg-f64-and +f64-true+)
+   (f64-or  two-arg-f64-or  +f64-false+)
+   (f64-xor two-arg-f64-xor +f64-false+)
+   (f64-max two-arg-f64-max nil)
+   (f64-min two-arg-f64-min nil)
+   (f64+    two-arg-f64+ 0d0)
+   (f64*    two-arg-f64* 1d0)
+   ( u8-and two-arg-u8-and +u8-true+)
+   ( u8-or  two-arg-u8-or  +u8-false+)
+   ( u8-xor two-arg-u8-xor +u8-false+)
+   ( u8-max two-arg-u8-max nil)
+   ( u8-min two-arg-u8-min nil)
+   ( u8+    two-arg-u8+ 0)
+   (u16-and two-arg-u16-and +u16-true+)
+   (u16-or  two-arg-u16-or  +u16-false+)
+   (u16-xor two-arg-u16-xor +u16-false+)
+   (u16-max two-arg-u16-max nil)
+   (u16-min two-arg-u16-min nil)
+   (u16+    two-arg-u16+ 0)
+   (u32-and two-arg-u32-and +u32-true+)
+   (u32-or  two-arg-u32-or  +u32-false+)
+   (u32-xor two-arg-u32-xor +u32-false+)
+   (u32-max two-arg-u32-max nil)
+   (u32-min two-arg-u32-min nil)
+   (u32+    two-arg-u32+ 0)
+   (u64-and two-arg-u64-and +u64-true+)
+   (u64-or  two-arg-u64-or  +u64-false+)
+   (u64-xor two-arg-u64-xor +u64-false+)
+   (u64-max two-arg-u64-max nil)
+   (u64-min two-arg-u64-min nil)
+   (u64+    two-arg-u64+ 0)
+   ( s8-and two-arg-s8-and +s8-true+)
+   ( s8-or  two-arg-s8-or  +s8-false+)
+   ( s8-xor two-arg-s8-xor +s8-false+)
+   ( s8-max two-arg-s8-max nil)
+   ( s8-min two-arg-s8-min nil)
+   ( s8+    two-arg-s8+ 0)
+   (s16-and two-arg-s16-and +s16-true+)
+   (s16-or  two-arg-s16-or  +s16-false+)
+   (s16-xor two-arg-s16-xor +s16-false+)
+   (s16-max two-arg-s16-max nil)
+   (s16-min two-arg-s16-min nil)
+   (s16+    two-arg-s16+ 0)
+   (s32-and two-arg-s32-and +s32-true+)
+   (s32-or  two-arg-s32-or  +s32-false+)
+   (s32-xor two-arg-s32-xor +s32-false+)
+   (s32-max two-arg-s32-max nil)
+   (s32-min two-arg-s32-min nil)
+   (s32+    two-arg-s32+ 0)
+   (s64-and two-arg-s64-and +s64-true+)
+   (s64-or  two-arg-s64-or  +s64-false+)
+   (s64-xor two-arg-s64-xor +s64-false+)
+   (s64-max two-arg-s64-max nil)
+   (s64-min two-arg-s64-min nil)
+   (s64+    two-arg-s64+ 0))
+  (:comparisons
+   (f32=  two-arg-f32=  u32-and +u32-true+)
+   (f32<  two-arg-f32<  u32-and +u32-true+)
+   (f32<= two-arg-f32<= u32-and +u32-true+)
+   (f32>  two-arg-f32>  u32-and +u32-true+)
+   (f32>= two-arg-f32>= u32-and +u32-true+)
+   (f64=  two-arg-f64=  u64-and +u64-true+)
+   (f64<  two-arg-f64<  u64-and +u64-true+)
+   (f64<= two-arg-f64<= u64-and +u64-true+)
+   (f64>  two-arg-f64>  u64-and +u64-true+)
+   (f64>= two-arg-f64>= u64-and +u64-true+)
+   (u8=  two-arg-u8=  u8-and +u8-true+)
+   (u8<  two-arg-u8<  u8-and +u8-true+)
+   (u8<= two-arg-u8<= u8-and +u8-true+)
+   (u8>  two-arg-u8>  u8-and +u8-true+)
+   (u8>= two-arg-u8>= u8-and +u8-true+)
+   (u16=  two-arg-u16=  u16-and +u16-true+)
+   (u16<  two-arg-u16<  u16-and +u16-true+)
+   (u16<= two-arg-u16<= u16-and +u16-true+)
+   (u16>  two-arg-u16>  u16-and +u16-true+)
+   (u16>= two-arg-u16>= u16-and +u16-true+)
+   (u32=  two-arg-u32=  u32-and +u32-true+)
+   (u32<  two-arg-u32<  u32-and +u32-true+)
+   (u32<= two-arg-u32<= u32-and +u32-true+)
+   (u32>  two-arg-u32>  u32-and +u32-true+)
+   (u32>= two-arg-u32>= u32-and +u32-true+)
+   (u64=  two-arg-u64=  u64-and +u64-true+)
+   (u64<  two-arg-u64<  u64-and +u64-true+)
+   (u64<= two-arg-u64<= u64-and +u64-true+)
+   (u64>  two-arg-u64>  u64-and +u64-true+)
+   (u64>= two-arg-u64>= u64-and +u64-true+)
+   (s8=  two-arg-s8=  u8-and +u8-true+)
+   (s8<  two-arg-s8<  u8-and +u8-true+)
+   (s8<= two-arg-s8<= u8-and +u8-true+)
+   (s8>  two-arg-s8>  u8-and +u8-true+)
+   (s8>= two-arg-s8>= u8-and +u8-true+)
+   (s16=  two-arg-s16=  u16-and +u16-true+)
+   (s16<  two-arg-s16<  u16-and +u16-true+)
+   (s16<= two-arg-s16<= u16-and +u16-true+)
+   (s16>  two-arg-s16>  u16-and +u16-true+)
+   (s16>= two-arg-s16>= u16-and +u16-true+)
+   (s32=  two-arg-s32=  u32-and +u32-true+)
+   (s32<  two-arg-s32<  u32-and +u32-true+)
+   (s32<= two-arg-s32<= u32-and +u32-true+)
+   (s32>  two-arg-s32>  u32-and +u32-true+)
+   (s32>= two-arg-s32>= u32-and +u32-true+)
+   (s64=  two-arg-s64=  u64-and +u64-true+)
+   (s64<  two-arg-s64<  u64-and +u64-true+)
+   (s64<= two-arg-s64<= u64-and +u64-true+)
+   (s64>  two-arg-s64>  u64-and +u64-true+)
+   (s64>= two-arg-s64>= u64-and +u64-true+))
+  (:reducers
+   (f32- two-arg-f32- 0f0)
+   (f32/ two-arg-f32/ 1f0)
+   (f64- two-arg-f64- 0d0)
+   (f64/ two-arg-f64/ 1d0)
+   (u8- two-arg-u8- 0)
+   (u16- two-arg-u16- 0)
+   (u32- two-arg-u32- 0)
+   (u64- two-arg-u64- 0)
+   (s8- two-arg-s8- 0)
+   (s16- two-arg-s16- 0)
+   (s32- two-arg-s32- 0)
+   (s64- two-arg-s64- 0))
+  (:unequals
+   (f32/= two-arg-f32/= u32-and +u32-true+)
+   (f64/= two-arg-f64/= u64-and +u64-true+)
+   (u8/= two-arg-u8/= u8-and +u8-true+)
+   (u16/= two-arg-u16/= u16-and +u16-true+)
+   (u32/= two-arg-u32/= u32-and +u32-true+)
+   (u64/= two-arg-u64/= u64-and +u64-true+)
+   (s8/= two-arg-s8/= u8-and +u8-true+)
+   (s16/= two-arg-s16/= u16-and +u16-true+)
+   (s32/= two-arg-s32/= u32-and +u32-true+)
+   (s64/= two-arg-s64/= u64-and +u64-true+)))

contrib/sb-simd/code/instruction-sets/sse.lisp

@@ -0,0 +1,105 @@
+(in-package #:sb-simd-sse)
+
+(define-instruction-set :sse
+  (:test (sse-supported-p))
+  (:include :x86-64)
+  (:scalars
+   (f32 32 single-float #:single-float (#:single-reg)))
+  (:simd-packs
+   (p128   nil 128 #:simd-pack (#:int-sse-reg #:double-sse-reg #:single-sse-reg))
+   (f32.4  f32 128 #:simd-pack-single (#:single-sse-reg)))
+  (:simd-casts
+   (f32.4 f32.4-broadcast))
+  (:reinterpret-casts
+   (f32!   f32!-from-p128)
+   (f32.4! f32.4!-from-f32))
+  (:instructions
+   ;; f32
+   (f32-from-s64    nil       (f32) (s64)     :cost 5 :encoding :custom)
+   (f32!-from-p128  nil       (f32) (p128)    :cost 1 :encoding :custom :always-translatable nil)
+   (two-arg-f32-and #:andps   (f32) (f32 f32) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32-or  #:orps    (f32) (f32 f32) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32-xor #:xorps   (f32) (f32 f32) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32-max #:maxss   (f32) (f32 f32) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32-min #:minss   (f32) (f32 f32) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32+    #:addss   (f32) (f32 f32) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32-    #:subss   (f32) (f32 f32) :cost 2 :encoding :sse)
+   (two-arg-f32*    #:mulss   (f32) (f32 f32) :cost 2 :encoding :sse :associative t)
+   (two-arg-f32/    #:divss   (f32) (f32 f32) :cost 8 :encoding :sse)
+   (two-arg-f32=    #:cmpss   (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:eq) :associative t)
+   (two-arg-f32/=   #:cmpss   (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:neq) :associative t)
+   (two-arg-f32<    #:cmpss   (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:lt))
+   (two-arg-f32<=   #:cmpss   (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:le))
+   (two-arg-f32>    #:cmpss   (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:nle))
+   (two-arg-f32>=   #:cmpss   (u32) (f32 f32) :cost 4 :encoding :custom :prefix '(:nlt))
+   (f32-andc1       #:andnps  (f32) (f32 f32) :cost 1 :encoding :sse)
+   (f32-not         nil       (f32) (f32)     :cost 1 :encoding :fake-vop)
+   (f32-reciprocal  #:rcpss   (f32) (f32)     :cost 5)
+   (f32-rsqrt       #:rsqrtss (f32) (f32)     :cost 5)
+   (f32-sqrt        #:sqrtss  (f32) (f32)     :cost 15)
+   ;; f32.4
+   (f32.4!-from-f32   #:movups   (f32.4) (f32)         :cost 1 :encoding :move)
+   (make-f32.4        nil        (f32.4) (f32 f32 f32 f32) :cost 1 :encoding :fake-vop)
+   (f32.4-values      nil        (f32 f32 f32 f32) (f32.4) :cost 1 :encoding :fake-vop)
+   (f32.4-broadcast   nil        (f32.4) (f32)         :cost 1 :encoding :fake-vop)
+   (two-arg-f32.4-and #:andps    (f32.4) (f32.4 f32.4) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32.4-or  #:orps     (f32.4) (f32.4 f32.4) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32.4-xor #:xorps    (f32.4) (f32.4 f32.4) :cost 1 :encoding :sse :associative t)
+   (two-arg-f32.4-max #:maxps    (f32.4) (f32.4 f32.4) :cost 3 :encoding :sse :associative t)
+   (two-arg-f32.4-min #:minps    (f32.4) (f32.4 f32.4) :cost 3 :encoding :sse :associative t)
+   (two-arg-f32.4+    #:addps    (f32.4) (f32.4 f32.4) :cost 2 :encoding :sse :associative t)
+   (two-arg-f32.4-    #:subps    (f32.4) (f32.4 f32.4) :cost 2 :encoding :sse)
+   (two-arg-f32.4*    #:mulps    (f32.4) (f32.4 f32.4) :cost 2 :encoding :sse :associative t)
+   (two-arg-f32.4/    #:divps    (f32.4) (f32.4 f32.4) :cost 8 :encoding :sse)
+   (f32.4-horizontal-and nil     (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal-or  nil     (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal-xor nil     (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal-max nil     (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal-min nil     (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal+ nil        (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-horizontal* nil        (f32)   (f32.4)       :cost 5 :encoding :fake-vop)
+   (f32.4-andc1       #:andnps   (f32.4) (f32.4 f32.4) :cost 1 :encoding :sse)
+   (f32.4-not         nil        (f32.4) (f32.4)       :cost 1 :encoding :fake-vop)
+   (f32.4-reciprocal  #:rcpps    (f32.4) (f32.4)       :cost 5)
+   (f32.4-rsqrt       #:rsqrtps  (f32.4) (f32.4)       :cost 5)
+   (f32.4-sqrt        #:sqrtps   (f32.4) (f32.4)       :cost 15)
+   (f32.4-shuffle     #:shufps   (f32.4) (f32.4 f32.4 sb-simd-x86-64::imm8) :cost 1 :encoding :sse)
+   (f32.4-unpacklo    #:unpcklps (f32.4) (f32.4 f32.4) :cost 1 :encoding :sse)
+   (f32.4-unpackhi    #:unpckhps (f32.4) (f32.4 f32.4) :cost 1 :encoding :sse)
+   (f32.4-movemask    #:movmskps (u4)    (f32.4)       :cost 1))
+  (:loads
+   (f32-load          #:movss   f32   f32vec f32-array f32-aref   f32-row-major-aref)
+   (f32.4-load        #:movups  f32.4 f32vec f32-array f32.4-aref f32.4-row-major-aref))
+  (:stores
+   (f32-store         #:movss   f32   f32vec f32-array f32-aref   f32-row-major-aref)
+   (f32.4-store       #:movups  f32.4 f32vec f32-array f32.4-aref f32.4-row-major-aref)
+   (f32.4-ntstore     #:movntps f32.4 f32vec f32-array f32.4-non-temporal-aref f32.4-non-temporal-row-major-aref))
+  (:associatives
+   (f32-and two-arg-f32-and +f32-true+)
+   (f32-or  two-arg-f32-or  +f32-false+)
+   (f32-xor two-arg-f32-xor +f32-false+)
+   (f32-max two-arg-f32-max nil)
+   (f32-min two-arg-f32-min nil)
+   (f32+    two-arg-f32+ 0f0)
+   (f32*    two-arg-f32* 1f0)
+   (f32.4-and two-arg-f32.4-and +f32-true+)
+   (f32.4-or  two-arg-f32.4-or  +f32-false+)
+   (f32.4-xor two-arg-f32.4-xor +f32-false+)
+   (f32.4-max two-arg-f32.4-max nil)
+   (f32.4-min two-arg-f32.4-min nil)
+   (f32.4+    two-arg-f32.4+ 0f0)
+   (f32.4*    two-arg-f32.4* 1f0))
+  (:comparisons
+   (f32=  two-arg-f32=  u32-and +u32-true+)
+   (f32<  two-arg-f32<  u32-and +u32-true+)
+   (f32<= two-arg-f32<= u32-and +u32-true+)
+   (f32>  two-arg-f32>  u32-and +u32-true+)
+   (f32>= two-arg-f32>= u32-and +u32-true+))
+  (:reducers
+   (f32- two-arg-f32- 0f0)
+   (f32/ two-arg-f32/ 1f0)
+   (f32.4- two-arg-f32.4- 0f0)
+   (f32.4/ two-arg-f32.4/ 1f0))
+  (:unequals
+   (f32/= two-arg-f32/= u32-and +u32-true+)))
+

contrib/sb-simd/code/instruction-sets/sse2.lisp

@@ -0,0 +1,459 @@
+(in-package #:sb-simd-sse2)
+
+(define-instruction-set :sse2
+  (:include :sse)
+  (:test (sse2-supported-p))
+  (:scalars
+   (f64 64 double-float #:double-float (#:double-reg)))
+  (:simd-packs
+   (f64.2 f64 128 #:simd-pack-double (#:double-sse-reg))
+   (u8.16 u8  128 #:simd-pack-ub8  (#:int-sse-reg))
+   (u16.8 u16 128 #:simd-pack-ub16 (#:int-sse-reg))
+   (u32.4 u32 128 #:simd-pack-ub32 (#:int-sse-reg))
+   (u64.2 u64 128 #:simd-pack-ub64 (#:int-sse-reg))
+   (s8.16 s8  128 #:simd-pack-sb8  (#:int-sse-reg))
+   (s16.8 s16 128 #:simd-pack-sb16 (#:int-sse-reg))
+   (s32.4 s32 128 #:simd-pack-sb32 (#:int-sse-reg))
+   (s64.2 s64 128 #:simd-pack-sb64 (#:int-sse-reg)))
+  (:simd-casts
+   (f64.2 f64.2-broadcast)
+   (u8.16 u8.16-broadcast)
+   (u16.8 u16.8-broadcast)
+   (u32.4 u32.4-broadcast)
+   (u64.2 u64.2-broadcast)
+   (s8.16 s8.16-broadcast)
+   (s16.8 s16.8-broadcast)
+   (s32.4 s32.4-broadcast)
+   (s64.2 s64.2-broadcast))
+  (:reinterpret-casts
+   (f64!   f64!-from-p128)
+   (u8!    u8!-from-p128)
+   (u16!   u16!-from-p128)
+   (u32!   u32!-from-p128)
+   (u64!   u64!-from-p128)
+   (f64.2! f64.2!-from-f64 f64.2!-from-p128)
+   (u8.16! u8.16!-from-u8 u8.16!-from-p128)
+   (u16.8! u16.8!-from-u16 u16.8!-from-p128)
+   (u32.4! u32.4!-from-u32 u32.4!-from-p128)
+   (u64.2! u64.2!-from-u64 u64.2!-from-p128)
+   (s8.16! s8.16!-from-s8 s8.16!-from-p128)
+   (s16.8! s16.8!-from-s16 s16.8!-from-p128)
+   (s32.4! s32.4!-from-s32 s32.4!-from-p128)
+   (s64.2! s64.2!-from-s64 s64.2!-from-p128))
+  (:instructions
+   ;; f32
+   (f32-from-f64      #:cvtsd2ss   (f32) (f64)     :cost 5)
+   ;; f64
+   (f64-from-f32      #:cvtss2sd   (f64) (f32)     :cost 5)
+   (f64-from-s64      nil          (f64) (s64)     :cost 5 :encoding :custom)
+   (f64!-from-p128    nil          (f64) (p128)    :cost 1 :encoding :custom :always-translatable nil)
+   (two-arg-f64-and   #:andpd      (f64) (f64 f64) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64-or    #:orpd       (f64) (f64 f64) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64-xor   #:xorpd      (f64) (f64 f64) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64-max   #:maxsd      (f64) (f64 f64) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64-min   #:minsd      (f64) (f64 f64) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64+      #:addsd      (f64) (f64 f64) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64-      #:subsd      (f64) (f64 f64) :cost 2 :encoding :sse)
+   (two-arg-f64*      #:mulsd      (f64) (f64 f64) :cost 2 :encoding :sse :associative t)
+   (two-arg-f64/      #:divsd      (f64) (f64 f64) :cost 8 :encoding :sse)
+   (two-arg-f64=      #:cmpsd      (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:eq) :associative t)
+   (two-arg-f64/=     #:cmpsd      (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:neq) :associative t)
+   (two-arg-f64<      #:cmpsd      (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:lt))
+   (two-arg-f64<=     #:cmpsd      (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:le))
+   (two-arg-f64>      #:cmpsd      (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:nle))
+   (two-arg-f64>=     #:cmpsd      (u64) (f64 f64) :cost 4 :encoding :custom :prefix '(:nlt))
+   (f64-andc1         #:andnpd     (f64) (f64 f64) :cost 1 :encoding :sse)
+   (f64-not           nil          (f64) (f64)     :cost 1 :encoding :fake-vop)
+   (f64.2-hsum        nil          (f64) (f64.2)   :cost 3 :encoding :fake-vop)
+   (f64.2-hprod       nil          (f64) (f64.2)   :cost 3 :encoding :fake-vop)
+   (f64-sqrt          #:sqrtsd     (f64) (f64)     :cost 15)
+   ;; scalar reinterpret casts
+   ( u8!-from-p128    nil          (u8)    (p128)        :cost 1 :encoding :fake-vop)
+   (u16!-from-p128    nil          (u16)   (p128)        :cost 1 :encoding :fake-vop)
+   (u32!-from-p128    nil          (u32)   (p128)        :cost 1 :encoding :fake-vop)
+   (u64!-from-p128    #:movq       (u64)   (p128)        :cost 1 :always-translatable nil)
+   ;; f32.4
+   (f32.4-from-s32.4  #:cvtdq2ps   (f32.4) (s32.4)       :cost 5)
+   (f32.4!-from-p128  #:movups     (f32.4) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (two-arg-f32.4=    #:cmpps      (u32.4) (f32.4 f32.4) :cost 4 :encoding :sse :prefix '(:eq) :associative t)
+   (two-arg-f32.4/=   #:cmpps      (u32.4) (f32.4 f32.4) :cost 4 :encoding :sse :prefix '(:neq) :associative t)
+   (two-arg-f32.4<    #:cmpps      (u32.4) (f32.4 f32.4) :cost 4 :encoding :sse :prefix '(:lt))
+   (two-arg-f32.4<=   #:cmpps      (u32.4) (f32.4 f32.4) :cost 4 :encoding :sse :prefix '(:le))
+   (two-arg-f32.4>    #:cmpps      (u32.4) (f32.4 f32.4) :cost 4 :encoding :sse :prefix '(:nle))
+   (two-arg-f32.4>=   #:cmpps      (u32.4) (f32.4 f32.4) :cost 4 :encoding :sse :prefix '(:nlt))
+   ;; f64.2
+   (f64.2!-from-f64   #:movupd     (f64.2) (f64)         :cost 1 :encoding :move)
+   (f64.2!-from-p128  #:movupd     (f64.2) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-f64.2        nil          (f64.2) (f64 f64)     :cost 1 :encoding :fake-vop)
+   (f64.2-values      nil          (f64 f64) (f64.2)     :cost 1 :encoding :fake-vop)
+   (f64.2-broadcast   nil          (f64.2) (f64)         :cost 1 :encoding :fake-vop)
+   (two-arg-f64.2-and #:andpd      (f64.2) (f64.2 f64.2) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64.2-or  #:orpd       (f64.2) (f64.2 f64.2) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64.2-xor #:xorpd      (f64.2) (f64.2 f64.2) :cost 1 :encoding :sse :associative t)
+   (two-arg-f64.2-max #:maxpd      (f64.2) (f64.2 f64.2) :cost 3 :encoding :sse :associative t)
+   (two-arg-f64.2-min #:minpd      (f64.2) (f64.2 f64.2) :cost 3 :encoding :sse :associative t)
+   (two-arg-f64.2+    #:addpd      (f64.2) (f64.2 f64.2) :cost 2 :encoding :sse :associative t)
+   (two-arg-f64.2-    #:subpd      (f64.2) (f64.2 f64.2) :cost 2 :encoding :sse)
+   (two-arg-f64.2*    #:mulpd      (f64.2) (f64.2 f64.2) :cost 2 :encoding :sse :associative t)
+   (two-arg-f64.2/    #:divpd      (f64.2) (f64.2 f64.2) :cost 8 :encoding :sse)
+   (two-arg-f64.2=    #:cmppd      (u64.2) (f64.2 f64.2) :cost 4 :encoding :sse :prefix '(:eq) :associative t)
+   (two-arg-f64.2/=   #:cmppd      (u64.2) (f64.2 f64.2) :cost 4 :encoding :sse :prefix '(:neq) :associative t)
+   (two-arg-f64.2<    #:cmppd      (u64.2) (f64.2 f64.2) :cost 4 :encoding :sse :prefix '(:lt))
+   (two-arg-f64.2<=   #:cmppd      (u64.2) (f64.2 f64.2) :cost 4 :encoding :sse :prefix '(:le))
+   (two-arg-f64.2>    #:cmppd      (u64.2) (f64.2 f64.2) :cost 4 :encoding :sse :prefix '(:nle))
+   (two-arg-f64.2>=   #:cmppd      (u64.2) (f64.2 f64.2) :cost 4 :encoding :sse :prefix '(:nlt))
+   (f64.2-horizontal-and nil       (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal-or  nil       (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal-xor nil       (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal-max nil       (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal-min nil       (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal+    nil       (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-horizontal*    nil       (f64)   (f64.2)       :cost 3 :encoding :fake-vop)
+   (f64.2-andc1       #:andnpd     (f64.2) (f64.2 f64.2) :cost 1 :encoding :sse)
+   (f64.2-not         nil          (f64.2) (f64.2)       :cost 1 :encoding :fake-vop)
+   (f64.2-sqrt        #:sqrtpd     (f64.2) (f64.2)       :cost 20)
+   (f64.2-unpackhi    #:unpckhpd   (f64.2) (f64.2 f64.2) :cost 1 :encoding :sse)
+   (f64.2-unpacklo    #:unpcklpd   (f64.2) (f64.2 f64.2) :cost 1 :encoding :sse)
+   (f64.2-shuffle     #:shufpd     (f64.2) (f64.2 imm2)  :cost 1)
+   (f64.2-movemask    #:movmskpd   (u2)    (f64.2)       :cost 1)
+   ;; u8.16
+   (u8.16!-from-u8    #:movq       (u8.16) (u8)          :cost 1)
+   (u8.16!-from-p128  #:movdqu     (u8.16) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-u8.16        nil          (u8.16) (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) :cost 1 :encoding :fake-vop)
+   (u8.16-values      nil          (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) (u8.16) :cost 1 :encoding :fake-vop)
+   (u8.16-broadcast   nil          (u8.16) (u8)          :cost 1 :encoding :fake-vop)
+   (two-arg-u8.16-and #:pand       (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse :associative t)
+   (two-arg-u8.16-or  #:por        (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse :associative t)
+   (two-arg-u8.16-xor #:pxor       (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse :associative t)
+   (u8.16-andc1       #:pandn      (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse)
+   (u8.16-not         nil          (u8.16) (u8.16)       :cost 1 :encoding :fake-vop)
+   (two-arg-u8.16+    #:paddb      (u8.16) (u8.16 u8.16) :cost 2 :encoding :sse :associative t)
+   (two-arg-u8.16-    #:psubb      (u8.16) (u8.16 u8.16) :cost 2 :encoding :sse)
+   (two-arg-u8.16=    #:pcmpeqb    (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse)
+   (two-arg-u8.16/=   nil          (u8.16) (u8.16 u8.16) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u8.16>~   #:pcmpgtb    (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse)
+   (two-arg-u8.16>    nil          (u8.16) (u8.16 u8.16) :cost 1 :encoding :fake-vop)
+   (two-arg-u8.16<    nil          (u8.16) (u8.16 u8.16) :cost 1 :encoding :fake-vop)
+   (two-arg-u8.16>=   nil          (u8.16) (u8.16 u8.16) :cost 2 :encoding :fake-vop)
+   (two-arg-u8.16<=   nil          (u8.16) (u8.16 u8.16) :cost 2 :encoding :fake-vop)
+   (u8.16-average     #:pavgw      (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse)
+   (u8.16-unpackhi    #:punpckhbw  (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse)
+   (u8.16-unpacklo    #:punpcklbw  (u8.16) (u8.16 u8.16) :cost 1 :encoding :sse)
+   (u8.16-movemask    #:pmovmskb   (u16)   (u8.16)       :cost 1)
+   ;; u16.8
+   (u16.8!-from-u16   #:movq       (u16.8) (u16)         :cost 1)
+   (u16.8!-from-p128  #:movdqu     (u16.8) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-u16.8        nil          (u16.8) (u16 u16 u16 u16 u16 u16 u16 u16) :cost 1 :encoding :fake-vop)
+   (u16.8-values      nil          (u16 u16 u16 u16 u16 u16 u16 u16) (u16.8) :cost 1 :encoding :fake-vop)
+   (u16.8-broadcast   nil          (u16.8) (u16)         :cost 1 :encoding :fake-vop)
+   (two-arg-u16.8-and #:pand       (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse :associative t)
+   (two-arg-u16.8-or  #:por        (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse :associative t)
+   (two-arg-u16.8-xor #:pxor       (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse :associative t)
+   (u16.8-andc1       #:pandn      (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse)
+   (u16.8-not         nil          (u16.8) (u16.8)       :cost 1 :encoding :fake-vop)
+   (two-arg-u16.8+    #:paddw      (u16.8) (u16.8 u16.8) :cost 2 :encoding :sse :associative t)
+   (two-arg-u16.8-    #:psubw      (u16.8) (u16.8 u16.8) :cost 2 :encoding :sse)
+   (two-arg-u16.8=    #:pcmpeqw    (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse)
+   (two-arg-u16.8/=   nil          (u16.8) (u16.8 u16.8) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u16.8>~   #:pcmpgtw    (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse)
+   (two-arg-u16.8>    nil          (u16.8) (u16.8 u16.8) :cost 1 :encoding :fake-vop)
+   (two-arg-u16.8<    nil          (u16.8) (u16.8 u16.8) :cost 1 :encoding :fake-vop)
+   (two-arg-u16.8>=   nil          (u16.8) (u16.8 u16.8) :cost 2 :encoding :fake-vop)
+   (two-arg-u16.8<=   nil          (u16.8) (u16.8 u16.8) :cost 2 :encoding :fake-vop)
+   (u16.8-average     #:pavgb      (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse)
+   (u16.8-unpackhi    #:punpckhwd  (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse)
+   (u16.8-unpacklo    #:punpcklwd  (u16.8) (u16.8 u16.8) :cost 1 :encoding :sse)
+   (u16.8-movemask    nil          (u8)    (u16.8)       :cost 1 :encoding :fake-vop)
+   (u16.8-elt         #:pextrw     (u16)   (u16.8 imm3)  :cost 1)
+   (u16.8-shufflehi   #:pshufhw    (u16.8) (u16.8 imm8)  :cost 1)
+   (u16.8-shufflelo   #:pshuflw    (u16.8) (u16.8 imm8)  :cost 1)
+   (u16.8-shiftl      #:psllw-imm  (u16.8) (u16.8 imm4)  :cost 1 :encoding :sse)
+   (u16.8-shiftr      #:psrlw-imm  (u16.8) (u16.8 imm4)  :cost 1 :encoding :sse)
+   ;; u32.4
+   (u32.4!-from-u32   #:movq       (u32.4) (u16)         :cost 1)
+   (u32.4!-from-p128  #:movdqu     (u32.4) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-u32.4        nil          (u32.4) (u32 u32 u32 u32) :cost 1 :encoding :fake-vop)
+   (u32.4-values      nil          (u32 u32 u32 u32) (u32.4) :cost 1 :encoding :fake-vop)
+   (u32.4-broadcast   nil          (u32.4) (u32)         :cost 1 :encoding :fake-vop)
+   (two-arg-u32.4-and #:pand       (u32.4) (u32.4 u32.4) :cost 1 :encoding :sse :associative t)
+   (two-arg-u32.4-or  #:por        (u32.4) (u32.4 u32.4) :cost 1 :encoding :sse :associative t)
+   (two-arg-u32.4-xor #:pxor       (u32.4) (u32.4 u32.4) :cost 1 :encoding :sse :associative t)
+   (u32.4-andc1       #:pandn      (u32.4) (u32.4 u32.4) :cost 1 :encoding :sse)
+   (u32.4-not         nil          (u32.4) (u32.4)       :cost 1 :encoding :fake-vop)
+   (two-arg-u32.4+    #:paddd      (u32.4) (u32.4 u32.4) :cost 2 :encoding :sse :associative t)
+   (two-arg-u32.4-    #:psubd      (u32.4) (u32.4 u32.4) :cost 2 :encoding :sse)
+   (two-arg-u32.4=    #:pcmpeqd    (u32.4) (u32.4 u32.4) :cost 1 :encoding :sse)
+   (two-arg-u32.4/=   nil          (u32.4) (u32.4 u32.4) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-u32.4>~   #:pcmpgtd    (u32.4) (u32.4 u32.4) :cost 1 :encoding :sse)
+   (two-arg-u32.4>    nil          (u32.4) (u32.4 u32.4) :cost 1 :encoding :fake-vop)
+   (two-arg-u32.4<    nil          (u32.4) (u32.4 u32.4) :cost 1 :encoding :fake-vop)
+   (two-arg-u32.4>=   nil          (u32.4) (u32.4 u32.4) :cost 2 :encoding :fake-vop)
+   (two-arg-u32.4<=   nil          (u32.4) (u32.4 u32.4) :cost 2 :encoding :fake-vop)
+   (u32.4-unpackhi    #:punpckhdq  (u32.4) (u32.4 u32.4) :cost 1 :encoding :sse)
+   (u32.4-unpacklo    #:punpckldq  (u32.4) (u32.4 u32.4) :cost 1 :encoding :sse)
+   (u32.4-movemask    #:movmskps   (u4)    (u32.4)       :cost 1)
+   (u32.4-shuffle     #:pshufd     (u32.4) (u32.4 imm8)  :cost 1)
+   (u32.4-shiftl      #:pslld-imm  (u32.4) (u32.4 imm5)  :cost 1 :encoding :sse)
+   (u32.4-shiftr      #:psrld-imm  (u32.4) (u32.4 imm5)  :cost 1 :encoding :sse)
+   ;; u64.2
+   (u64.2!-from-u64   #:movq       (u64.2) (u64)         :cost 1)
+   (u64.2!-from-p128  #:movdqu     (u64.2) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-u64.2        nil          (u64.2) (u64 u64)     :cost 1 :encoding :fake-vop)
+   (u64.2-values      nil          (u64 u64) (u64.2)     :cost 1 :encoding :fake-vop)
+   (u64.2-broadcast   nil          (u64.2) (u64)         :cost 1 :encoding :fake-vop)
+   (two-arg-u64.2-and #:pand       (u64.2) (u64.2 u64.2) :cost 1 :encoding :sse :associative t)
+   (two-arg-u64.2-or  #:por        (u64.2) (u64.2 u64.2) :cost 1 :encoding :sse :associative t)
+   (two-arg-u64.2-xor #:pxor       (u64.2) (u64.2 u64.2) :cost 1 :encoding :sse :associative t)
+   (u64.2-andc1       #:pandn      (u64.2) (u64.2 u64.2) :cost 1 :encoding :sse)
+   (u64.2-not         nil          (u64.2) (u64.2)       :cost 1 :encoding :fake-vop)
+   (two-arg-u64.2+    #:paddq      (u64.2) (u64.2 u64.2) :cost 2 :encoding :sse :associative t)
+   (two-arg-u64.2-    #:psubq      (u64.2) (u64.2 u64.2) :cost 2 :encoding :sse)
+   (u64.2-unpackhi    #:punpckhqdq (u64.2) (u64.2 u64.2) :cost 1 :encoding :sse)
+   (u64.2-unpacklo    #:punpcklqdq (u64.2) (u64.2 u64.2) :cost 1 :encoding :sse)
+   (u64.2-movemask    #:movmskpd   (u2)    (u64.2)       :cost 1)
+   (u64.2-shiftl      #:psllq-imm  (u64.2) (u64.2 imm6)  :cost 1 :encoding :sse)
+   (u64.2-shiftr      #:psrlq-imm  (u64.2) (u64.2 imm6)  :cost 1 :encoding :sse)
+   ;; s8.16
+   (s8.16!-from-s8    nil          (s8.16) (s8)          :cost 1 :encoding :fake-vop)
+   (s8.16!-from-p128  #:movdqu     (s8.16) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-s8.16        nil          (s8.16) (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) :cost 1 :encoding :fake-vop)
+   (s8.16-values      nil          (s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8 s8) (s8.16) :cost 1 :encoding :fake-vop)
+   (s8.16-broadcast   nil          (s8.16) (s8)          :cost 1 :encoding :fake-vop)
+   (two-arg-s8.16-and #:pand       (s8.16) (s8.16 s8.16) :cost 1 :encoding :sse :associative t)
+   (two-arg-s8.16-or  #:por        (s8.16) (s8.16 s8.16) :cost 1 :encoding :sse :associative t)
+   (two-arg-s8.16-xor #:pxor       (s8.16) (s8.16 s8.16) :cost 1 :encoding :sse :associative t)
+   (s8.16-andc1       #:pandn      (s8.16) (s8.16 s8.16) :cost 1 :encoding :sse)
+   (s8.16-not         nil          (s8.16) (s8.16)       :cost 1 :encoding :fake-vop)
+   (two-arg-s8.16+    #:paddb      (s8.16) (s8.16 s8.16) :cost 2 :encoding :sse :associative t)
+   (two-arg-s8.16-    #:psubb      (s8.16) (s8.16 s8.16) :cost 2 :encoding :sse)
+   (two-arg-s8.16=    #:pcmpeqb    (u8.16) (s8.16 s8.16) :cost 1 :encoding :sse)
+   (two-arg-s8.16/=   nil          (u8.16) (s8.16 s8.16) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s8.16>    #:pcmpgtb    (u8.16) (s8.16 s8.16) :cost 1 :encoding :sse)
+   (two-arg-s8.16<    nil          (u8.16) (s8.16 s8.16) :cost 1 :encoding :fake-vop)
+   (two-arg-s8.16>=   nil          (u8.16) (s8.16 s8.16) :cost 2 :encoding :fake-vop)
+   (two-arg-s8.16<=   nil          (u8.16) (s8.16 s8.16) :cost 2 :encoding :fake-vop)
+   (s8.16-unpackhi    #:punpckhbw  (s8.16) (s8.16 s8.16) :cost 1 :encoding :sse)
+   (s8.16-unpacklo    #:punpcklbw  (s8.16) (s8.16 s8.16) :cost 1 :encoding :sse)
+   (s8.16-movemask    #:pmovmskb   (u16)   (s8.16)       :cost 1)
+   ;; s16.8
+   (s16.8!-from-s16   nil          (s16.8) (s16)         :cost 1 :encoding :fake-vop)
+   (s16.8!-from-p128  #:movdqu     (s16.8) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-s16.8        nil          (s16.8) (s16 s16 s16 s16 s16 s16 s16 s16) :cost 1 :encoding :fake-vop)
+   (s16.8-values      nil          (s16 s16 s16 s16 s16 s16 s16 s16) (s16.8) :cost 1 :encoding :fake-vop)
+   (s16.8-broadcast   nil          (s16.8) (s16)         :cost 1 :encoding :fake-vop)
+   (two-arg-s16.8-and #:pand       (s16.8) (s16.8 s16.8) :cost 1 :encoding :sse :associative t)
+   (two-arg-s16.8-or  #:por        (s16.8) (s16.8 s16.8) :cost 1 :encoding :sse :associative t)
+   (two-arg-s16.8-xor #:pxor       (s16.8) (s16.8 s16.8) :cost 1 :encoding :sse :associative t)
+   (s16.8-andc1       #:pandn      (s16.8) (s16.8 s16.8) :cost 1 :encoding :sse)
+   (s16.8-not         nil          (s16.8) (s16.8)       :cost 1 :encoding :fake-vop)
+   (two-arg-s16.8+    #:paddw      (s16.8) (s16.8 s16.8) :cost 2 :encoding :sse :associative t)
+   (two-arg-s16.8-    #:psubw      (s16.8) (s16.8 s16.8) :cost 2 :encoding :sse)
+   (two-arg-s16.8=    #:pcmpeqw    (u16.8) (s16.8 s16.8) :cost 1 :encoding :sse)
+   (two-arg-s16.8/=   nil          (u16.8) (s16.8 s16.8) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s16.8>    #:pcmpgtw    (u16.8) (s16.8 s16.8) :cost 1 :encoding :sse)
+   (two-arg-s16.8<    nil          (u16.8) (s16.8 s16.8) :cost 1 :encoding :fake-vop)
+   (two-arg-s16.8>=   nil          (u16.8) (s16.8 s16.8) :cost 2 :encoding :fake-vop)
+   (two-arg-s16.8<=   nil          (u16.8) (s16.8 s16.8) :cost 2 :encoding :fake-vop)
+   (s16.8-unpackhi    #:punpckhwd  (s16.8) (s16.8 s16.8) :cost 1 :encoding :sse)
+   (s16.8-unpacklo    #:punpcklwd  (s16.8) (s16.8 s16.8) :cost 1 :encoding :sse)
+   (s16.8-movemask    nil          (u8)    (s16.8)       :cost 1 :encoding :fake-vop)
+   (s16.8-mullo       #:pmullw     (s16.8) (s16.8 s16.8) :cost 1 :encoding :sse)
+   (s16.8-elt         #:pextrw     (s16)   (s16.8 imm3)  :cost 1)
+   (s16.8-shufflehi   #:pshufhw    (s16.8) (s16.8 imm8)  :cost 1)
+   (s16.8-shufflelo   #:pshuflw    (s16.8) (s16.8 imm8)  :cost 1)
+   (s16.8-shiftl      #:psllw-imm  (s16.8) (s16.8 imm4)  :cost 1 :encoding :sse)
+   (s16.8-shiftr      #:psrlw-imm  (s16.8) (s16.8 imm4)  :cost 1 :encoding :sse)
+   ;; s32.4
+   (s32.4!-from-s32   nil          (s32.4) (s32)         :cost 1 :encoding :fake-vop)
+   (s32.4!-from-p128  #:movdqu     (s32.4) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-s32.4        nil          (s32.4) (s32 s32 s32 s32) :cost 1 :encoding :fake-vop)
+   (s32.4-values      nil          (s32 s32 s32 s32) (s32.4) :cost 1 :encoding :fake-vop)
+   (s32.4-broadcast   nil          (s32.4) (s32)         :cost 1 :encoding :fake-vop)
+   (s32.4-from-f32.4  #:cvtps2dq   (s32.4) (f32.4)       :cost 5)
+   (s32.4-from-f64.2  #:cvtpd2dq   (s32.4) (f64.2)       :cost 1)
+   (two-arg-s32.4-and #:pand       (s32.4) (s32.4 s32.4) :cost 1 :encoding :sse :associative t)
+   (two-arg-s32.4-or  #:por        (s32.4) (s32.4 s32.4) :cost 1 :encoding :sse :associative t)
+   (two-arg-s32.4-xor #:pxor       (s32.4) (s32.4 s32.4) :cost 1 :encoding :sse :associative t)
+   (s32.4-andc1       #:pandn      (s32.4) (s32.4 s32.4) :cost 1 :encoding :sse)
+   (s32.4-not         nil          (s32.4) (s32.4)       :cost 1 :encoding :fake-vop)
+   (two-arg-s32.4+    #:paddd      (s32.4) (s32.4 s32.4) :cost 2 :encoding :sse :associative t)
+   (two-arg-s32.4-    #:psubd      (s32.4) (s32.4 s32.4) :cost 2 :encoding :sse)
+   (two-arg-s32.4=    #:pcmpeqd    (u32.4) (s32.4 s32.4) :cost 1 :encoding :sse)
+   (two-arg-s32.4/=   nil          (u32.4) (s32.4 s32.4) :cost 2 :associative t :encoding :fake-vop)
+   (two-arg-s32.4>    #:pcmpgtd    (u32.4) (s32.4 s32.4) :cost 1 :encoding :sse)
+   (two-arg-s32.4<    nil          (u32.4) (s32.4 s32.4) :cost 1 :encoding :fake-vop)
+   (two-arg-s32.4>=   nil          (u32.4) (s32.4 s32.4) :cost 2 :encoding :fake-vop)
+   (two-arg-s32.4<=   nil          (u32.4) (s32.4 s32.4) :cost 2 :encoding :fake-vop)
+   (s32.4-unpackhi    #:punpckhdq  (s32.4) (s32.4 s32.4) :cost 1 :encoding :sse)
+   (s32.4-unpacklo    #:punpckldq  (s32.4) (s32.4 s32.4) :cost 1 :encoding :sse)
+   (s32.4-movemask    #:movmskps   (u4)    (s32.4)       :cost 1)
+   (s32.4-shuffle     #:pshufd     (s32.4) (s32.4 imm8)  :cost 1)
+   (s32.4-shiftl      #:pslld-imm  (s32.4) (s32.4 imm5)  :cost 1 :encoding :sse)
+   (s32.4-shiftr      #:psrld-imm  (s32.4) (s32.4 imm5)  :cost 1 :encoding :sse)
+   ;; s64.2
+   (s64.2!-from-s64   nil          (s64.2) (s64)         :cost 1 :encoding :fake-vop)
+   (s64.2!-from-p128  #:movdqu     (s64.2) (p128)        :cost 1 :encoding :move :always-translatable nil)
+   (make-s64.2        nil          (s64.2) (s64 s64)     :cost 1 :encoding :fake-vop)
+   (s64.2-values      nil          (s64 s64) (s64.2)     :cost 1 :encoding :fake-vop)
+   (s64.2-broadcast   nil          (s64.2) (s64)         :cost 1 :encoding :fake-vop)
+   (two-arg-s64.2-and #:pand       (s64.2) (s64.2 s64.2) :cost 1 :encoding :sse :associative t)
+   (two-arg-s64.2-or  #:por        (s64.2) (s64.2 s64.2) :cost 1 :encoding :sse :associative t)
+   (two-arg-s64.2-xor #:pxor       (s64.2) (s64.2 s64.2) :cost 1 :encoding :sse :associative t)
+   (s64.2-andc1       #:pandn      (s64.2) (s64.2 s64.2) :cost 1 :encoding :sse)
+   (s64.2-not         nil          (s64.2) (s64.2)       :cost 1 :encoding :fake-vop)
+   (two-arg-s64.2+    #:paddq      (s64.2) (s64.2 s64.2) :cost 2 :encoding :sse :associative t)
+   (two-arg-s64.2-    #:psubq      (s64.2) (s64.2 s64.2) :cost 2 :encoding :sse)
+   (s64.2-unpackhi    #:punpckhqdq (s64.2) (s64.2 s64.2) :cost 1 :encoding :sse)
+   (s64.2-unpacklo    #:punpcklqdq (s64.2) (s64.2 s64.2) :cost 1 :encoding :sse)
+   (s64.2-movemask    #:movmskpd   (u2)    (s64.2)       :cost 1)
+   (s64.2-shiftl      #:psllq-imm  (s64.2) (s64.2 imm6)  :cost 1 :encoding :sse)
+   (s64.2-shiftr      #:psrlq-imm  (s64.2) (s64.2 imm6)  :cost 1 :encoding :sse))
+  (:loads
+   (u32.4-load-from-string #:movdqu u32.4 charvec char-array u32.4-string-ref u32.4-row-major-string-ref)
+   (f64-load   #:movsd  f64   f64vec f64-array f64-aref   f64-row-major-aref)
+   (f64.2-load #:movupd f64.2 f64vec f64-array f64.2-aref f64.2-row-major-aref)
+   (u8.16-load #:movdqu u8.16  u8vec  u8-array u8.16-aref u8.16-row-major-aref)
+   (u16.8-load #:movdqu u16.8 u16vec u16-array u16.8-aref u16.8-row-major-aref)
+   (u32.4-load #:movdqu u32.4 u32vec u32-array u32.4-aref u32.4-row-major-aref)
+   (u64.2-load #:movdqu u64.2 u64vec u64-array u64.2-aref u64.2-row-major-aref)
+   (s8.16-load #:movdqu s8.16  s8vec  s8-array s8.16-aref s8.16-row-major-aref)
+   (s16.8-load #:movdqu s16.8 s16vec s16-array s16.8-aref s16.8-row-major-aref)
+   (s32.4-load #:movdqu s32.4 s32vec s32-array s32.4-aref s32.4-row-major-aref)
+   (s64.2-load #:movdqu s64.2 s64vec s64-array s64.2-aref s64.2-row-major-aref))
+  (:stores
+   (u32.4-store-into-string #:movdqu u32.4 charvec char-array u32.4-string-ref u32.4-row-major-string-ref)
+   (f64-store   #:movsd  f64   f64vec f64-array f64-aref   f64-row-major-aref)
+   (f64.2-store #:movupd f64.2 f64vec f64-array f64.2-aref f64.2-row-major-aref)
+   (u8.16-store #:movdqu u8.16  u8vec  u8-array u8.16-aref u8.16-row-major-aref)
+   (u16.8-store #:movdqu u16.8 u16vec u16-array u16.8-aref u16.8-row-major-aref)
+   (u32.4-store #:movdqu u32.4 u32vec u32-array u32.4-aref u32.4-row-major-aref)
+   (u64.2-store #:movdqu u64.2 u64vec u64-array u64.2-aref u64.2-row-major-aref)
+   (s8.16-store #:movdqu s8.16  s8vec  s8-array s8.16-aref s8.16-row-major-aref)
+   (s16.8-store #:movdqu s16.8 s16vec s16-array s16.8-aref s16.8-row-major-aref)
+   (s32.4-store #:movdqu s32.4 s32vec s32-array s32.4-aref s32.4-row-major-aref)
+   (s64.2-store #:movdqu s64.2 s64vec s64-array s64.2-aref s64.2-row-major-aref)
+   (f64.2-ntstore #:movntpd f64.2 f64vec f64-array f64.2-non-temporal-aref f64.2-non-temporal-row-major-aref)
+   (u8.16-ntstore #:movntdq u8.16  u8vec  u8-array u8.16-non-temporal-aref u8.16-non-temporal-row-major-aref)
+   (u16.8-ntstore #:movntdq u16.8 u16vec u16-array u16.8-non-temporal-aref u16.8-non-temporal-row-major-aref)
+   (u32.4-ntstore #:movntdq u32.4 u32vec u32-array u32.4-non-temporal-aref u32.4-non-temporal-row-major-aref)
+   (u64.2-ntstore #:movntdq u64.2 u64vec u64-array u64.2-non-temporal-aref u64.2-non-temporal-row-major-aref)
+   (s8.16-ntstore #:movntdq s8.16  s8vec  s8-array s8.16-non-temporal-aref s8.16-non-temporal-row-major-aref)
+   (s16.8-ntstore #:movntdq s16.8 s16vec s16-array s16.8-non-temporal-aref s16.8-non-temporal-row-major-aref)
+   (s32.4-ntstore #:movntdq s32.4 s32vec s32-array s32.4-non-temporal-aref s32.4-non-temporal-row-major-aref)
+   (s64.2-ntstore #:movntdq s64.2 s64vec s64-array s64.2-non-temporal-aref s64.2-non-temporal-row-major-aref))
+  (:associatives
+   (f64-and two-arg-f64-and +f64-true+)
+   (f64-or  two-arg-f64-or  +f64-false+)
+   (f64-xor two-arg-f64-xor +f64-false+)
+   (f64-max two-arg-f64-max nil)
+   (f64-min two-arg-f64-min nil)
+   (f64+    two-arg-f64+ 0d0)
+   (f64*    two-arg-f64* 1d0)
+   (f64.2-and two-arg-f64.2-and +f64-true+)
+   (f64.2-or  two-arg-f64.2-or  +f64-false+)
+   (f64.2-xor two-arg-f64.2-xor +f64-false+)
+   (f64.2-max two-arg-f64.2-max nil)
+   (f64.2-min two-arg-f64.2-min nil)
+   (f64.2+    two-arg-f64.2+ 0d0)
+   (f64.2*    two-arg-f64.2* 1d0)
+   (u8.16-and two-arg-u8.16-and +u8-true+)
+   (u8.16-or  two-arg-u8.16-or  +u8-false+)
+   (u8.16-xor two-arg-u8.16-xor +u8-false+)
+   (u8.16+    two-arg-u8.16+    0)
+   (u16.8-and two-arg-u16.8-and +u16-true+)
+   (u16.8-or  two-arg-u16.8-or  +u16-false+)
+   (u16.8-xor two-arg-u16.8-xor +u16-false+)
+   (u16.8+    two-arg-u16.8+    0)
+   (u32.4-and two-arg-u32.4-and +u32-true+)
+   (u32.4-or  two-arg-u32.4-or  +u32-false+)
+   (u32.4-xor two-arg-u32.4-xor +u32-false+)
+   (u32.4+    two-arg-u32.4+    0)
+   (u64.2-and two-arg-u64.2-and +u64-true+)
+   (u64.2-or  two-arg-u64.2-or  +u64-false+)
+   (u64.2-xor two-arg-u64.2-xor +u64-false+)
+   (u64.2+    two-arg-u64.2+    0)
+   (s8.16-and two-arg-s8.16-and +s8-true+)
+   (s8.16-or  two-arg-s8.16-or  +s8-false+)
+   (s8.16-xor two-arg-s8.16-xor +s8-false+)
+   (s8.16+    two-arg-s8.16+    0)
+   (s16.8-and two-arg-s16.8-and +s16-true+)
+   (s16.8-or  two-arg-s16.8-or  +s16-false+)
+   (s16.8-xor two-arg-s16.8-xor +s16-false+)
+   (s16.8+    two-arg-s16.8+    0)
+   (s32.4-and two-arg-s32.4-and +s32-true+)
+   (s32.4-or  two-arg-s32.4-or  +s32-false+)
+   (s32.4-xor two-arg-s32.4-xor +s32-false+)
+   (s32.4+    two-arg-s32.4+    0)
+   (s64.2-and two-arg-s64.2-and +s64-true+)
+   (s64.2-or  two-arg-s64.2-or  +s64-false+)
+   (s64.2-xor two-arg-s64.2-xor +s64-false+)
+   (s64.2+    two-arg-s64.2+    0))
+  (:comparisons
+   (f64=  two-arg-f64=  u64-and +u64-true+)
+   (f64<  two-arg-f64<  u64-and +u64-true+)
+   (f64<= two-arg-f64<= u64-and +u64-true+)
+   (f64>  two-arg-f64>  u64-and +u64-true+)
+   (f64>= two-arg-f64>= u64-and +u64-true+)
+   (f32.4=  two-arg-f32.4=  u32.4-and +u32-true+)
+   (f32.4<  two-arg-f32.4<  u32.4-and +u32-true+)
+   (f32.4<= two-arg-f32.4<= u32.4-and +u32-true+)
+   (f32.4>  two-arg-f32.4>  u32.4-and +u32-true+)
+   (f32.4>= two-arg-f32.4>= u32.4-and +u32-true+)
+   (f64.2=  two-arg-f64.2=  u64.2-and +u64-true+)
+   (f64.2<  two-arg-f64.2<  u64.2-and +u64-true+)
+   (f64.2<= two-arg-f64.2<= u64.2-and +u64-true+)
+   (f64.2>  two-arg-f64.2>  u64.2-and +u64-true+)
+   (f64.2>= two-arg-f64.2>= u64.2-and +u64-true+)
+   (u8.16=  two-arg-u8.16=  u8.16-and +u8-true+)
+   (u8.16<  two-arg-u8.16<  u8.16-and +u8-true+)
+   (u8.16<= two-arg-u8.16<= u8.16-and +u8-true+)
+   (u8.16>  two-arg-u8.16>  u8.16-and +u8-true+)
+   (u8.16>= two-arg-u8.16>= u8.16-and +u8-true+)
+   (u16.8=  two-arg-u16.8=  u16.8-and +u16-true+)
+   (u16.8<  two-arg-u16.8<  u16.8-and +u16-true+)
+   (u16.8<= two-arg-u16.8<= u16.8-and +u16-true+)
+   (u16.8>  two-arg-u16.8>  u16.8-and +u16-true+)
+   (u16.8>= two-arg-u16.8>= u16.8-and +u16-true+)
+   (u32.4=  two-arg-u32.4=  u32.4-and +u32-true+)
+   (u32.4<  two-arg-u32.4<  u32.4-and +u32-true+)
+   (u32.4<= two-arg-u32.4<= u32.4-and +u32-true+)
+   (u32.4>  two-arg-u32.4>  u32.4-and +u32-true+)
+   (u32.4>= two-arg-u32.4>= u32.4-and +u32-true+)
+   (s8.16=  two-arg-s8.16=  u8.16-and +u8-true+)
+   (s8.16<  two-arg-s8.16<  u8.16-and +u8-true+)
+   (s8.16<= two-arg-s8.16<= u8.16-and +u8-true+)
+   (s8.16>  two-arg-s8.16>  u8.16-and +u8-true+)
+   (s8.16>= two-arg-s8.16>= u8.16-and +u8-true+)
+   (s16.8=  two-arg-s16.8=  u16.8-and +u16-true+)
+   (s16.8<  two-arg-s16.8<  u16.8-and +u16-true+)
+   (s16.8<= two-arg-s16.8<= u16.8-and +u16-true+)
+   (s16.8>  two-arg-s16.8>  u16.8-and +u16-true+)
+   (s16.8>= two-arg-s16.8>= u16.8-and +u16-true+)
+   (s32.4=  two-arg-s32.4=  u32.4-and +u32-true+)
+   (s32.4<  two-arg-s32.4<  u32.4-and +u32-true+)
+   (s32.4<= two-arg-s32.4<= u32.4-and +u32-true+)
+   (s32.4>  two-arg-s32.4>  u32.4-and +u32-true+)
+   (s32.4>= two-arg-s32.4>= u32.4-and +u32-true+))
+  (:reducers
+   (f64- two-arg-f64- 0d0)
+   (f64/ two-arg-f64/ 1d0)
+   (f64.2- two-arg-f64.2- 0d0)
+   (f64.2/ two-arg-f64.2/ 1d0)
+   (u8.16- two-arg-u8.16- 0)
+   (u16.8- two-arg-u16.8- 0)
+   (u32.4- two-arg-u32.4- 0)
+   (u64.2- two-arg-u64.2- 0)
+   (s8.16- two-arg-s8.16- 0)
+   (s16.8- two-arg-s16.8- 0)
+   (s32.4- two-arg-s32.4- 0)
+   (s64.2- two-arg-s64.2- 0))
+  (:unequals
+   (f64/= two-arg-f64/= u64-and +u64-true+)
+   (f32.4/= two-arg-f32.4/= sb-simd-sse2:u32.4-and +u32-true+)
+   (f64.2/= two-arg-f64.2/= u64.2-and +u64-true+)
+   (u8.16/= two-arg-u8.16/= u8.16-and +u8-true+)
+   (u16.8/= two-arg-u16.8/= u16.8-and +u16-true+)
+   (u32.4/= two-arg-u32.4/= u32.4-and +u32-true+)
+   (s8.16/= two-arg-s8.16/= u8.16-and +u8-true+)
+   (s16.8/= two-arg-s16.8/= u16.8-and +u16-true+)
+   (s32.4/= two-arg-s32.4/= u32.4-and +u32-true+)))

contrib/sb-simd/code/instruction-sets/sse3.lisp

@@ -0,0 +1,11 @@
+(in-package #:sb-simd-sse3)
+
+(define-instruction-set :sse3
+  (:include :sse2)
+  (:test (sse3-supported-p))
+  (:instructions
+   (f32.4-hadd  #:haddps   (f32.4) (f32.4 f32.4) :cost 1 :encoding :sse)
+   (f32.4-hdup  #:movshdup (f32.4) (f32.4) :cost 1)
+   (f32.4-ldup  #:movsldup (f32.4) (f32.4) :cost 1)
+   (f64.2-hadd  #:haddpd   (f64.2) (f64.2 f64.2) :cost 1 :encoding :sse)))
+

contrib/sb-simd/code/instruction-sets/sse4-1.lisp

@@ -0,0 +1,125 @@
+(in-package #:sb-simd-sse4.1)
+
+(define-instruction-set :sse4.1
+  (:include :ssse3)
+  (:test (sse4.1-supported-p))
+  (:instructions
+   ;; f32.4
+   (f32.4-blend         #:blendvps  (f32.4) (f32.4 f32.4 u32.4) :cost 1 :encoding :sse+xmm0)
+   (f32.4-blendc        #:blendps   (f32.4) (f32.4 f32.4 imm4) :cost 1 :encoding :sse)
+   (f32.4-%round        #:roundps   (f32.4) (f32.4 imm3)       :cost 2)
+   #+(or) ;; TODO: Broken in SBCL
+   (f32.4-elt           #:extractps (f32)   (f32.4 imm2)       :cost 1)
+   (f32.4-insert        #:insertps  (f32.4) (f32.4 f32.4 imm8) :cost 1 :encoding :sse)
+   ;; f64.2
+   (f64.2-blend         #:blendvpd  (f64.2) (f64.2 f64.2 u64.2) :cost 1 :encoding :sse+xmm0)
+   (f64.2-blendc        #:blendpd   (f64.2) (f64.2 f64.2 imm2) :cost 1 :encoding :sse)
+   (f64.2-%round        #:roundpd   (f64.2) (f64.2 imm3)       :cost 2)
+   ;; u8.16
+   (u8.16-blend         #:pblendvb  (u8.16) (u8.16 u8.16 u8.16) :cost 1 :encoding :sse+xmm0)
+   (u8.16-elt           #:pextrb    (u8)    (u8.16 imm4)       :cost 1)
+   (u8.16-insert        #:pinsrb    (u8.16) (u8.16 u8 imm4)    :cost 1 :encoding :sse)
+   ;; u16.8
+   (u16.8-blend         #:pblendvb  (u16.8) (u16.8 u16.8 u16.8) :cost 1 :encoding :sse+xmm0)
+   (two-arg-u16.8-max   #:pmaxuw    (u16.8) (u16.8 u16.8)      :cost 2 :encoding :sse :associative t)
+   (two-arg-u16.8-min   #:pminuw    (u16.8) (u16.8 u16.8)      :cost 2 :encoding :sse :associative t)
+   (u16.8-minpos        #:pminuw    (u16.8) (u16.8)            :cost 5)
+   ;; u32.4
+   (u32.4-blend         #:pblendvb  (u32.4) (u32.4 u32.4 u32.4) :cost 1 :encoding :sse+xmm0)
+   (two-arg-u32.4-max   #:pmaxud    (u32.4) (u32.4 u32.4)      :cost 2 :encoding :sse :associative t)
+   (two-arg-u32.4-min   #:pminud    (u32.4) (u32.4 u32.4)      :cost 2 :encoding :sse :associative t)
+   (u32.4-elt           #:pextrd    (u32)   (u32.4 imm2)       :cost 1)
+   (u32.4-insert        #:pinsrd    (u32.4) (u32.4 u32 imm2)   :cost 1 :encoding :sse)
+   ;; u64.2
+   (u64.2-blend         #:pblendvb  (u64.2) (u64.2 u64.2 u64.2) :cost 1 :encoding :sse+xmm0)
+   (two-arg-u64.2=      #:pcmpeqq   (u64.2) (u64.2 u64.2)      :cost 1 :encoding :sse :associative t)
+   (two-arg-u64.2/=     nil         (u64.2) (u64.2 u64.2)      :cost 2 :encoding :fake-vop :associative t)
+   (u64.2-elt           #:pextrq    (u64)   (u64.2 imm1)       :cost 1)
+   #+(or) ;; TODO: PINSRQ is currently missing in SBCL.
+   (u64.2-insert        #:pinsrq    (u64.2) (u64.2 u64 imm1)   :cost 1 :encoding :sse)
+   ;; s8.16
+   (s8.16-blend         #:pblendvb  (s8.16) (s8.16 s8.16 u8.16) :cost 1 :encoding :sse+xmm0)
+   (two-arg-s8.16-max   #:pmaxsb    (s8.16) (s8.16 s8.16)      :cost 2 :encoding :sse :associative t)
+   (two-arg-s8.16-min   #:pminsb    (s8.16) (s8.16 s8.16)      :cost 2 :encoding :sse :associative t)
+   (s8.16-elt           #:pextrb    (s8)    (s8.16 imm4)       :cost 1)
+   (s8.16-insert        #:pinsrb    (s8.16) (s8.16 s8 imm4)    :cost 1 :encoding :sse)
+   ;; s16.8
+   (s16.8-blend         #:pblendvb  (s16.8) (s16.8 s16.8 u16.8) :cost 1 :encoding :sse+xmm0)
+   (s16.8-from-u8.16    #:pmovsxbw  (s16.8) (u8.16)            :cost 5)
+   (s16.8-from-s8.16    #:pmovsxbw  (s16.8) (s8.16)            :cost 5)
+   (s16.8-pack          #:packusdw  (s16.8) (s32.4 s32.4)      :cost 1 :encoding :sse)
+   ;; s32.4
+   (s32.4-blend         #:pblendvb  (s32.4) (s32.4 s32.4 u32.4) :cost 1 :encoding :sse+xmm0)
+   (two-arg-s32.4-max   #:pmaxsd    (s32.4) (s32.4 s32.4)      :cost 2 :encoding :sse :associative t)
+   (two-arg-s32.4-min   #:pminsd    (s32.4) (s32.4 s32.4)      :cost 2 :encoding :sse :associative t)
+   (s32.4-from-u8.16    #:pmovsxbd  (s32.4) (u8.16)            :cost 5)
+   (s32.4-from-s8.16    #:pmovsxbd  (s32.4) (s8.16)            :cost 5)
+   (s32.4-from-u16.8    #:pmovsxwd  (s32.4) (u16.8)            :cost 5)
+   (s32.4-from-s16.8    #:pmovsxwd  (s32.4) (s16.8)            :cost 5)
+   (two-arg-s32.4-mullo #:pmulld    (s32.4) (s32.4 s32.4)      :cost 1 :encoding :sse :associative t)
+   (s32.4-elt           #:pextrd    (s32)   (s32.4 imm2)       :cost 1)
+   (s32.4-insert        #:pinsrd    (s32.4) (s32.4 s32 imm2)   :cost 1 :encoding :sse)
+   ;; s64.2
+   (s64.2-blend         #:pblendvb  (s64.2) (s64.2 s64.2 u64.2) :cost 1 :encoding :sse+xmm0)
+   (s64.2-from-u8.16    #:pmovsxbq  (s64.2) (u8.16)            :cost 5)
+   (s64.2-from-s8.16    #:pmovsxbq  (s64.2) (s8.16)            :cost 5)
+   (s64.2-from-u16.8    #:pmovsxwq  (s64.2) (u16.8)            :cost 5)
+   (s64.2-from-s16.8    #:pmovsxwq  (s64.2) (s16.8)            :cost 5)
+   (s64.2-from-u32.4    #:pmovsxdq  (s64.2) (u32.4)            :cost 5)
+   (s64.2-from-s32.4    #:pmovsxdq  (s64.2) (s32.4)            :cost 5)
+   (two-arg-s64.2-mul   #:pmuldq    (s64.2) (s64.2 s64.2)      :cost 2 :encoding :sse :associative t)
+   (two-arg-s64.2=      #:pcmpeqq   (u64.2) (s64.2 s64.2)      :cost 1 :encoding :sse :associative t)
+   (two-arg-s64.2/=     nil         (u64.2) (s64.2 s64.2)      :cost 2 :encoding :fake-vop :associative t)
+   (s64.2-elt           #:pextrq    (s64)   (s64.2 imm1)       :cost 1)
+   #+(or) ;; TODO: PINSRQ is currently missing in SBCL.
+   (s64.2-insert        #:pinsrq    (s64.2) (s64.2 s64 imm1)   :cost 1 :encoding :sse))
+  (:loads
+   (f32.4-ntload #:movntdqa f32.4 f32vec f32-array f32.4-non-temporal-aref f32.4-non-temporal-row-major-aref)
+   (f64.2-ntload #:movntdqa f64.2 f64vec f64-array f64.2-non-temporal-aref f64.2-non-temporal-row-major-aref)
+   (u8.16-ntload #:movntdqa u8.16  u8vec  u8-array u8.16-non-temporal-aref u8.16-non-temporal-row-major-aref)
+   (u16.8-ntload #:movntdqa u16.8 u16vec u16-array u16.8-non-temporal-aref u16.8-non-temporal-row-major-aref)
+   (u32.4-ntload #:movntdqa u32.4 u32vec u32-array u32.4-non-temporal-aref u32.4-non-temporal-row-major-aref)
+   (u64.2-ntload #:movntdqa u64.2 u64vec u64-array u64.2-non-temporal-aref u64.2-non-temporal-row-major-aref)
+   (s8.16-ntload #:movntdqa s8.16  s8vec  s8-array s8.16-non-temporal-aref s8.16-non-temporal-row-major-aref)
+   (s16.8-ntload #:movntdqa s16.8 s16vec s16-array s16.8-non-temporal-aref s16.8-non-temporal-row-major-aref)
+   (s32.4-ntload #:movntdqa s32.4 s32vec s32-array s32.4-non-temporal-aref s32.4-non-temporal-row-major-aref)
+   (s64.2-ntload #:movntdqa s64.2 s64vec s64-array s64.2-non-temporal-aref s64.2-non-temporal-row-major-aref))
+  (:stores
+   (f32.4-ntstore #:movntps f32.4 f32vec f32-array f32.4-non-temporal-aref f32.4-non-temporal-row-major-aref)
+   (f64.2-ntstore #:movntpd f64.2 f64vec f64-array f64.2-non-temporal-aref f64.2-non-temporal-row-major-aref)
+   (u8.16-ntstore #:movntdq u8.16  u8vec  u8-array u8.16-non-temporal-aref u8.16-non-temporal-row-major-aref)
+   (u16.8-ntstore #:movntdq u16.8 u16vec u16-array u16.8-non-temporal-aref u16.8-non-temporal-row-major-aref)
+   (u32.4-ntstore #:movntdq u32.4 u32vec u32-array u32.4-non-temporal-aref u32.4-non-temporal-row-major-aref)
+   (u64.2-ntstore #:movntdq u64.2 u64vec u64-array u64.2-non-temporal-aref u64.2-non-temporal-row-major-aref)
+   (s8.16-ntstore #:movntdq s8.16  s8vec  s8-array s8.16-non-temporal-aref s8.16-non-temporal-row-major-aref)
+   (s16.8-ntstore #:movntdq s16.8 s16vec s16-array s16.8-non-temporal-aref s16.8-non-temporal-row-major-aref)
+   (s32.4-ntstore #:movntdq s32.4 s32vec s32-array s32.4-non-temporal-aref s32.4-non-temporal-row-major-aref)
+   (s64.2-ntstore #:movntdq s64.2 s64vec s64-array s64.2-non-temporal-aref s64.2-non-temporal-row-major-aref))
+  (:associatives
+   (u16.8-max two-arg-u16.8-max nil)
+   (u16.8-min two-arg-u16.8-min nil)
+   (u32.4-max two-arg-u32.4-max nil)
+   (u32.4-min two-arg-u32.4-min nil)
+   (s8.16-max two-arg-s8.16-max nil)
+   (s8.16-min two-arg-s8.16-min nil)
+   (s32.4-max two-arg-s32.4-max nil)
+   (s32.4-min two-arg-s32.4-min nil)
+   (s32.4-mullo two-arg-s32.4-mullo 1)
+   (s64.2-mul two-arg-s64.2-mul 1))
+  (:comparisons
+   (u64.2=  two-arg-u64.2=  u64.2-and +u64-true+)
+   (s64.2=  two-arg-s64.2=  u64.2-and +u64-true+))
+  (:ifs
+   (f32.4-if f32.4-blend)
+   (f64.2-if f64.2-blend)
+   (u8.16-if u8.16-blend)
+   (u16.8-if u16.8-blend)
+   (u32.4-if u32.4-blend)
+   (u64.2-if u64.2-blend)
+   (s8.16-if s8.16-blend)
+   (s16.8-if s16.8-blend)
+   (s32.4-if s32.4-blend)
+   (s64.2-if s64.2-blend))
+  (:unequals
+   (u64.2/= two-arg-u64.2/= u64.2-and +u64-true+)
+   (s64.2/= two-arg-s64.2/= u64.2-and +u64-true+)))

contrib/sb-simd/code/instruction-sets/sse4-2.lisp

@@ -0,0 +1,26 @@
+(in-package #:sb-simd-sse4.2)
+
+(define-instruction-set :sse4.2
+  (:include :sse4.1)
+  (:test (sse4.2-supported-p))
+  (:instructions
+   ;; u64.2
+   (two-arg-u64.2>~ #:pcmpgtq (u64.2) (u64.2 u64.2) :cost 3 :encoding :sse)
+   (two-arg-u64.2>  nil       (u64.2) (u64.2 u64.2) :cost 3 :encoding :fake-vop)
+   (two-arg-u64.2>= nil       (u64.2) (u64.2 u64.2) :cost 4 :encoding :fake-vop)
+   (two-arg-u64.2<  nil       (u64.2) (u64.2 u64.2) :cost 3 :encoding :fake-vop)
+   (two-arg-u64.2<= nil       (u64.2) (u64.2 u64.2) :cost 4 :encoding :fake-vop)
+   ;; s64.2
+   (two-arg-s64.2>  #:pcmpgtq (u64.2) (s64.2 s64.2) :cost 3 :encoding :sse)
+   (two-arg-s64.2>= nil       (u64.2) (s64.2 s64.2) :cost 4 :encoding :fake-vop)
+   (two-arg-s64.2<  nil       (u64.2) (s64.2 s64.2) :cost 3 :encoding :fake-vop)
+   (two-arg-s64.2<= nil       (u64.2) (s64.2 s64.2) :cost 4 :encoding :fake-vop))
+  (:comparisons
+   (u64.2<  two-arg-u64.2<  u64.2-and +u64-true+)
+   (u64.2<= two-arg-u64.2<= u64.2-and +u64-true+)
+   (u64.2>  two-arg-u64.2>  u64.2-and +u64-true+)
+   (u64.2>= two-arg-u64.2>= u64.2-and +u64-true+)
+   (s64.2<  two-arg-s64.2<  u64.2-and +u64-true+)
+   (s64.2<= two-arg-s64.2<= u64.2-and +u64-true+)
+   (s64.2>  two-arg-s64.2>  u64.2-and +u64-true+)
+   (s64.2>= two-arg-s64.2>= u64.2-and +u64-true+)))

contrib/sb-simd/code/instruction-sets/ssse3.lisp

@@ -0,0 +1,31 @@
+(in-package #:sb-simd-ssse3)
+
+(define-instruction-set :ssse3
+  (:include :sse3)
+  (:test (ssse3-supported-p))
+  (:instructions
+   ;; u16.8
+   (u16.8-hadd    #:phaddw    (u16.8) (u16.8 u16.8) :cost 3 :encoding :sse)
+   (u16.8-hsub    #:phsubw    (u16.8) (u16.8 u16.8) :cost 3 :encoding :sse)
+   ;; u32.4
+   (u32.4-hadd    #:phaddd    (u32.4) (u32.4 u32.4) :cost 3 :encoding :sse)
+   (u32.4-hsub    #:phsubd    (u32.4) (u32.4 u32.4) :cost 3 :encoding :sse)
+   ;; s8.16
+   (s8.16-shuffle #:pshufb    (s8.16) (s8.16 s8.16) :cost 1 :encoding :sse)
+   (s8.16-abs     #:pabsb     (s8.16) (s8.16)       :cost 2)
+   (s8.16-sign    #:psignb    (s8.16) (s8.16 s8.16) :cost 1 :encoding :sse)
+   ;; s16.8
+   (two-arg-s16.8-mulhrs #:pmulhrsw  (s16.8) (s16.8 s16.8) :cost 1 :encoding :sse :associative t)
+   (s16.8-abs     #:pabsw     (s16.8) (s16.8)       :cost 2)
+   (s16.8-maddubs #:pmaddubsw (s16.8) (u16.8 s16.8) :cost 2 :encoding :sse)
+   (s16.8-sign    #:psignw    (s16.8) (s16.8 s16.8) :cost 2 :encoding :sse)
+   ;; s16.8
+   (s16.8-hadd    #:phaddw    (s16.8) (s16.8 s16.8) :cost 3 :encoding :sse)
+   (s16.8-hsub    #:phsubw    (s16.8) (s16.8 s16.8) :cost 3 :encoding :sse)
+   ;; s32.4
+   (s32.4-abs     #:pabsd     (s32.4) (s32.4)       :cost 2)
+   (s32.4-sign    #:psignd    (s32.4) (s32.4 s32.4) :cost 3 :encoding :sse)
+   (s32.4-hadd    #:phaddd    (s32.4) (s32.4 s32.4) :cost 3 :encoding :sse)
+   (s32.4-hsub    #:phsubd    (s32.4) (s32.4 s32.4) :cost 3 :encoding :sse))
+  (:associatives
+   (s16.8-mulhrs two-arg-s16.8-mulhrs 1)))

contrib/sb-simd/code/instruction-sets/x86-64.lisp

@@ -0,0 +1,13 @@
+(in-package #:sb-simd-x86-64)
+
+(define-instruction-set :x86-64
+  (:include :sb-simd)
+  (:scalars
+   (imm1 1 (unsigned-byte 1) (:constant (unsigned-byte 1)))
+   (imm2 2 (unsigned-byte 2) (:constant (unsigned-byte 2)))
+   (imm3 3 (unsigned-byte 3) (:constant (unsigned-byte 3)))
+   (imm4 4 (unsigned-byte 4) (:constant (unsigned-byte 4)))
+   (imm5 5 (unsigned-byte 5) (:constant (unsigned-byte 5)))
+   (imm6 6 (unsigned-byte 6) (:constant (unsigned-byte 6)))
+   (imm7 7 (unsigned-byte 7) (:constant (unsigned-byte 7)))
+   (imm8 8 (unsigned-byte 8) (:constant (unsigned-byte 8)))))

contrib/sb-simd/code/missing-instruction.lisp

@@ -0,0 +1,35 @@
+(in-package #:sb-simd-internals)
+
+;;; Our library always defines all instruction sets and instructions, but
+;;; usually, only a subset of them are actually available.  All the
+;;; remaining instructions are replaced by functions that signal a suitable
+;;; error message.
+
+(define-condition missing-instruction (error)
+  ((%record
+    :initarg :record
+    :reader missing-instruction-record))
+  (:report
+   (lambda (c s)
+     (with-accessors ((instruction-set instruction-record-instruction-set)
+                      (instruction-name instruction-record-name))
+         (missing-instruction-record c)
+       (format s "Missing ~S instruction ~S."
+               (instruction-set-name instruction-set)
+               instruction-name)))))
+
+(defun missing-instruction (instruction-record)
+  (error 'missing-instruction :record instruction-record))
+
+(defmacro define-missing-instruction
+    (name &key (required-arguments '()) (optional-arguments '()) (rest-argument nil))
+  (assert (find-function-record name))
+  `(defun ,name (,@required-arguments
+                 ,@optional-arguments
+                 ,@(when rest-argument `(&rest ,rest-argument)))
+     (declare (ignore ,@required-arguments
+                      ,@optional-arguments
+                      ,@(when rest-argument `(,rest-argument))))
+     (missing-instruction
+      (load-time-value
+       (find-function-record ',name)))))

contrib/sb-simd/code/printable.lisp

@@ -0,0 +1,24 @@
+(in-package #:sb-simd-internals)
+
+(defclass printable () ())
+
+;;; Gather a plist of all class slots.  We use this plist to conveniently
+;;; define PRINT-OBJECT.
+(defgeneric printable-slot-plist (printable)
+  (:method-combination append :most-specific-last))
+
+(defmethod printable-slot-plist append (printable)
+  '())
+
+(defmethod print-object ((printable printable) stream)
+  (print-unreadable-object (printable stream :type t)
+    (pprint-logical-block (stream (printable-slot-plist printable))
+      (loop
+        (pprint-exit-if-list-exhausted)
+        (write (pprint-pop) :stream stream)
+        (pprint-exit-if-list-exhausted)
+        (write-char #\space stream)
+        (write (pprint-pop) :stream stream)
+        (pprint-exit-if-list-exhausted)
+        (write-char #\space stream)
+        (pprint-newline :linear stream)))))

contrib/sb-simd/code/sb-simd.asd

@@ -0,0 +1,50 @@
+(defsystem #:sb-simd
+  :description "A convenient SIMD interface for SBCL."
+  :author "Marco Heisig <marco.heisig@fau.de>"
+  :license "MIT"
+
+  :in-order-to ((test-op (test-op "sb-simd-test-suite")))
+
+  :serial t
+  :components
+  ((:file "packages")
+   (:file "constants")
+   (:file "utilities")
+   (:file "printable")
+   (:file "cpu-identification")
+   (:file "instruction-set")
+   (:file "instruction-set-case")
+   (:file "record")
+   (:file "missing-instruction")
+   (:module "instruction-sets"
+    :components
+    ((:file "sb-simd")
+     (:file "x86-64")
+     (:file "sse")
+     (:file "sse2")
+     (:file "sse3")
+     (:file "ssse3")
+     (:file "sse4-1")
+     (:file "sse4-2")
+     (:file "avx")
+     (:file "avx2")
+     (:file "fma")))
+   (:file "define-types")
+   (:file "define-instruction-vops")
+   (:file "define-vref-vops")
+   (:file "define-custom-vops")
+   (:file "define-vop-functions")
+   (:file "define-scalar-casts")
+   (:file "define-fake-vops")
+   (:file "define-simd-casts")
+   (:file "define-instructions")
+   (:file "define-vrefs")
+   (:file "define-reffers")
+   (:file "define-arefs")
+   (:file "define-ifs")
+   (:file "define-associatives")
+   (:file "define-reducers")
+   (:file "define-comparisons")
+   (:file "define-unequals")
+   (:file "define-rounders")
+   (:file "define-modify-macros")))

contrib/sb-simd/code/utilities.lisp

@@ -0,0 +1,106 @@
+(in-package #:sb-simd-internals)
+
+;;; Types
+
+(deftype type-specifier ()
+  '(or symbol cons))
+
+(deftype non-nil-symbol ()
+  '(and symbol (not null)))
+
+(deftype function-name ()
+  '(or non-nil-symbol (cons (eql setf) (cons non-nil-symbol null))))
+
+(deftype index ()
+  '(integer
+    (#.(- (1- array-total-size-limit)))
+    (#.(1- array-total-size-limit))))
+
+;;; Functions
+
+(defun index+ (&rest indices)
+  (the index (apply #'+ indices)))
+
+(define-compiler-macro index+ (&rest indices)
+  `(the index (+ ,@(loop for index in indices collect `(the index ,index)))))
+
+(defun index- (index &rest more-indices)
+  (the index (apply #'- index more-indices)))
+
+(define-compiler-macro index- (index &rest more-indices)
+  `(the index (- (the index ,index) ,@(loop for index in more-indices collect `(the index ,index)))))
+
+(defun index* (&rest indices)
+  (the index (apply #'* indices)))
+
+(define-compiler-macro index* (&rest indices)
+  `(the index (* ,@(loop for index in indices collect `(the index ,index)))))
+
+(defun ensure-package (name)
+  (or (find-package name)
+      (make-package name)))
+
+(defun mksym (package &rest string-designators)
+  (intern
+   (apply #'concatenate 'string (mapcar #'string string-designators))
+   package))
+
+(defun prefixed-symbols (prefix n &optional (package *package*))
+  (loop for index below n
+        collect
+        (mksym package prefix (format nil "~D" index))))
+
+(declaim (notinline touch))
+(defun touch (&rest arguments &aux value)
+  (declare (ignore arguments))
+  (declare (special value))
+  (values-list (loop repeat (random 100) collect value)))
+
+(defun required-argument (initarg)
+  (error "Required argument: ~S" initarg))
+
+(defun macroexpand-all (form &optional env)
+  (let ((sb-walker::*walk-form-expand-macros-p* t))
+    (sb-walker:walk-form form env)))
+
+(defun ensure-list (x)
+  (if (listp x)
+      x
+      (list x)))
+
+(defun lambda-expression-p (x)
+  (and (listp x)
+       (> (list-length x) 1)
+       (eq (first x) 'lambda)
+       (listp (second x))))
+
+(defun parse-function-name (function-name)
+  (typecase function-name
+    (non-nil-symbol
+     (values function-name nil))
+    ((cons (eql setf) (cons non-nil-symbol null))
+     (values (second function-name) t))
+    (otherwise
+     (error "Not a valid function name: ~S" function-name))))
+
+;;; Macros
+
+(defmacro define-inline (name lambda-list &body body)
+  `(progn
+     (declaim (inline ,name))
+     (defun ,name ,lambda-list ,@body)))
+
+(defmacro define-notinline (name lambda-list &body body)
+  `(progn
+     (declaim (notinline ,name))
+     (defun ,name ,lambda-list ,@body)))
+
+(defun integer-type-specifier-inclusive-bounds (type-specifier)
+  (assert (= 2 (length type-specifier)))
+  (let ((bits (the (integer 1) (second type-specifier))))
+    (ecase (first type-specifier)
+      (unsigned-byte
+       (values 0 (1- (expt 2 bits))))
+      (signed-byte
+       (values (- (expt 2 (1- bits)))
+               (1- (expt 2 (1- bits))))))))

contrib/sb-simd/examples/benchmarks.lisp

@@ -0,0 +1,281 @@
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Benchmarking code
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(asdf:load-system :sb-simd)
+(ql:quickload :alexandria :silent t)
+(use-package :sb-simd-avx)
+
+(defmacro time-total (n &body body)
+  "N-average the execution time of BODY in seconds"
+  (declare (optimize (speed 0)))
+  (alexandria:with-gensyms (start end)
+    `(let (,start ,end)
+       (sb-ext:gc :full t)
+       (setf ,start (get-internal-real-time))
+       (loop for i below ,n
+             do ,@body)
+       (setf ,end (get-internal-real-time))
+       (coerce (/ (- ,end ,start) internal-time-units-per-second)
+               'float))))
+
+(defun simd-sum (array &aux (n (array-total-size array)))
+  (declare (type f64vec array)
+           (optimize speed (safety 0)))
+  (do ((index 0 (the (integer 0 #.(- array-total-size-limit 16)) (+ index 16)))
+       (acc1 (f64.4 0) (f64.4+ acc1 (f64.4-aref array (+ index 0))))
+       (acc2 (f64.4 0) (f64.4+ acc2 (f64.4-aref array (+ index 4))))
+       (acc3 (f64.4 0) (f64.4+ acc3 (f64.4-aref array (+ index 8))))
+       (acc4 (f64.4 0) (f64.4+ acc4 (f64.4-aref array (+ index 12)))))
+      ((> index (- n 16))
+       (do ((result (multiple-value-call #'+ (f64.4-values (f64.4+ acc1 acc2 acc3 acc4)))
+                    (+ result (row-major-aref array index)))
+            (index index (1+ index)))
+           ((>= index n) result)))))
+
+(defun simd-vdot (array1 array2 &aux (n (min (array-total-size array1) (array-total-size array2))))
+  (declare (type f64vec array1 array2)
+           (optimize speed (safety 0)))
+  (do ((index 0 (the (integer 0 #.(- array-total-size-limit 16)) (+ index 16)))
+       (acc1 (f64.4 0) (f64.4-incf acc1 (f64.4* (f64.4-aref array1 (+ index 0))
+                                                (f64.4-aref array2 (+ index 0)))))
+       (acc2 (f64.4 0) (f64.4-incf acc2 (f64.4* (f64.4-aref array1 (+ index 4))
+                                                (f64.4-aref array2 (+ index 4)))))
+       (acc3 (f64.4 0) (f64.4-incf acc3 (f64.4* (f64.4-aref array1 (+ index 8))
+                                                (f64.4-aref array2 (+ index 8)))))
+       (acc4 (f64.4 0) (f64.4-incf acc4 (f64.4* (f64.4-aref array1 (+ index 12))
+                                                (f64.4-aref array2 (+ index 12))))))
+      ((> index (- n 16))
+       (do ((result (multiple-value-call #'+ (f64.4-values (f64.4+ acc1 acc2 acc3 acc4)))
+                    (+ result (* (row-major-aref array1 index)
+                                 (row-major-aref array2 index))))
+            (index index (1+ index)))
+           ((>= index n) result)))))
+
+;; For some reasons it floating-point-overflows
+(defun fma-vdot (array1 array2 &aux (n (min (array-total-size array1) (array-total-size array2))))
+  (declare (type f64vec array1 array2)
+           (optimize speed (safety 0)))
+  (do ((index 0 (the (integer 0 #.(- array-total-size-limit 16)) (+ index 16)))
+       (acc1 (f64.4 0) (setf acc1 (sb-simd-avx2:f64.4-fmadd acc1 (f64.4-aref array1 (+ index 0))
+                                                            (f64.4-aref array2 (+ index 0)))))
+       (acc2 (f64.4 0) (setf acc2 (sb-simd-avx2:f64.4-fmadd acc2 (f64.4-aref array1 (+ index 4))
+                                                            (f64.4-aref array2 (+ index 4)))))
+       (acc3 (f64.4 0) (setf acc3 (sb-simd-avx2:f64.4-fmadd acc3 (f64.4-aref array1 (+ index 8))
+                                                            (f64.4-aref array2 (+ index 8)))))
+       (acc4 (f64.4 0) (setf acc4 (sb-simd-avx2:f64.4-fmadd acc4 (f64.4-aref array1 (+ index 12))
+                                                            (f64.4-aref array2 (+ index 12))))))
+      ((> index (- n 16))
+       (do ((result (f64.4-hsum (f64.4+ acc1 acc2 acc3 acc4))
+                    (+ result (* (row-major-aref array1 index)
+                                 (row-major-aref array2 index))))
+            (index index (1+ index)))
+           ((>= index n) result)))))
+
+(declaim (inline f64.4-fma-vdot))
+(defun f64.4-fma-vdot (array1 array2 &aux (n (min (array-total-size array1)
+                                                  (array-total-size array2))))
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (type f64vec array1 array2))
+  (let ((n0 (- n (mod n 16))))
+    (+ (loop with acc1 of-type f64.4 = (f64.4 0)
+             with acc2 of-type f64.4 = (f64.4 0)
+             with acc3 of-type f64.4 = (f64.4 0)
+             with acc4 of-type f64.4 = (f64.4 0)
+             for i of-type fixnum below n0 by 16
+             do (let ((array1-i0  (f64.4-aref array1 (+ i 0)))
+                      (array1-i4  (f64.4-aref array1 (+ i 4)))
+                      (array1-i8  (f64.4-aref array1 (+ i 8)))
+                      (array1-i12 (f64.4-aref array1 (+ i 12))))
+                  (setf acc1 (sb-simd-avx2:f64.4-fmadd acc1 array1-i0
+                                                       (f64.4-aref array2 (+ i 0)))
+                        acc2 (sb-simd-avx2:f64.4-fmadd acc2 array1-i4
+                                                       (f64.4-aref array2 (+ i 4)))
+                        acc3 (sb-simd-avx2:f64.4-fmadd acc3 array1-i8
+                                                       (f64.4-aref array2 (+ i 8)))
+                        acc4 (sb-simd-avx2:f64.4-fmadd acc4 array1-i12
+                                                       (f64.4-aref array2 (+ i 12)))))
+             finally (return (f64.4-hsum (f64.4+ acc1 acc2 acc3 acc4))))
+       (loop for i of-type fixnum from n0 below n
+             summing (* (aref array1 i) (aref array2 i))
+               into sum of-type double-float
+             finally (return sum)))))
+
+(defun benchmark-f64.4-vdot-double (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline sb-simd-avx2:f64.4-vdot))
+  (loop for len in v-lengths
+        do (format t "Doing dot product of two ~A long double float vectors 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1d0))
+                                                 (alexandria:iota len))))
+                      (v (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.2d0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f64vec u v))
+                  (time-total 1e6 (sb-simd-avx2::f64.4-vdot u v)))))
+
+(defun benchmark-f64.4-vdot2-double (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline sb-simd-avx2:f64.4-vdot2))
+  (loop for len in v-lengths
+        do (format t "Doing dot product of two ~A long double float vectors 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1d0))
+                                                 (alexandria:iota len))))
+                      (v (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.2d0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f64vec u v))
+                  (time-total 1e6 (sb-simd-avx2::f64.4-vdot2 u v)))))
+
+(defun benchmark-f64.4-fma-vdot-double (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline f64.4-fma-vdot))
+  (loop for len in v-lengths
+        do (format t "Doing dot product of two ~A long double float vectors 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1d0))
+                                                 (alexandria:iota len))))
+                      (v (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.2d0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f64vec u v))
+                  (time-total 1e6 (f64.4-fma-vdot u v)))))
+
+(defun benchmark-simd-vdot-double (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline f64.4-vdot))
+  (loop for len in v-lengths
+        do (format t "Doing dot product of two ~A long double float vectors 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1d0))
+                                                 (alexandria:iota len))))
+                      (v (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.2d0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f64vec u v))
+                  (time-total 1e6 (simd-vdot u v)))))
+
+(defun benchmark-fma-vdot-double (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline fma-vdot))
+  (loop for len in v-lengths
+        do (format t "Doing dot product of two ~A long double float vectors 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1d0))
+                                                 (alexandria:iota len))))
+                      (v (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.2d0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f64vec u v))
+                  (time-total 1e6 (fma-vdot u v)))))
+
+(defun benchmark-avx-single (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline f32.8-vdot))
+  (loop for len in v-lengths
+        do (format t "Doing dot product of two ~A long single float vectors 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'single-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1f0))
+                                                 (alexandria:iota len))))
+                      (v (make-array len :element-type 'single-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.2f0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f32vec u v))
+                  (time-total 1e6 (f32.8-vdot u v)))))
+
+(defun benchmark-sse-double (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline f64.2-vdot))
+  (loop for len in v-lengths
+        do (format t "Doing dot product of two ~A long double float vectors 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1d0))
+                                                 (alexandria:iota len))))
+                      (v (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.2d0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f32vec u v))
+                  (time-total 1e6 (f64.2-vdot u v)))))
+
+(defun benchmark-sse-single (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline f32.4-vdot))
+  (loop for len in v-lengths
+        do (format t "Doing dot product of two ~A long single float vectors 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'single-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1f0))
+                                                 (alexandria:iota len))))
+                      (v (make-array len :element-type 'single-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.2f0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f32vec u v))
+                  (time-total 1e6 (f32.4-vdot u v)))))
+
+(defun benchmark-vsum-double (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline f64.4-vsum))
+  (loop for len in v-lengths
+        do (format t "Doing sum of a ~A long double float vector 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1d0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f64vec u))
+                  (time-total 1e6 (f64.4-vsum u)))))
+
+(defun benchmark-simd-sum-double (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline simd-sum))
+  (loop for len in v-lengths
+        do (format t "Doing sum of a ~A long double float vector 1e6 times~%" len)
+        collect (let ((u (make-array len :element-type 'double-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1d0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f64vec u))
+                  (time-total 1e6 (simd-sum u)))))
+
+(defun benchmark-vsum-single (&rest v-lengths)
+  (declare (optimize (speed 3) (safety 0) (debug 0))
+           (notinline f32.8-vsum))
+  (loop for len in v-lengths
+        do (format t "Doing sum of a ~A long single float vector 1e6 x~%" len)
+        collect (let ((u (make-array len :element-type 'single-float
+                                         :initial-contents
+                                         (mapcar (lambda (i) (+ i 0.1f0))
+                                                 (alexandria:iota len)))))
+                  (declare (type f32vec u))
+                  (time-total 1e6 (f32.8-vsum u)))))
+#+(or)
+(benchmark-f64.4-vdot-double 10 100 200 400 800 1200 2400 4800 9600)
+#+(or)
+(benchmark-f64.4-vdot2-double 10 100 200 400 800 1200 2400 4800 9600)
+#+(or)
+(benchmark-avx-single 10 100 200 400 800 1200 2400 4800 9600)
+#+(or)
+(benchmark-sse-double 10 100 200 400 800 1200 2400 4800 9600)
+#+(or)
+(benchmark-sse-single 10 100 200 400 800 1200 2400 4800 9600)
+#+(or)
+(benchmark-vsum-double 10 100 200 400 800 1200 2400 4800 9600)
+#+(or)
+(benchmark-vsum-single 10 100 200 400 800 1200 2400 4800 9600)
+#+(or)
+(apply 'benchmark-avx-double (loop for i from 1 to 10 collect (* 1000 i)))

contrib/sb-simd/examples/simd-dot.lisp

@@ -0,0 +1,20 @@
+(in-package :sb-simd-avx)
+
+(defun simd-dot (array1 array2 &aux (n (min (array-total-size array1) (array-total-size array2))))
+  (declare (type (simple-array double-float 1) array1 array2)
+           (optimize speed (safety 0)))
+  (do ((index 0 (the (integer 0 #.(- array-total-size-limit 16)) (+ index 16)))
+       (acc1 (f64.4 0) (f64.4+ acc1 (f64.4* (f64.4-row-major-aref array1 (+ index 0))
+                                            (f64.4-row-major-aref array2 (+ index 0)))))
+       (acc2 (f64.4 0) (f64.4+ acc2 (f64.4* (f64.4-row-major-aref array1 (+ index 4))
+                                            (f64.4-row-major-aref array2 (+ index 4)))))
+       (acc3 (f64.4 0) (f64.4+ acc3 (f64.4* (f64.4-row-major-aref array1 (+ index 8))
+                                            (f64.4-row-major-aref array2 (+ index 8)))))
+       (acc4 (f64.4 0) (f64.4+ acc4 (f64.4* (f64.4-row-major-aref array1 (+ index 12))
+                                            (f64.4-row-major-aref array2 (+ index 12))))))
+      ((>= index (- n 16))
+       (do ((result (f64.4-horizontal+ (f64.4+ acc1 acc2 acc3 acc4))
+                    (+ result (* (row-major-aref array1 index)
+                                 (row-major-aref array2 index))))
+            (index index (1+ index)))
+           ((>= index n) result)))))

contrib/sb-simd/examples/simd-sum.lisp

@@ -0,0 +1,15 @@
+(in-package :sb-simd-avx)
+
+(defun simd-sum (array &aux (n (array-total-size array)))
+  (declare (type (simple-array double-float (*)) array)
+           (optimize speed (safety 0)))
+  (do ((index 0 (the (integer 0 #.(- array-total-size-limit 16)) (+ index 16)))
+       (acc1 (f64.4 0) (f64.4+ acc1 (f64.4-aref array (+ index 0))))
+       (acc2 (f64.4 0) (f64.4+ acc2 (f64.4-aref array (+ index 4))))
+       (acc3 (f64.4 0) (f64.4+ acc3 (f64.4-aref array (+ index 8))))
+       (acc4 (f64.4 0) (f64.4+ acc4 (f64.4-aref array (+ index 12)))))
+      ((>= index (- n 16))
+       (do ((result (f64.4-horizontal+ (f64.4+ acc1 acc2 acc3 acc4))
+                    (+ result (row-major-aref array index)))
+            (index index (1+ index)))
+           ((>= index n) result)))))

contrib/sb-simd/sb-simd.texinfo

@@ -0,0 +1,230 @@
+@node sb-simd
+@section sb-simd
+@cindex SIMD
+
+The @code{sb-simd} module provides a convenient interface for SIMD
+programming in SBCL.  It provides one package per SIMD instruction set,
+plus functions and macros for querying whether an instruction set is
+available and what functions and data types it exports.
+
+@subsection Data Types
+
+The central data type in @code{sb-simd} is the SIMD pack.  A SIMD pack
+is very similar to a specialized vector, except that its length must be
+a particular power of two that depends on its element type and the
+underlying hardware.  The set of element types that are supported for
+SIMD packs is similar to that of SBCL's specialized array element types,
+except that there is currently no support for SIMD packs of complex
+numbers or characters.
+
+The supported scalar types are @code{f32}, @code{f64}, @code{sN}, and
+@code{uN}, where @code{N} is either 8, 16, 32, or 64.  These scalar
+types are abbreviations for the Common Lisp types @code{single-float},
+@code{double-float}, @code{signed-byte}, and @code{unsigned-byte},
+respectively.  For each scalar data type @code{X}, there exists one or
+more SIMD data type @code{X.Y} with @code{Y} elements.  For example, in
+AVX there are two supported SIMD data types with element type
+@code{f64}, namely @code{f64.2} (128 bit) and @code{f64.4} (256 bit).
+
+SIMD packs are regular Common Lisp objects that have a type, a class,
+and can be passed as function arguments.  The price for this is that
+SIMD packs have both a boxed and an unboxed representation.  The unboxed
+representation of a SIMD pack has zero overhead and fits into a CPU
+register, but can only be used within a function and when the compiler
+can statically determine the SIMD pack's type.  Otherwise, the SIMD pack
+is boxed, i.e., spilled to the heap together with its type information.
+In practice, boxing of SIMD packs can usually be avoided via inlining,
+or by loading and storing them to specialized arrays instead of passing
+them around as function arguments.
+
+@subsection Casts
+
+For each scalar data type @code{X}, there is a function named @code{X}
+that is equivalent to @code{(lambda (v) (coerce v 'X))}.  For each SIMD
+data type @code{X.Y}, there is a function named @code{X.Y} that ensures
+that its argument is of type @code{X.Y}, or, if the argument is a number,
+calls the cast function of @code{X} and broadcasts the result.
+
+All functions provided by @code{sb-simd} (apart from the casts
+themselves) implicitly cast each argument to its expected type.  So to
+add the number five to each single float in a SIMD pack @code{x} of type
+@code{f32.8}, it is sufficient to write @code{(f32.8+ x 5)}.  We don't
+mention this implicit conversion explicitly in the following sections,
+so if any function description states that an argument must be of type
+@code{X.Y}, the argument can actually be of any type that is a suitable
+argument of the cast function named @code{X.Y}.
+
+@subsection Constructors
+
+For each SIMD data type @code{X.Y}, there is a constructor named
+@code{make-X.Y} that takes @code{Y} arguments of type @code{X} and
+returns a SIMD pack whose elements are the supplied values.
+
+@subsection Unpackers
+
+For each SIMD data type @code{X.Y}, there is a function named
+@code{X.Y-values} that returns, as @code{Y} multiple values, the
+elements of the supplied SIMD pack of type @code{X.Y}.
+
+@subsection Reinterpret Casts
+
+For each SIMD data type @code{X.Y}, there is a function named
+@code{X.Y!} that takes any SIMD pack or scalar datum and interprets its
+bits as a SIMD pack of type @code{X.Y}.  If the supplied datum has more
+bits than the resulting value, the excess bits are discarded.  If the
+supplied datum has less bits than the resulting value, the missing bits are
+assumed to be zero.
+
+@subsection Associatives
+
+For each associative binary function, e.g., @code{two-arg-X.Y-OP}, there
+is a function @code{X.Y-OP} that takes any number of arguments and
+combines them with this binary function in a tree-like fashion.  If the
+binary function has an identity element, it is possible to call the
+function with zero arguments, in which case the identity element is
+returned.  If there is no identity element, the function must receive at
+least one argument.
+
+Examples of associative functions are @code{f32.8+}, for summing any
+number of 256 bit packs of single floats, and @code{u8.32-max}, for
+computing the element-wise maximum of one or more 256 bit packs of 8 bit
+integers.
+
+@subsection Reducers
+
+For binary functions @code{two-arg-X.Y-OP} that are not associative, but
+that have a neutral element, there are functions @code{X.Y-OP} that take
+any positive number of arguments and return the reduction of all
+arguments with the binary function.  In the special case of a single
+supplied argument, the binary function is invoked on the neutral element
+and that argument.  Reducers have been introduced to generate Lisp-style
+subtraction and division functions.
+
+Examples of reducers are @code{f32.8/}, for successively dividing a pack
+of 32 bit single floats by all further supplied packs of 32 bit single
+floats, or @code{u32.8-} for subtracting any number of supplied packs of
+32 bit unsigned integers from the first supplied one, except in the case
+of a single argument, where @code{u32.8-} simply negates all values in
+the pack.
+
+@subsection Rounding
+
+For each floating-point SIMD data type @code{X.Y} there are several
+functions that round the values of a supplied SIMD pack to nearby
+floating-point values whose fractional digits are all zero.  Those
+functions are @code{X.Y-round}, @code{X.Y-floor}, @code{X.Y-ceiling},
+and @code{X.Y-truncate}, and they have the same semantics as the one
+argument versions of @code{cl:round}, @code{cl:floor},
+@code{cl:ceiling}, and @code{cl:truncate}, respectively.
+
+@subsection Comparisons
+
+For each SIMD data type @code{X.Y}, there exist conversion functions
+@code{X.Y<}, @code{X.Y<=}, @code{X.Y>}, @code{X.Y>=}, and
+@code{X.Y=} that check whether the supplied arguments are strictly
+monotonically increasing, monotonically increasing, strictly monotonically
+decreasing, monotonically decreasing, equal, or nowhere equal,
+respectively.  In contrast to the Common Lisp functions @code{<},
+@code{<=}, @code{>}, @code{>=}, @code{=}, and @code{/=} the SIMD
+comparison functions don't return a generalized boolean, but a SIMD pack of
+unsigned integers with @code{Y} elements.  The bits of each unsigned
+integer are either all one, if the values of the arguments at that position
+satisfy the test, or all zero, if they don't.  We call a SIMD packs of such
+unsigned integers a mask.
+
+@subsection Conditionals
+
+The SIMD paradigm is inherently incompatible with fine-grained control
+flow.  A piece of code containing an @code{if} special form cannot be
+vectorized in a straightforward way, because doing so would require as
+many instruction pointers and processor states as there are values in
+the desired SIMD data type.  Instead, most SIMD instruction sets provide
+an operator for selecting values from one of two supplied SIMD packs
+based on a mask.  The mask is a SIMD pack with as many elements as the
+other two arguments, but whose elements are unsigned integers whose bits
+must be either all zeros or all ones.  This selection mechanism can be
+used to emulate the effect of an @code{if} special form, at the price
+that both operands have to be computed each time.
+
+In @code{sb-simd}, all conditional operations and comparisons emit
+suitable mask fields, and there is a @code{X.Y-if} function for each
+SIMD data type with element type @code{X} and number of elements
+@code{Y} whose first arguments must be a suitable mask, whose second and
+third argument must be objects that can be converted to the SIMD data
+type @code{X.Y}, and that returns a value of type @code{X.Y} where each
+element is from the second operand if the corresponding mask bits are
+set, and from the third operand if the corresponding mask bits are not
+set.
+
+@subsection Loads and Stores
+
+In practice, a SIMD pack @code{X.Y} is usually not constructed by
+calling its constructor, but by loading @code{Y} consecutive elements
+from a specialized array with element type @code{X}.  The functions for
+doing so are called @code{X.Y-aref} and @code{X.Y-row-major-aref}, and
+have similar semantics as Common Lisp's @code{aref} and
+@code{row-major-aref}.  In addition to that, some instruction sets
+provide the functions @code{X.Y-non-temporal-aref} and
+@code{X.Y-non-temporal-row-major-aref}, for accessing a memory location
+without loading the referenced values into the CPU's cache.
+
+For each function @code{X.Y-foo} for loading SIMD packs from an array,
+there also exists a corresponding function @code{(setf X.Y-foo)} for
+storing a SIMD pack in the specified memory location.  An exception to
+this rule is that some instruction sets (e.g., SSE) only provide
+functions for non-temporal stores, but not for the corresponding
+non-temporal loads.
+
+One difficulty when treating the data of a Common Lisp array as a SIMD
+pack is that some hardware instructions require a particular alignment
+of the address being referenced.  Luckily, most architectures provide
+instructions for unaligned loads and stores that are, at least on modern
+CPUs, not slower than their aligned equivalents.  So by default we
+translate all array references as unaligned loads and stores.  An
+exception are the instructions for non-temporal loads and stores, that
+always require a certain alignment.  We do not handle this case
+specially, so without special handling by the user, non-temporal loads
+and stores will only work on certain array indices that depend on the
+actual placement of that array in memory.
+
+@subsection Specialized Scalar Operations
+
+Finally, for each SIMD function @code{X.Y-OP} that applies a certain
+operation @code{OP} element-wise to the @code{Y} elements of type
+@code{X}, there exists also a functions @code{X-OP} for applying that
+operation only to a single element.  For example, the SIMD function
+@code{f64.4+} has a corresponding function @code{f64+} that differs from
+@code{cl:+} in that it only accepts arguments of type double float, and
+that it adds its supplied arguments in a fixed order that is the same as
+the one used by @code{f64.4}.
+
+There are good reasons for exporting scalar functions from a SIMD
+library, too.  The most obvious one is that they obey the same naming
+convention and hence make it easier to locate the correct functions.
+Another benefit is that the semantics of each scalar operation is
+precisely the same as that of the corresponding SIMD function, so they
+can be used to write reference implementations for testing.  A final
+reason is that these scalar functions can be used to simplify the life
+of tools for automatic vectorization.
+
+@subsection Instruction Set Dispatch
+
+One challenge that is unique to image-based programming systems such as
+Lisp is that a program can run on one machine, be dumped as an image,
+and then resumed on another machine.  While nobody expects this feature
+to work across machines with different architectures, it is quite likely
+that the machine where the image is dumped and the one where execution
+is resumed provide different instruction set extensions.
+
+As a practical example, consider a game developer that develops software
+on an x86-64 machine with all SIMD extensions up to AVX2, but then dumps
+it as an image and ships it to a customer whose machine only supports
+SIMD extensions up to SSE2.  Ideally, the image should contain multiple
+optimized versions of all crucial functions, and dynamically select the
+most appropriate version based on the instruction set extensions that
+are actually available.
+
+This kind of run time instruction set dispatch is explicitly supported
+by means of the @code{instruction-set-case} macro.  The code resulting
+from an invocation of this macro compiles to an efficient jump table
+whose index is recomputed on each startup of the Lisp image.

contrib/sb-simd/test-suite/numbers.lisp

@@ -0,0 +1,28 @@
+(in-package #:sb-simd-test-suite)
+
+(defparameter *numbers*
+  (remove-duplicates
+   `(,most-positive-double-float
+     ,most-negative-double-float
+     ,least-positive-normalized-double-float
+     ,least-negative-normalized-double-float
+     ,least-positive-double-float
+     ,least-negative-double-float
+     ,most-positive-single-float
+     ,most-negative-single-float
+     ,least-positive-normalized-single-float
+     ,least-negative-normalized-single-float
+     ,least-positive-single-float
+     ,least-negative-single-float
+     ,@(loop for type in '(single-float double-float integer)
+             append (loop for i from 0 to 64
+                          for n = (expt 2 i)
+                          collect (coerce n type)
+                          collect (coerce (- n) type)
+                          collect (coerce (1- n) type)
+                          collect (coerce (1+ n) type)
+                          collect (coerce (1- (- n)) type)
+                          collect (coerce (1+ (- n)) type))))))
+
+(defun numbers-of-type (type)
+  (remove-if-not (lambda (x) (typep x type)) *numbers*))

contrib/sb-simd/test-suite/packages.lisp

@@ -0,0 +1,15 @@
+(cl:in-package #:common-lisp-user)
+
+(defpackage #:sb-simd-test-suite
+  (:use #:common-lisp #:sb-simd-internals)
+  (:export
+   #:run-test-suite
+   #:define-test
+   #:is
+   #:signals
+   #:all-tests
+   #:check-package
+   #:run-tests
+   #:define-simple-simd-test
+   #:define-horizontal-test
+   #:define-aref-test))

contrib/sb-simd/test-suite/sb-simd-test-suite.asd

@@ -0,0 +1,21 @@
+(defsystem "sb-simd-test-suite"
+  :description "The sb-simd test suite."
+  :author "Marco Heisig <marco.heisig@fau.de>"
+  :license "MIT"
+
+  :depends-on ("sb-simd")
+
+  :perform
+  (test-op (o c) (symbol-call '#:sb-simd-test-suite '#:run-test-suite))
+
+  :serial t
+  :components
+  ((:file "packages")
+   (:file "numbers")
+   (:file "utilities")
+   (:file "test-suite")
+   (:file "test-arefs")
+   (:file "test-simple-simd-functions")
+   (:file "test-horizontal-functions")
+   (:file "test-hairy-simd-functions")
+   (:file "test-packages")))

contrib/sb-simd/test-suite/test-arefs.lisp

@@ -0,0 +1,92 @@
+(in-package #:sb-simd-test-suite)
+
+(defmacro define-aref-test (aref element-type simd-width &optional (unpacker 'identity))
+  (let ((value-symbols (prefixed-symbols "V" simd-width))
+        (zero (coerce 0 element-type))
+        (one (coerce 1 element-type)))
+    `(define-test ,aref
+       ;; Create an array of zeros and successively replace zeros with
+       ;; ones.  After each replacement, check whether a load still
+       ;; produces the expected result.
+       (let ((array (make-array '(,simd-width)
+                                 :element-type ',element-type
+                                 :initial-element ,zero)))
+         (multiple-value-bind ,value-symbols (,unpacker (,aref array 0))
+           ,@(loop for value-symbol in value-symbols
+                   collect `(is (= ,value-symbol ,zero))))
+         (loop for index below ,simd-width do
+           (setf (aref array index) ,one)
+           (loop for number in (multiple-value-list (,unpacker (,aref array 0)))
+                 for position from 0
+                 do (if (<= position index)
+                        (is (= number ,one))
+                        (is (= number ,zero))))))
+       ;; Create an array with twice as many elements as the width of the
+       ;; SIMD data type, and whose lower half consists of all zeros and
+       ;; whose upper half consists of all ones.  Check that all valid
+       ;; loads from this array have the expected state.
+       (let ((array (make-array '(,(* 2 simd-width))
+                                :element-type ',element-type
+                                :initial-contents
+                                (append (make-list ,simd-width :initial-element ,zero)
+                                        (make-list ,simd-width :initial-element ,one)))))
+         (loop for index below ,simd-width do
+           (multiple-value-bind ,value-symbols
+               (,unpacker (,aref array index))
+             ,@(loop for value-symbol in value-symbols
+                     for position from 0
+                     collect `(if (< (+ ,position index) ,simd-width)
+                                  (is (= ,value-symbol ,zero))
+                                  (is (= ,value-symbol ,one))))))))))
+
+(in-package #:sb-simd)
+
+(sb-simd-test-suite:define-aref-test u8-aref  (unsigned-byte 8) 1)
+(sb-simd-test-suite:define-aref-test u16-aref (unsigned-byte 16) 1)
+(sb-simd-test-suite:define-aref-test u32-aref (unsigned-byte 32) 1)
+(sb-simd-test-suite:define-aref-test u64-aref (unsigned-byte 64) 1)
+(sb-simd-test-suite:define-aref-test s8-aref  (signed-byte 8) 1)
+(sb-simd-test-suite:define-aref-test s16-aref (signed-byte 16) 1)
+(sb-simd-test-suite:define-aref-test s32-aref (signed-byte 32) 1)
+(sb-simd-test-suite:define-aref-test s64-aref (signed-byte 64) 1)
+(sb-simd-test-suite:define-aref-test f32-aref single-float 1)
+(sb-simd-test-suite:define-aref-test f64-aref double-float 1)
+
+(in-package #:sb-simd-sse)
+
+(sb-simd-test-suite:define-aref-test f32.4-aref single-float 4 f32.4-values)
+
+(in-package #:sb-simd-sse2)
+
+(sb-simd-test-suite:define-aref-test f64.2-aref double-float       2 f64.2-values)
+(sb-simd-test-suite:define-aref-test u8.16-aref (unsigned-byte 8) 16 u8.16-values)
+(sb-simd-test-suite:define-aref-test u16.8-aref (unsigned-byte 16) 8 u16.8-values)
+(sb-simd-test-suite:define-aref-test u32.4-aref (unsigned-byte 32) 4 u32.4-values)
+(sb-simd-test-suite:define-aref-test u64.2-aref (unsigned-byte 64) 2 u64.2-values)
+(sb-simd-test-suite:define-aref-test s8.16-aref (signed-byte 8)   16 s8.16-values)
+(sb-simd-test-suite:define-aref-test s16.8-aref (signed-byte 16)   8 s16.8-values)
+(sb-simd-test-suite:define-aref-test s32.4-aref (signed-byte 32)   4 s32.4-values)
+(sb-simd-test-suite:define-aref-test s64.2-aref (signed-byte 64)   2 s64.2-values)
+
+(in-package #:sb-simd-avx)
+
+(sb-simd-test-suite:define-aref-test f32.4-aref single-float 4 f32.4-values)
+(sb-simd-test-suite:define-aref-test f32.8-aref single-float 8 f32.8-values)
+(sb-simd-test-suite:define-aref-test f64.2-aref double-float 2 f64.2-values)
+(sb-simd-test-suite:define-aref-test f64.4-aref double-float 4 f64.4-values)
+(sb-simd-test-suite:define-aref-test u8.16-aref (unsigned-byte 8) 16 u8.16-values)
+(sb-simd-test-suite:define-aref-test u16.8-aref (unsigned-byte 16) 8 u16.8-values)
+(sb-simd-test-suite:define-aref-test u32.4-aref (unsigned-byte 32) 4 u32.4-values)
+(sb-simd-test-suite:define-aref-test u64.2-aref (unsigned-byte 64) 2 u64.2-values)
+(sb-simd-test-suite:define-aref-test s8.16-aref (signed-byte 8) 16 s8.16-values)
+(sb-simd-test-suite:define-aref-test s16.8-aref (signed-byte 16) 8 s16.8-values)
+(sb-simd-test-suite:define-aref-test s32.4-aref (signed-byte 32) 4 s32.4-values)
+(sb-simd-test-suite:define-aref-test s64.2-aref (signed-byte 64) 2 s64.2-values)
+(sb-simd-test-suite:define-aref-test u8.32-aref (unsigned-byte 8) 32 u8.32-values)
+(sb-simd-test-suite:define-aref-test u16.16-aref (unsigned-byte 16) 16 u16.16-values)
+(sb-simd-test-suite:define-aref-test u32.8-aref (unsigned-byte 32) 8 u32.8-values)
+(sb-simd-test-suite:define-aref-test u64.4-aref (unsigned-byte 64) 4 u64.4-values)
+(sb-simd-test-suite:define-aref-test s8.32-aref (signed-byte 8) 32 s8.32-values)
+(sb-simd-test-suite:define-aref-test s16.16-aref (signed-byte 16) 16 s16.16-values)
+(sb-simd-test-suite:define-aref-test s32.8-aref (signed-byte 32) 8 s32.8-values)
+(sb-simd-test-suite:define-aref-test s64.4-aref (signed-byte 64) 4 s64.4-values)

contrib/sb-simd/test-suite/test-hairy-simd-functions.lisp

@@ -0,0 +1,9 @@
+(in-package #:sb-simd-test-suite)
+
+(in-package #:sb-simd-sse)
+
+(in-package #:sb-simd-sse2)
+
+(in-package #:sb-simd-avx)
+
+(in-package #:sb-simd-avx2)

contrib/sb-simd/test-suite/test-horizontal-functions.lisp

@@ -0,0 +1,76 @@
+(in-package #:sb-simd-test-suite)
+
+(defmacro define-horizontal-test (horizontal-fn scalar-fn)
+  (let* ((horizontal-record (find-function-record horizontal-fn))
+         (scalar-record (find-function-record scalar-fn))
+         (argument-record (first (function-record-required-argument-records horizontal-record)))
+         (result-record (function-record-result-record horizontal-record))
+         (width (value-record-simd-width argument-record))
+         (args (loop repeat width collect (gensym)))
+         (pack (third (simd-info (value-record-name argument-record)))))
+    (assert (eq (function-record-result-record scalar-record) result-record))
+    `(define-test ,horizontal-fn
+       (let ((generator (find-generator ',(value-record-name result-record))))
+         (loop repeat 99 do
+           (let ,(loop for arg in args collect `(,arg (funcall generator)))
+             (handler-case
+                 (assert (bitwise= (,horizontal-fn (,pack ,@args))
+                                   (,scalar-fn ,@args)))
+               (floating-point-overflow ())
+               (floating-point-overflow ()))))))))
+
+(in-package #:sb-simd-sse)
+
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-and f32-and)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-or  f32-or)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-xor f32-xor)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-max f32-max)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-min f32-min)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal+ f32+)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal* f32*)
+
+(in-package #:sb-simd-sse2)
+
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-and f64-and)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-or  f64-or)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-xor f64-xor)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-max f64-max)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-min f64-min)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal+ f64+)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal* f64*)
+
+(in-package #:sb-simd-avx)
+
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-and f32-and)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-or  f32-or)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-xor f32-xor)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-max f32-max)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal-min f32-min)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal+ f32+)
+(sb-simd-test-suite:define-horizontal-test f32.4-horizontal* f32*)
+
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-and f64-and)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-or  f64-or)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-xor f64-xor)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-max f64-max)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal-min f64-min)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal+ f64+)
+(sb-simd-test-suite:define-horizontal-test f64.2-horizontal* f64*)
+
+(sb-simd-test-suite:define-horizontal-test f32.8-horizontal-and f32-and)
+(sb-simd-test-suite:define-horizontal-test f32.8-horizontal-or  f32-or)
+(sb-simd-test-suite:define-horizontal-test f32.8-horizontal-xor f32-xor)
+(sb-simd-test-suite:define-horizontal-test f32.8-horizontal-max f32-max)
+(sb-simd-test-suite:define-horizontal-test f32.8-horizontal-min f32-min)
+(sb-simd-test-suite:define-horizontal-test f32.8-horizontal+ f32+)
+(sb-simd-test-suite:define-horizontal-test f32.8-horizontal* f32*)
+
+(sb-simd-test-suite:define-horizontal-test f64.4-horizontal-and f64-and)
+(sb-simd-test-suite:define-horizontal-test f64.4-horizontal-or  f64-or)
+(sb-simd-test-suite:define-horizontal-test f64.4-horizontal-xor f64-xor)
+(sb-simd-test-suite:define-horizontal-test f64.4-horizontal-max f64-max)
+(sb-simd-test-suite:define-horizontal-test f64.4-horizontal-min f64-min)
+(sb-simd-test-suite:define-horizontal-test f64.4-horizontal+ f64+)
+(sb-simd-test-suite:define-horizontal-test f64.4-horizontal* f64*)
+
+(in-package #:sb-simd-avx2)

contrib/sb-simd/test-suite/test-packages.lisp

@@ -0,0 +1,16 @@
+(in-package #:sb-simd-test-suite)
+
+(define-test packages)
+
+(define-test packages
+  (check-package '#:sb-simd-internals)
+  (check-package '#:sb-simd)
+  (check-package '#:sb-simd-x86-64)
+  (check-package '#:sb-simd-sse)
+  (check-package '#:sb-simd-sse2)
+  (check-package '#:sb-simd-sse3)
+  (check-package '#:sb-simd-ssse3)
+  (check-package '#:sb-simd-sse4.1)
+  (check-package '#:sb-simd-sse4.2)
+  (check-package '#:sb-simd-avx)
+  (check-package '#:sb-simd-avx2))

contrib/sb-simd/test-suite/test-simple-simd-functions.lisp

@@ -0,0 +1,543 @@
+(in-package #:sb-simd-test-suite)
+
+(defmacro define-simple-simd-test (simd-foo result-types argtypes foo)
+  `(define-test ,simd-foo
+     ,@(loop for argtype-variant in (argtypes-variants argtypes)
+             collect `(test-simple-simd-function ,simd-foo ,result-types ,argtype-variant ,foo))))
+
+(defmacro test-simple-simd-function (simd-foo result-types simplified-argtypes foo)
+  (let* ((result-infos (mapcar #'simd-info result-types))
+         (argument-infos (mapcar #'simd-info simplified-argtypes))
+         (argument-symbols (prefixed-symbols "ARGUMENT-" (length argument-infos)))
+         (result-symbols (prefixed-symbols "RESULT-" (length result-infos)))
+         (output-symbols (prefixed-symbols "OUTPUT-" (length result-infos)))
+         (simd-width (the integer (second (first result-infos)))))
+    (assert (apply #'= (append (mapcar #'second result-infos) (mapcar #'second argument-infos))))
+    `(loop repeat ,(min (expt 99 (length argument-infos)) 10000) do
+      (multiple-value-bind (inputs outputs)
+          (find-valid-simd-call
+           (lambda ,argument-symbols
+             (declare ,@(loop for argument-symbol in argument-symbols
+                              for argument-type in (mapcar #'first argument-infos)
+                              collect `(type ,argument-type ,argument-symbol)))
+             (,foo ,@argument-symbols))
+           ',(mapcar #'find-generator (mapcar #'first argument-infos))
+           ,simd-width
+           ',(mapcar #'third argument-infos)
+           ',(mapcar #'third result-infos))
+        (destructuring-bind ,argument-symbols inputs
+          (destructuring-bind ,output-symbols outputs
+            (multiple-value-bind ,result-symbols (,simd-foo ,@argument-symbols)
+              ,@(loop for result-type in result-types
+                      for result-symbol in result-symbols
+                      for output-symbol in output-symbols
+                      collect
+                      `(is (simd= ,result-symbol ,output-symbol))))))))))
+
+(in-package #:sb-simd-sse)
+
+(sb-simd-test-suite:define-simple-simd-test f32.4-and (f32.4) (&rest f32.4) f32-and)
+(sb-simd-test-suite:define-simple-simd-test f32.4-or (f32.4) (&rest f32.4) f32-or)
+(sb-simd-test-suite:define-simple-simd-test f32.4-xor (f32.4) (&rest f32.4) f32-xor)
+(sb-simd-test-suite:define-simple-simd-test f32.4-andc1 (f32.4) (f32.4 f32.4) f32-andc1)
+(sb-simd-test-suite:define-simple-simd-test f32.4-not (f32.4) (f32.4) f32-not)
+(sb-simd-test-suite:define-simple-simd-test f32.4-max (f32.4) (f32.4 &rest f32.4) f32-max)
+(sb-simd-test-suite:define-simple-simd-test f32.4-min (f32.4) (f32.4 &rest f32.4) f32-min)
+(sb-simd-test-suite:define-simple-simd-test f32.4+ (f32.4) (&rest f32.4) f32+)
+(sb-simd-test-suite:define-simple-simd-test f32.4- (f32.4) (f32.4 &rest f32.4) f32-)
+(sb-simd-test-suite:define-simple-simd-test f32.4* (f32.4) (&rest f32.4) f32*)
+(sb-simd-test-suite:define-simple-simd-test f32.4/ (f32.4) (f32.4 &rest f32.4) f32/)
+
+(in-package #:sb-simd-sse2)
+
+(sb-simd-test-suite:define-simple-simd-test f32.4= (u32.4) (f32.4 &rest f32.4) f32=)
+(sb-simd-test-suite:define-simple-simd-test f32.4/= (u32.4) (f32.4 &rest f32.4) f32/=)
+(sb-simd-test-suite:define-simple-simd-test f32.4< (u32.4) (f32.4 &rest f32.4) f32<)
+(sb-simd-test-suite:define-simple-simd-test f32.4<= (u32.4) (f32.4 &rest f32.4) f32<=)
+(sb-simd-test-suite:define-simple-simd-test f32.4> (u32.4) (f32.4 &rest f32.4) f32>)
+(sb-simd-test-suite:define-simple-simd-test f32.4>= (u32.4) (f32.4 &rest f32.4) f32>=)
+
+(sb-simd-test-suite:define-simple-simd-test f64.2-and (f64.2) (&rest f64.2) f64-and)
+(sb-simd-test-suite:define-simple-simd-test f64.2-or (f64.2) (&rest f64.2) f64-or)
+(sb-simd-test-suite:define-simple-simd-test f64.2-xor (f64.2) (&rest f64.2) f64-xor)
+(sb-simd-test-suite:define-simple-simd-test f64.2-andc1 (f64.2) (f64.2 f64.2) f64-andc1)
+(sb-simd-test-suite:define-simple-simd-test f64.2-not (f64.2) (f64.2) f64-not)
+(sb-simd-test-suite:define-simple-simd-test f64.2-max (f64.2) (f64.2 &rest f64.2) f64-max)
+(sb-simd-test-suite:define-simple-simd-test f64.2-min (f64.2) (f64.2 &rest f64.2) f64-min)
+(sb-simd-test-suite:define-simple-simd-test f64.2+ (f64.2) (&rest f64.2) f64+)
+(sb-simd-test-suite:define-simple-simd-test f64.2- (f64.2) (f64.2 &rest f64.2) f64-)
+(sb-simd-test-suite:define-simple-simd-test f64.2* (f64.2) (&rest f64.2) f64*)
+(sb-simd-test-suite:define-simple-simd-test f64.2/ (f64.2) (f64.2 &rest f64.2) f64/)
+(sb-simd-test-suite:define-simple-simd-test f64.2= (u64.2) (f64.2 &rest f64.2) f64=)
+(sb-simd-test-suite:define-simple-simd-test f64.2/= (u64.2) (f64.2 &rest f64.2) f64/=)
+(sb-simd-test-suite:define-simple-simd-test f64.2< (u64.2) (f64.2 &rest f64.2) f64<)
+(sb-simd-test-suite:define-simple-simd-test f64.2<= (u64.2) (f64.2 &rest f64.2) f64<=)
+(sb-simd-test-suite:define-simple-simd-test f64.2> (u64.2) (f64.2 &rest f64.2) f64>)
+(sb-simd-test-suite:define-simple-simd-test f64.2>= (u64.2) (f64.2 &rest f64.2) f64>=)
+
+(sb-simd-test-suite:define-simple-simd-test u8.16-and (u8.16) (&rest u8.16) u8-and)
+(sb-simd-test-suite:define-simple-simd-test u8.16-or (u8.16) (&rest u8.16) u8-or)
+(sb-simd-test-suite:define-simple-simd-test u8.16-xor (u8.16) (&rest u8.16) u8-xor)
+(sb-simd-test-suite:define-simple-simd-test u8.16-andc1 (u8.16) (u8.16 u8.16) u8-andc1)
+(sb-simd-test-suite:define-simple-simd-test u8.16-not (u8.16) (u8.16) u8-not)
+(sb-simd-test-suite:define-simple-simd-test u8.16+ (u8.16) (&rest u8.16) u8+)
+(sb-simd-test-suite:define-simple-simd-test u8.16- (u8.16) (u8.16 &rest u8.16) u8-)
+(sb-simd-test-suite:define-simple-simd-test u8.16= (u8.16) (u8.16 &rest u8.16) u8=)
+(sb-simd-test-suite:define-simple-simd-test u8.16/= (u8.16) (u8.16 &rest u8.16) u8/=)
+(sb-simd-test-suite:define-simple-simd-test u8.16< (u8.16) (u8.16 &rest u8.16) u8<)
+(sb-simd-test-suite:define-simple-simd-test u8.16<= (u8.16) (u8.16 &rest u8.16) u8<=)
+(sb-simd-test-suite:define-simple-simd-test u8.16> (u8.16) (u8.16 &rest u8.16) u8>)
+(sb-simd-test-suite:define-simple-simd-test u8.16>= (u8.16) (u8.16 &rest u8.16) u8>=)
+
+(sb-simd-test-suite:define-simple-simd-test u16.8-and (u16.8) (&rest u16.8) u16-and)
+(sb-simd-test-suite:define-simple-simd-test u16.8-or (u16.8) (&rest u16.8) u16-or)
+(sb-simd-test-suite:define-simple-simd-test u16.8-xor (u16.8) (&rest u16.8) u16-xor)
+(sb-simd-test-suite:define-simple-simd-test u16.8-andc1 (u16.8) (u16.8 u16.8) u16-andc1)
+(sb-simd-test-suite:define-simple-simd-test u16.8-not (u16.8) (u16.8) u16-not)
+(sb-simd-test-suite:define-simple-simd-test u16.8+ (u16.8) (&rest u16.8) u16+)
+(sb-simd-test-suite:define-simple-simd-test u16.8- (u16.8) (u16.8 &rest u16.8) u16-)
+(sb-simd-test-suite:define-simple-simd-test u16.8= (u16.8) (u16.8 &rest u16.8) u16=)
+(sb-simd-test-suite:define-simple-simd-test u16.8/= (u16.8) (u16.8 &rest u16.8) u16/=)
+(sb-simd-test-suite:define-simple-simd-test u16.8< (u16.8) (u16.8 &rest u16.8) u16<)
+(sb-simd-test-suite:define-simple-simd-test u16.8<= (u16.8) (u16.8 &rest u16.8) u16<=)
+(sb-simd-test-suite:define-simple-simd-test u16.8> (u16.8) (u16.8 &rest u16.8) u16>)
+(sb-simd-test-suite:define-simple-simd-test u16.8>= (u16.8) (u16.8 &rest u16.8) u16>=)
+
+(sb-simd-test-suite:define-simple-simd-test u32.4-and (u32.4) (&rest u32.4) u32-and)
+(sb-simd-test-suite:define-simple-simd-test u32.4-or (u32.4) (&rest u32.4) u32-or)
+(sb-simd-test-suite:define-simple-simd-test u32.4-xor (u32.4) (&rest u32.4) u32-xor)
+(sb-simd-test-suite:define-simple-simd-test u32.4-andc1 (u32.4) (u32.4 u32.4) u32-andc1)
+(sb-simd-test-suite:define-simple-simd-test u32.4-not (u32.4) (u32.4) u32-not)
+(sb-simd-test-suite:define-simple-simd-test u32.4+ (u32.4) (&rest u32.4) u32+)
+(sb-simd-test-suite:define-simple-simd-test u32.4- (u32.4) (u32.4 &rest u32.4) u32-)
+(sb-simd-test-suite:define-simple-simd-test u32.4= (u32.4) (u32.4 &rest u32.4) u32=)
+(sb-simd-test-suite:define-simple-simd-test u32.4/= (u32.4) (u32.4 &rest u32.4) u32/=)
+(sb-simd-test-suite:define-simple-simd-test u32.4< (u32.4) (u32.4 &rest u32.4) u32<)
+(sb-simd-test-suite:define-simple-simd-test u32.4<= (u32.4) (u32.4 &rest u32.4) u32<=)
+(sb-simd-test-suite:define-simple-simd-test u32.4> (u32.4) (u32.4 &rest u32.4) u32>)
+(sb-simd-test-suite:define-simple-simd-test u32.4>= (u32.4) (u32.4 &rest u32.4) u32>=)
+
+(sb-simd-test-suite:define-simple-simd-test u64.2-and (u64.2) (&rest u64.2) u64-and)
+(sb-simd-test-suite:define-simple-simd-test u64.2-or (u64.2) (&rest u64.2) u64-or)
+(sb-simd-test-suite:define-simple-simd-test u64.2-xor (u64.2) (&rest u64.2) u64-xor)
+(sb-simd-test-suite:define-simple-simd-test u64.2-andc1 (u64.2) (u64.2 u64.2) u64-andc1)
+(sb-simd-test-suite:define-simple-simd-test u64.2-not (u64.2) (u64.2) u64-not)
+(sb-simd-test-suite:define-simple-simd-test u64.2+ (u64.2) (&rest u64.2) u64+)
+(sb-simd-test-suite:define-simple-simd-test u64.2- (u64.2) (u64.2 &rest u64.2) u64-)
+
+(sb-simd-test-suite:define-simple-simd-test s8.16-and (s8.16) (&rest s8.16) s8-and)
+(sb-simd-test-suite:define-simple-simd-test s8.16-or (s8.16) (&rest s8.16) s8-or)
+(sb-simd-test-suite:define-simple-simd-test s8.16-xor (s8.16) (&rest s8.16) s8-xor)
+(sb-simd-test-suite:define-simple-simd-test s8.16-andc1 (s8.16) (s8.16 s8.16) s8-andc1)
+(sb-simd-test-suite:define-simple-simd-test s8.16-not (s8.16) (s8.16) s8-not)
+(sb-simd-test-suite:define-simple-simd-test s8.16+ (s8.16) (&rest s8.16) s8+)
+(sb-simd-test-suite:define-simple-simd-test s8.16- (s8.16) (s8.16 &rest s8.16) s8-)
+(sb-simd-test-suite:define-simple-simd-test s8.16= (u8.16) (s8.16 &rest s8.16) s8=)
+(sb-simd-test-suite:define-simple-simd-test s8.16/= (u8.16) (s8.16 &rest s8.16) s8/=)
+(sb-simd-test-suite:define-simple-simd-test s8.16< (u8.16) (s8.16 &rest s8.16) s8<)
+(sb-simd-test-suite:define-simple-simd-test s8.16<= (u8.16) (s8.16 &rest s8.16) s8<=)
+(sb-simd-test-suite:define-simple-simd-test s8.16> (u8.16) (s8.16 &rest s8.16) s8>)
+(sb-simd-test-suite:define-simple-simd-test s8.16>= (u8.16) (s8.16 &rest s8.16) s8>=)
+
+(sb-simd-test-suite:define-simple-simd-test s16.8-and (s16.8) (&rest s16.8) s16-and)
+(sb-simd-test-suite:define-simple-simd-test s16.8-or (s16.8) (&rest s16.8) s16-or)
+(sb-simd-test-suite:define-simple-simd-test s16.8-xor (s16.8) (&rest s16.8) s16-xor)
+(sb-simd-test-suite:define-simple-simd-test s16.8-andc1 (s16.8) (s16.8 s16.8) s16-andc1)
+(sb-simd-test-suite:define-simple-simd-test s16.8-not (s16.8) (s16.8) s16-not)
+(sb-simd-test-suite:define-simple-simd-test s16.8+ (s16.8) (&rest s16.8) s16+)
+(sb-simd-test-suite:define-simple-simd-test s16.8- (s16.8) (s16.8 &rest s16.8) s16-)
+(sb-simd-test-suite:define-simple-simd-test s16.8= (u16.8) (s16.8 &rest s16.8) s16=)
+(sb-simd-test-suite:define-simple-simd-test s16.8/= (u16.8) (s16.8 &rest s16.8) s16/=)
+(sb-simd-test-suite:define-simple-simd-test s16.8< (u16.8) (s16.8 &rest s16.8) s16<)
+(sb-simd-test-suite:define-simple-simd-test s16.8<= (u16.8) (s16.8 &rest s16.8) s16<=)
+(sb-simd-test-suite:define-simple-simd-test s16.8> (u16.8) (s16.8 &rest s16.8) s16>)
+(sb-simd-test-suite:define-simple-simd-test s16.8>= (u16.8) (s16.8 &rest s16.8) s16>=)
+
+(sb-simd-test-suite:define-simple-simd-test s32.4-and (s32.4) (&rest s32.4) s32-and)
+(sb-simd-test-suite:define-simple-simd-test s32.4-or (s32.4) (&rest s32.4) s32-or)
+(sb-simd-test-suite:define-simple-simd-test s32.4-xor (s32.4) (&rest s32.4) s32-xor)
+(sb-simd-test-suite:define-simple-simd-test s32.4-andc1 (s32.4) (s32.4 s32.4) s32-andc1)
+(sb-simd-test-suite:define-simple-simd-test s32.4-not (s32.4) (s32.4) s32-not)
+(sb-simd-test-suite:define-simple-simd-test s32.4+ (s32.4) (&rest s32.4) s32+)
+(sb-simd-test-suite:define-simple-simd-test s32.4- (s32.4) (s32.4 &rest s32.4) s32-)
+(sb-simd-test-suite:define-simple-simd-test s32.4= (u32.4) (s32.4 &rest s32.4) s32=)
+(sb-simd-test-suite:define-simple-simd-test s32.4/= (u32.4) (s32.4 &rest s32.4) s32/=)
+(sb-simd-test-suite:define-simple-simd-test s32.4< (u32.4) (s32.4 &rest s32.4) s32<)
+(sb-simd-test-suite:define-simple-simd-test s32.4<= (u32.4) (s32.4 &rest s32.4) s32<=)
+(sb-simd-test-suite:define-simple-simd-test s32.4> (u32.4) (s32.4 &rest s32.4) s32>)
+(sb-simd-test-suite:define-simple-simd-test s32.4>= (u32.4) (s32.4 &rest s32.4) s32>=)
+
+(sb-simd-test-suite:define-simple-simd-test s64.2-and (s64.2) (&rest s64.2) s64-and)
+(sb-simd-test-suite:define-simple-simd-test s64.2-or (s64.2) (&rest s64.2) s64-or)
+(sb-simd-test-suite:define-simple-simd-test s64.2-xor (s64.2) (&rest s64.2) s64-xor)
+(sb-simd-test-suite:define-simple-simd-test s64.2-andc1 (s64.2) (s64.2 s64.2) s64-andc1)
+(sb-simd-test-suite:define-simple-simd-test s64.2-not (s64.2) (s64.2) s64-not)
+(sb-simd-test-suite:define-simple-simd-test s64.2+ (s64.2) (&rest s64.2) s64+)
+(sb-simd-test-suite:define-simple-simd-test s64.2- (s64.2) (s64.2 &rest s64.2) s64-)
+
+(in-package #:sb-simd-sse4.1)
+
+(sb-simd-test-suite:define-simple-simd-test f32.4-if (f32.4) (u32.4 f32.4 f32.4) f32-if)
+(sb-simd-test-suite:define-simple-simd-test f64.2-if (f64.2) (u64.2 f64.2 f64.2) f64-if)
+(sb-simd-test-suite:define-simple-simd-test u8.16-if (u8.16) (u8.16 u8.16 u8.16) u8-if)
+(sb-simd-test-suite:define-simple-simd-test u16.8-if (u16.8) (u16.8 u16.8 u16.8) u16-if)
+(sb-simd-test-suite:define-simple-simd-test u32.4-if (u32.4) (u32.4 u32.4 u32.4) u32-if)
+(sb-simd-test-suite:define-simple-simd-test u64.2-if (u64.2) (u64.2 u64.2 u64.2) u64-if)
+(sb-simd-test-suite:define-simple-simd-test s8.16-if (s8.16) (u8.16 s8.16 s8.16) s8-if)
+(sb-simd-test-suite:define-simple-simd-test s16.8-if (s16.8) (u16.8 s16.8 s16.8) s16-if)
+(sb-simd-test-suite:define-simple-simd-test s32.4-if (s32.4) (u32.4 s32.4 s32.4) s32-if)
+(sb-simd-test-suite:define-simple-simd-test s64.2-if (s64.2) (u64.2 s64.2 s64.2) s64-if)
+
+(sb-simd-test-suite:define-simple-simd-test u64.2= (u64.2) (u64.2 &rest u64.2) u64=)
+(sb-simd-test-suite:define-simple-simd-test u64.2/= (u64.2) (u64.2 &rest u64.2) u64/=)
+
+(sb-simd-test-suite:define-simple-simd-test s64.2= (u64.2) (s64.2 &rest s64.2) s64=)
+(sb-simd-test-suite:define-simple-simd-test s64.2/= (u64.2) (s64.2 &rest s64.2) s64/=)
+
+(in-package #:sb-simd-sse4.2)
+
+(sb-simd-test-suite:define-simple-simd-test u64.2< (u64.2) (u64.2 &rest u64.2) u64<)
+(sb-simd-test-suite:define-simple-simd-test u64.2<= (u64.2) (u64.2 &rest u64.2) u64<=)
+(sb-simd-test-suite:define-simple-simd-test u64.2> (u64.2) (u64.2 &rest u64.2) u64>)
+(sb-simd-test-suite:define-simple-simd-test u64.2>= (u64.2) (u64.2 &rest u64.2) u64>=)
+
+(sb-simd-test-suite:define-simple-simd-test s64.2< (u64.2) (s64.2 &rest s64.2) s64<)
+(sb-simd-test-suite:define-simple-simd-test s64.2<= (u64.2) (s64.2 &rest s64.2) s64<=)
+(sb-simd-test-suite:define-simple-simd-test s64.2> (u64.2) (s64.2 &rest s64.2) s64>)
+(sb-simd-test-suite:define-simple-simd-test s64.2>= (u64.2) (s64.2 &rest s64.2) s64>=)
+
+(in-package #:sb-simd-avx)
+
+(sb-simd-test-suite:define-simple-simd-test f32.4-if (f32.4) (u32.4 f32.4 f32.4) f32-if)
+(sb-simd-test-suite:define-simple-simd-test f32.4-and (f32.4) (&rest f32.4) f32-and)
+(sb-simd-test-suite:define-simple-simd-test f32.4-or (f32.4) (&rest f32.4) f32-or)
+(sb-simd-test-suite:define-simple-simd-test f32.4-xor (f32.4) (&rest f32.4) f32-xor)
+(sb-simd-test-suite:define-simple-simd-test f32.4-andc1 (f32.4) (f32.4 f32.4) f32-andc1)
+(sb-simd-test-suite:define-simple-simd-test f32.4-not (f32.4) (f32.4) f32-not)
+(sb-simd-test-suite:define-simple-simd-test f32.4-max (f32.4) (f32.4 &rest f32.4) f32-max)
+(sb-simd-test-suite:define-simple-simd-test f32.4-min (f32.4) (f32.4 &rest f32.4) f32-min)
+(sb-simd-test-suite:define-simple-simd-test f32.4+ (f32.4) (&rest f32.4) f32+)
+(sb-simd-test-suite:define-simple-simd-test f32.4- (f32.4) (f32.4 &rest f32.4) f32-)
+(sb-simd-test-suite:define-simple-simd-test f32.4* (f32.4) (&rest f32.4) f32*)
+(sb-simd-test-suite:define-simple-simd-test f32.4/ (f32.4) (f32.4 &rest f32.4) f32/)
+(sb-simd-test-suite:define-simple-simd-test f32.4= (u32.4) (f32.4 &rest f32.4) f32=)
+(sb-simd-test-suite:define-simple-simd-test f32.4/= (u32.4) (f32.4 &rest f32.4) f32/=)
+(sb-simd-test-suite:define-simple-simd-test f32.4< (u32.4) (f32.4 &rest f32.4) f32<)
+(sb-simd-test-suite:define-simple-simd-test f32.4<= (u32.4) (f32.4 &rest f32.4) f32<=)
+(sb-simd-test-suite:define-simple-simd-test f32.4> (u32.4) (f32.4 &rest f32.4) f32>)
+(sb-simd-test-suite:define-simple-simd-test f32.4>= (u32.4) (f32.4 &rest f32.4) f32>=)
+
+(sb-simd-test-suite:define-simple-simd-test f32.8-if (f32.8) (u32.8 f32.8 f32.8) f32-if)
+(sb-simd-test-suite:define-simple-simd-test f32.8-and (f32.8) (&rest f32.8) f32-and)
+(sb-simd-test-suite:define-simple-simd-test f32.8-or (f32.8) (&rest f32.8) f32-or)
+(sb-simd-test-suite:define-simple-simd-test f32.8-xor (f32.8) (&rest f32.8) f32-xor)
+(sb-simd-test-suite:define-simple-simd-test f32.8-andc1 (f32.8) (f32.8 f32.8) f32-andc1)
+(sb-simd-test-suite:define-simple-simd-test f32.8-not (f32.8) (f32.8) f32-not)
+(sb-simd-test-suite:define-simple-simd-test f32.8-max (f32.8) (f32.8 &rest f32.8) f32-max)
+(sb-simd-test-suite:define-simple-simd-test f32.8-min (f32.8) (f32.8 &rest f32.8) f32-min)
+(sb-simd-test-suite:define-simple-simd-test f32.8+ (f32.8) (&rest f32.8) f32+)
+(sb-simd-test-suite:define-simple-simd-test f32.8- (f32.8) (f32.8 &rest f32.8) f32-)
+(sb-simd-test-suite:define-simple-simd-test f32.8* (f32.8) (&rest f32.8) f32*)
+(sb-simd-test-suite:define-simple-simd-test f32.8/ (f32.8) (f32.8 &rest f32.8) f32/)
+(sb-simd-test-suite:define-simple-simd-test f32.8= (u32.8) (f32.8 &rest f32.8) f32=)
+(sb-simd-test-suite:define-simple-simd-test f32.8/= (u32.8) (f32.8 &rest f32.8) f32/=)
+(sb-simd-test-suite:define-simple-simd-test f32.8< (u32.8) (f32.8 &rest f32.8) f32<)
+(sb-simd-test-suite:define-simple-simd-test f32.8<= (u32.8) (f32.8 &rest f32.8) f32<=)
+(sb-simd-test-suite:define-simple-simd-test f32.8> (u32.8) (f32.8 &rest f32.8) f32>)
+(sb-simd-test-suite:define-simple-simd-test f32.8>= (u32.8) (f32.8 &rest f32.8) f32>=)
+
+(sb-simd-test-suite:define-simple-simd-test f64.2-if (f64.2) (u64.2 f64.2 f64.2) f64-if)
+(sb-simd-test-suite:define-simple-simd-test f64.2-and (f64.2) (&rest f64.2) f64-and)
+(sb-simd-test-suite:define-simple-simd-test f64.2-or (f64.2) (&rest f64.2) f64-or)
+(sb-simd-test-suite:define-simple-simd-test f64.2-xor (f64.2) (&rest f64.2) f64-xor)
+(sb-simd-test-suite:define-simple-simd-test f64.2-andc1 (f64.2) (f64.2 f64.2) f64-andc1)
+(sb-simd-test-suite:define-simple-simd-test f64.2-not (f64.2) (f64.2) f64-not)
+(sb-simd-test-suite:define-simple-simd-test f64.2-max (f64.2) (f64.2 &rest f64.2) f64-max)
+(sb-simd-test-suite:define-simple-simd-test f64.2-min (f64.2) (f64.2 &rest f64.2) f64-min)
+(sb-simd-test-suite:define-simple-simd-test f64.2+ (f64.2) (&rest f64.2) f64+)
+(sb-simd-test-suite:define-simple-simd-test f64.2- (f64.2) (f64.2 &rest f64.2) f64-)
+(sb-simd-test-suite:define-simple-simd-test f64.2* (f64.2) (&rest f64.2) f64*)
+(sb-simd-test-suite:define-simple-simd-test f64.2/ (f64.2) (f64.2 &rest f64.2) f64/)
+(sb-simd-test-suite:define-simple-simd-test f64.2= (u64.2) (f64.2 &rest f64.2) f64=)
+(sb-simd-test-suite:define-simple-simd-test f64.2/= (u64.2) (f64.2 &rest f64.2) f64/=)
+(sb-simd-test-suite:define-simple-simd-test f64.2< (u64.2) (f64.2 &rest f64.2) f64<)
+(sb-simd-test-suite:define-simple-simd-test f64.2<= (u64.2) (f64.2 &rest f64.2) f64<=)
+(sb-simd-test-suite:define-simple-simd-test f64.2> (u64.2) (f64.2 &rest f64.2) f64>)
+(sb-simd-test-suite:define-simple-simd-test f64.2>= (u64.2) (f64.2 &rest f64.2) f64>=)
+
+(sb-simd-test-suite:define-simple-simd-test f64.4-if (f64.4) (u64.4 f64.4 f64.4) f64-if)
+(sb-simd-test-suite:define-simple-simd-test f64.4-and (f64.4) (&rest f64.4) f64-and)
+(sb-simd-test-suite:define-simple-simd-test f64.4-or (f64.4) (&rest f64.4) f64-or)
+(sb-simd-test-suite:define-simple-simd-test f64.4-xor (f64.4) (&rest f64.4) f64-xor)
+(sb-simd-test-suite:define-simple-simd-test f64.4-andc1 (f64.4) (f64.4 f64.4) f64-andc1)
+(sb-simd-test-suite:define-simple-simd-test f64.4-not (f64.4) (f64.4) f64-not)
+(sb-simd-test-suite:define-simple-simd-test f64.4-max (f64.4) (f64.4 &rest f64.4) f64-max)
+(sb-simd-test-suite:define-simple-simd-test f64.4-min (f64.4) (f64.4 &rest f64.4) f64-min)
+(sb-simd-test-suite:define-simple-simd-test f64.4+ (f64.4) (&rest f64.4) f64+)
+(sb-simd-test-suite:define-simple-simd-test f64.4- (f64.4) (f64.4 &rest f64.4) f64-)
+(sb-simd-test-suite:define-simple-simd-test f64.4* (f64.4) (&rest f64.4) f64*)
+(sb-simd-test-suite:define-simple-simd-test f64.4/ (f64.4) (f64.4 &rest f64.4) f64/)
+(sb-simd-test-suite:define-simple-simd-test f64.4= (u64.4) (f64.4 &rest f64.4) f64=)
+(sb-simd-test-suite:define-simple-simd-test f64.4/= (u64.4) (f64.4 &rest f64.4) f64/=)
+(sb-simd-test-suite:define-simple-simd-test f64.4< (u64.4) (f64.4 &rest f64.4) f64<)
+(sb-simd-test-suite:define-simple-simd-test f64.4<= (u64.4) (f64.4 &rest f64.4) f64<=)
+(sb-simd-test-suite:define-simple-simd-test f64.4> (u64.4) (f64.4 &rest f64.4) f64>)
+(sb-simd-test-suite:define-simple-simd-test f64.4>= (u64.4) (f64.4 &rest f64.4) f64>=)
+
+(sb-simd-test-suite:define-simple-simd-test u8.16-if (u8.16) (u8.16 u8.16 u8.16) u8-if)
+(sb-simd-test-suite:define-simple-simd-test u8.16-and (u8.16) (&rest u8.16) u8-and)
+(sb-simd-test-suite:define-simple-simd-test u8.16-or (u8.16) (&rest u8.16) u8-or)
+(sb-simd-test-suite:define-simple-simd-test u8.16-xor (u8.16) (&rest u8.16) u8-xor)
+(sb-simd-test-suite:define-simple-simd-test u8.16-andc1 (u8.16) (u8.16 u8.16) u8-andc1)
+(sb-simd-test-suite:define-simple-simd-test u8.16-not (u8.16) (u8.16) u8-not)
+(sb-simd-test-suite:define-simple-simd-test u8.16+ (u8.16) (&rest u8.16) u8+)
+(sb-simd-test-suite:define-simple-simd-test u8.16- (u8.16) (u8.16 &rest u8.16) u8-)
+(sb-simd-test-suite:define-simple-simd-test u8.16= (u8.16) (u8.16 &rest u8.16) u8=)
+(sb-simd-test-suite:define-simple-simd-test u8.16/= (u8.16) (u8.16 &rest u8.16) u8/=)
+(sb-simd-test-suite:define-simple-simd-test u8.16< (u8.16) (u8.16 &rest u8.16) u8<)
+(sb-simd-test-suite:define-simple-simd-test u8.16<= (u8.16) (u8.16 &rest u8.16) u8<=)
+(sb-simd-test-suite:define-simple-simd-test u8.16> (u8.16) (u8.16 &rest u8.16) u8>)
+(sb-simd-test-suite:define-simple-simd-test u8.16>= (u8.16) (u8.16 &rest u8.16) u8>=)
+
+(sb-simd-test-suite:define-simple-simd-test u16.8-if (u16.8) (u16.8 u16.8 u16.8) u16-if)
+(sb-simd-test-suite:define-simple-simd-test u16.8-and (u16.8) (&rest u16.8) u16-and)
+(sb-simd-test-suite:define-simple-simd-test u16.8-or (u16.8) (&rest u16.8) u16-or)
+(sb-simd-test-suite:define-simple-simd-test u16.8-xor (u16.8) (&rest u16.8) u16-xor)
+(sb-simd-test-suite:define-simple-simd-test u16.8-andc1 (u16.8) (u16.8 u16.8) u16-andc1)
+(sb-simd-test-suite:define-simple-simd-test u16.8-not (u16.8) (u16.8) u16-not)
+(sb-simd-test-suite:define-simple-simd-test u16.8+ (u16.8) (&rest u16.8) u16+)
+(sb-simd-test-suite:define-simple-simd-test u16.8- (u16.8) (u16.8 &rest u16.8) u16-)
+(sb-simd-test-suite:define-simple-simd-test u16.8= (u16.8) (u16.8 &rest u16.8) u16=)
+(sb-simd-test-suite:define-simple-simd-test u16.8/= (u16.8) (u16.8 &rest u16.8) u16/=)
+(sb-simd-test-suite:define-simple-simd-test u16.8< (u16.8) (u16.8 &rest u16.8) u16<)
+(sb-simd-test-suite:define-simple-simd-test u16.8<= (u16.8) (u16.8 &rest u16.8) u16<=)
+(sb-simd-test-suite:define-simple-simd-test u16.8> (u16.8) (u16.8 &rest u16.8) u16>)
+(sb-simd-test-suite:define-simple-simd-test u16.8>= (u16.8) (u16.8 &rest u16.8) u16>=)
+
+(sb-simd-test-suite:define-simple-simd-test u32.4-if (u32.4) (u32.4 u32.4 u32.4) u32-if)
+(sb-simd-test-suite:define-simple-simd-test u32.4-and (u32.4) (&rest u32.4) u32-and)
+(sb-simd-test-suite:define-simple-simd-test u32.4-or (u32.4) (&rest u32.4) u32-or)
+(sb-simd-test-suite:define-simple-simd-test u32.4-xor (u32.4) (&rest u32.4) u32-xor)
+(sb-simd-test-suite:define-simple-simd-test u32.4-andc1 (u32.4) (u32.4 u32.4) u32-andc1)
+(sb-simd-test-suite:define-simple-simd-test u32.4-not (u32.4) (u32.4) u32-not)
+(sb-simd-test-suite:define-simple-simd-test u32.4+ (u32.4) (&rest u32.4) u32+)
+(sb-simd-test-suite:define-simple-simd-test u32.4- (u32.4) (u32.4 &rest u32.4) u32-)
+(sb-simd-test-suite:define-simple-simd-test u32.4= (u32.4) (u32.4 &rest u32.4) u32=)
+(sb-simd-test-suite:define-simple-simd-test u32.4/= (u32.4) (u32.4 &rest u32.4) u32/=)
+(sb-simd-test-suite:define-simple-simd-test u32.4< (u32.4) (u32.4 &rest u32.4) u32<)
+(sb-simd-test-suite:define-simple-simd-test u32.4<= (u32.4) (u32.4 &rest u32.4) u32<=)
+(sb-simd-test-suite:define-simple-simd-test u32.4> (u32.4) (u32.4 &rest u32.4) u32>)
+(sb-simd-test-suite:define-simple-simd-test u32.4>= (u32.4) (u32.4 &rest u32.4) u32>=)
+
+(sb-simd-test-suite:define-simple-simd-test u64.2-if (u64.2) (u64.2 u64.2 u64.2) u64-if)
+(sb-simd-test-suite:define-simple-simd-test u64.2-and (u64.2) (&rest u64.2) u64-and)
+(sb-simd-test-suite:define-simple-simd-test u64.2-or (u64.2) (&rest u64.2) u64-or)
+(sb-simd-test-suite:define-simple-simd-test u64.2-xor (u64.2) (&rest u64.2) u64-xor)
+(sb-simd-test-suite:define-simple-simd-test u64.2-andc1 (u64.2) (u64.2 u64.2) u64-andc1)
+(sb-simd-test-suite:define-simple-simd-test u64.2-not (u64.2) (u64.2) u64-not)
+(sb-simd-test-suite:define-simple-simd-test u64.2+ (u64.2) (&rest u64.2) u64+)
+(sb-simd-test-suite:define-simple-simd-test u64.2- (u64.2) (u64.2 &rest u64.2) u64-)
+(sb-simd-test-suite:define-simple-simd-test u64.2= (u64.2) (u64.2 &rest u64.2) u64=)
+(sb-simd-test-suite:define-simple-simd-test u64.2/= (u64.2) (u64.2 &rest u64.2) u64/=)
+(sb-simd-test-suite:define-simple-simd-test u64.2< (u64.2) (u64.2 &rest u64.2) u64<)
+(sb-simd-test-suite:define-simple-simd-test u64.2<= (u64.2) (u64.2 &rest u64.2) u64<=)
+(sb-simd-test-suite:define-simple-simd-test u64.2> (u64.2) (u64.2 &rest u64.2) u64>)
+(sb-simd-test-suite:define-simple-simd-test u64.2>= (u64.2) (u64.2 &rest u64.2) u64>=)
+
+(sb-simd-test-suite:define-simple-simd-test s8.16-if (s8.16) (u8.16 s8.16 s8.16) s8-if)
+(sb-simd-test-suite:define-simple-simd-test s8.16-and (s8.16) (&rest s8.16) s8-and)
+(sb-simd-test-suite:define-simple-simd-test s8.16-or (s8.16) (&rest s8.16) s8-or)
+(sb-simd-test-suite:define-simple-simd-test s8.16-xor (s8.16) (&rest s8.16) s8-xor)
+(sb-simd-test-suite:define-simple-simd-test s8.16-andc1 (s8.16) (s8.16 s8.16) s8-andc1)
+(sb-simd-test-suite:define-simple-simd-test s8.16-not (s8.16) (s8.16) s8-not)
+(sb-simd-test-suite:define-simple-simd-test s8.16+ (s8.16) (&rest s8.16) s8+)
+(sb-simd-test-suite:define-simple-simd-test s8.16- (s8.16) (s8.16 &rest s8.16) s8-)
+(sb-simd-test-suite:define-simple-simd-test s8.16= (u8.16) (s8.16 &rest s8.16) s8=)
+(sb-simd-test-suite:define-simple-simd-test s8.16/= (u8.16) (s8.16 &rest s8.16) s8/=)
+(sb-simd-test-suite:define-simple-simd-test s8.16< (u8.16) (s8.16 &rest s8.16) s8<)
+(sb-simd-test-suite:define-simple-simd-test s8.16<= (u8.16) (s8.16 &rest s8.16) s8<=)
+(sb-simd-test-suite:define-simple-simd-test s8.16> (u8.16) (s8.16 &rest s8.16) s8>)
+(sb-simd-test-suite:define-simple-simd-test s8.16>= (u8.16) (s8.16 &rest s8.16) s8>=)
+
+(sb-simd-test-suite:define-simple-simd-test s16.8-if (s16.8) (u16.8 s16.8 s16.8) s16-if)
+(sb-simd-test-suite:define-simple-simd-test s16.8-and (s16.8) (&rest s16.8) s16-and)
+(sb-simd-test-suite:define-simple-simd-test s16.8-or (s16.8) (&rest s16.8) s16-or)
+(sb-simd-test-suite:define-simple-simd-test s16.8-xor (s16.8) (&rest s16.8) s16-xor)
+(sb-simd-test-suite:define-simple-simd-test s16.8-andc1 (s16.8) (s16.8 s16.8) s16-andc1)
+(sb-simd-test-suite:define-simple-simd-test s16.8-not (s16.8) (s16.8) s16-not)
+(sb-simd-test-suite:define-simple-simd-test s16.8+ (s16.8) (&rest s16.8) s16+)
+(sb-simd-test-suite:define-simple-simd-test s16.8- (s16.8) (s16.8 &rest s16.8) s16-)
+(sb-simd-test-suite:define-simple-simd-test s16.8= (u16.8) (s16.8 &rest s16.8) s16=)
+(sb-simd-test-suite:define-simple-simd-test s16.8/= (u16.8) (s16.8 &rest s16.8) s16/=)
+(sb-simd-test-suite:define-simple-simd-test s16.8< (u16.8) (s16.8 &rest s16.8) s16<)
+(sb-simd-test-suite:define-simple-simd-test s16.8<= (u16.8) (s16.8 &rest s16.8) s16<=)
+(sb-simd-test-suite:define-simple-simd-test s16.8> (u16.8) (s16.8 &rest s16.8) s16>)
+(sb-simd-test-suite:define-simple-simd-test s16.8>= (u16.8) (s16.8 &rest s16.8) s16>=)
+
+(sb-simd-test-suite:define-simple-simd-test s32.4-if (s32.4) (u32.4 s32.4 s32.4) s32-if)
+(sb-simd-test-suite:define-simple-simd-test s32.4-and (s32.4) (&rest s32.4) s32-and)
+(sb-simd-test-suite:define-simple-simd-test s32.4-or (s32.4) (&rest s32.4) s32-or)
+(sb-simd-test-suite:define-simple-simd-test s32.4-xor (s32.4) (&rest s32.4) s32-xor)
+(sb-simd-test-suite:define-simple-simd-test s32.4-andc1 (s32.4) (s32.4 s32.4) s32-andc1)
+(sb-simd-test-suite:define-simple-simd-test s32.4-not (s32.4) (s32.4) s32-not)
+(sb-simd-test-suite:define-simple-simd-test s32.4+ (s32.4) (&rest s32.4) s32+)
+(sb-simd-test-suite:define-simple-simd-test s32.4- (s32.4) (s32.4 &rest s32.4) s32-)
+(sb-simd-test-suite:define-simple-simd-test s32.4= (u32.4) (s32.4 &rest s32.4) s32=)
+(sb-simd-test-suite:define-simple-simd-test s32.4/= (u32.4) (s32.4 &rest s32.4) s32/=)
+(sb-simd-test-suite:define-simple-simd-test s32.4< (u32.4) (s32.4 &rest s32.4) s32<)
+(sb-simd-test-suite:define-simple-simd-test s32.4<= (u32.4) (s32.4 &rest s32.4) s32<=)
+(sb-simd-test-suite:define-simple-simd-test s32.4> (u32.4) (s32.4 &rest s32.4) s32>)
+(sb-simd-test-suite:define-simple-simd-test s32.4>= (u32.4) (s32.4 &rest s32.4) s32>=)
+
+(sb-simd-test-suite:define-simple-simd-test s64.2-if (s64.2) (u64.2 s64.2 s64.2) s64-if)
+(sb-simd-test-suite:define-simple-simd-test s64.2-and (s64.2) (&rest s64.2) s64-and)
+(sb-simd-test-suite:define-simple-simd-test s64.2-or (s64.2) (&rest s64.2) s64-or)
+(sb-simd-test-suite:define-simple-simd-test s64.2-xor (s64.2) (&rest s64.2) s64-xor)
+(sb-simd-test-suite:define-simple-simd-test s64.2-andc1 (s64.2) (s64.2 s64.2) s64-andc1)
+(sb-simd-test-suite:define-simple-simd-test s64.2-not (s64.2) (s64.2) s64-not)
+(sb-simd-test-suite:define-simple-simd-test s64.2+ (s64.2) (&rest s64.2) s64+)
+(sb-simd-test-suite:define-simple-simd-test s64.2- (s64.2) (s64.2 &rest s64.2) s64-)
+(sb-simd-test-suite:define-simple-simd-test s64.2= (u64.2) (s64.2 &rest s64.2) s64=)
+(sb-simd-test-suite:define-simple-simd-test s64.2< (u64.2) (s64.2 &rest s64.2) s64<)
+(sb-simd-test-suite:define-simple-simd-test s64.2<= (u64.2) (s64.2 &rest s64.2) s64<=)
+(sb-simd-test-suite:define-simple-simd-test s64.2> (u64.2) (s64.2 &rest s64.2) s64>)
+(sb-simd-test-suite:define-simple-simd-test s64.2>= (u64.2) (s64.2 &rest s64.2) s64>=)
+
+(in-package #:sb-simd-avx2)
+
+(sb-simd-test-suite:define-simple-simd-test u8.32-if (u8.32) (u8.32 u8.32 u8.32) u8-if)
+(sb-simd-test-suite:define-simple-simd-test u8.32-and (u8.32) (&rest u8.32) u8-and)
+(sb-simd-test-suite:define-simple-simd-test u8.32-or (u8.32) (&rest u8.32) u8-or)
+(sb-simd-test-suite:define-simple-simd-test u8.32-xor (u8.32) (&rest u8.32) u8-xor)
+(sb-simd-test-suite:define-simple-simd-test u8.32-andc1 (u8.32) (u8.32 u8.32) u8-andc1)
+(sb-simd-test-suite:define-simple-simd-test u8.32-not (u8.32) (u8.32) u8-not)
+(sb-simd-test-suite:define-simple-simd-test u8.32+ (u8.32) (&rest u8.32) u8+)
+(sb-simd-test-suite:define-simple-simd-test u8.32- (u8.32) (u8.32 &rest u8.32) u8-)
+(sb-simd-test-suite:define-simple-simd-test u8.32= (u8.32) (u8.32 &rest u8.32) u8=)
+(sb-simd-test-suite:define-simple-simd-test u8.32/= (u8.32) (u8.32 &rest u8.32) u8/=)
+(sb-simd-test-suite:define-simple-simd-test u8.32< (u8.32) (u8.32 &rest u8.32) u8<)
+(sb-simd-test-suite:define-simple-simd-test u8.32<= (u8.32) (u8.32 &rest u8.32) u8<=)
+(sb-simd-test-suite:define-simple-simd-test u8.32> (u8.32) (u8.32 &rest u8.32) u8>)
+(sb-simd-test-suite:define-simple-simd-test u8.32>= (u8.32) (u8.32 &rest u8.32) u8>=)
+
+(sb-simd-test-suite:define-simple-simd-test u16.16-if (u16.16) (u16.16 u16.16 u16.16) u16-if)
+(sb-simd-test-suite:define-simple-simd-test u16.16-and (u16.16) (&rest u16.16) u16-and)
+(sb-simd-test-suite:define-simple-simd-test u16.16-or (u16.16) (&rest u16.16) u16-or)
+(sb-simd-test-suite:define-simple-simd-test u16.16-xor (u16.16) (&rest u16.16) u16-xor)
+(sb-simd-test-suite:define-simple-simd-test u16.16-andc1 (u16.16) (u16.16 u16.16) u16-andc1)
+(sb-simd-test-suite:define-simple-simd-test u16.16-not (u16.16) (u16.16) u16-not)
+(sb-simd-test-suite:define-simple-simd-test u16.16+ (u16.16) (&rest u16.16) u16+)
+(sb-simd-test-suite:define-simple-simd-test u16.16- (u16.16) (u16.16 &rest u16.16) u16-)
+(sb-simd-test-suite:define-simple-simd-test u16.16= (u16.16) (u16.16 &rest u16.16) u16=)
+(sb-simd-test-suite:define-simple-simd-test u16.16/= (u16.16) (u16.16 &rest u16.16) u16/=)
+(sb-simd-test-suite:define-simple-simd-test u16.16< (u16.16) (u16.16 &rest u16.16) u16<)
+(sb-simd-test-suite:define-simple-simd-test u16.16<= (u16.16) (u16.16 &rest u16.16) u16<=)
+(sb-simd-test-suite:define-simple-simd-test u16.16> (u16.16) (u16.16 &rest u16.16) u16>)
+(sb-simd-test-suite:define-simple-simd-test u16.16>= (u16.16) (u16.16 &rest u16.16) u16>=)
+
+(sb-simd-test-suite:define-simple-simd-test u32.8-if (u32.8) (u32.8 u32.8 u32.8) u32-if)
+(sb-simd-test-suite:define-simple-simd-test u32.8-and (u32.8) (&rest u32.8) u32-and)
+(sb-simd-test-suite:define-simple-simd-test u32.8-or (u32.8) (&rest u32.8) u32-or)
+(sb-simd-test-suite:define-simple-simd-test u32.8-xor (u32.8) (&rest u32.8) u32-xor)
+(sb-simd-test-suite:define-simple-simd-test u32.8-andc1 (u32.8) (u32.8 u32.8) u32-andc1)
+(sb-simd-test-suite:define-simple-simd-test u32.8-not (u32.8) (u32.8) u32-not)
+(sb-simd-test-suite:define-simple-simd-test u32.8+ (u32.8) (&rest u32.8) u32+)
+(sb-simd-test-suite:define-simple-simd-test u32.8- (u32.8) (u32.8 &rest u32.8) u32-)
+(sb-simd-test-suite:define-simple-simd-test u32.8= (u32.8) (u32.8 &rest u32.8) u32=)
+(sb-simd-test-suite:define-simple-simd-test u32.8/= (u32.8) (u32.8 &rest u32.8) u32/=)
+(sb-simd-test-suite:define-simple-simd-test u32.8< (u32.8) (u32.8 &rest u32.8) u32<)
+(sb-simd-test-suite:define-simple-simd-test u32.8<= (u32.8) (u32.8 &rest u32.8) u32<=)
+(sb-simd-test-suite:define-simple-simd-test u32.8> (u32.8) (u32.8 &rest u32.8) u32>)
+(sb-simd-test-suite:define-simple-simd-test u32.8>= (u32.8) (u32.8 &rest u32.8) u32>=)
+
+(sb-simd-test-suite:define-simple-simd-test u64.4-if (u64.4) (u64.4 u64.4 u64.4) u64-if)
+(sb-simd-test-suite:define-simple-simd-test u64.4-and (u64.4) (&rest u64.4) u64-and)
+(sb-simd-test-suite:define-simple-simd-test u64.4-or (u64.4) (&rest u64.4) u64-or)
+(sb-simd-test-suite:define-simple-simd-test u64.4-xor (u64.4) (&rest u64.4) u64-xor)
+(sb-simd-test-suite:define-simple-simd-test u64.4-andc1 (u64.4) (u64.4 u64.4) u64-andc1)
+(sb-simd-test-suite:define-simple-simd-test u64.4-not (u64.4) (u64.4) u64-not)
+(sb-simd-test-suite:define-simple-simd-test u64.4+ (u64.4) (&rest u64.4) u64+)
+(sb-simd-test-suite:define-simple-simd-test u64.4- (u64.4) (u64.4 &rest u64.4) u64-)
+(sb-simd-test-suite:define-simple-simd-test u64.4= (u64.4) (u64.4 &rest u64.4) u64=)
+(sb-simd-test-suite:define-simple-simd-test u64.4/= (u64.4) (u64.4 &rest u64.4) u64/=)
+(sb-simd-test-suite:define-simple-simd-test u64.4< (u64.4) (u64.4 &rest u64.4) u64<)
+(sb-simd-test-suite:define-simple-simd-test u64.4<= (u64.4) (u64.4 &rest u64.4) u64<=)
+(sb-simd-test-suite:define-simple-simd-test u64.4> (u64.4) (u64.4 &rest u64.4) u64>)
+(sb-simd-test-suite:define-simple-simd-test u64.4>= (u64.4) (u64.4 &rest u64.4) u64>=)
+
+(sb-simd-test-suite:define-simple-simd-test s8.32-if (s8.32) (u8.32 s8.32 s8.32) s8-if)
+(sb-simd-test-suite:define-simple-simd-test s8.32-and (s8.32) (&rest s8.32) s8-and)
+(sb-simd-test-suite:define-simple-simd-test s8.32-or (s8.32) (&rest s8.32) s8-or)
+(sb-simd-test-suite:define-simple-simd-test s8.32-xor (s8.32) (&rest s8.32) s8-xor)
+(sb-simd-test-suite:define-simple-simd-test s8.32-andc1 (s8.32) (s8.32 s8.32) s8-andc1)
+(sb-simd-test-suite:define-simple-simd-test s8.32-not (s8.32) (s8.32) s8-not)
+(sb-simd-test-suite:define-simple-simd-test s8.32+ (s8.32) (&rest s8.32) s8+)
+(sb-simd-test-suite:define-simple-simd-test s8.32- (s8.32) (s8.32 &rest s8.32) s8-)
+(sb-simd-test-suite:define-simple-simd-test s8.32= (u8.32) (s8.32 &rest s8.32) s8=)
+(sb-simd-test-suite:define-simple-simd-test s8.32/= (u8.32) (s8.32 &rest s8.32) s8/=)
+(sb-simd-test-suite:define-simple-simd-test s8.32< (u8.32) (s8.32 &rest s8.32) s8<)
+(sb-simd-test-suite:define-simple-simd-test s8.32<= (u8.32) (s8.32 &rest s8.32) s8<=)
+(sb-simd-test-suite:define-simple-simd-test s8.32> (u8.32) (s8.32 &rest s8.32) s8>)
+(sb-simd-test-suite:define-simple-simd-test s8.32>= (u8.32) (s8.32 &rest s8.32) s8>=)
+
+(sb-simd-test-suite:define-simple-simd-test s16.16-if (s16.16) (u16.16 s16.16 s16.16) s16-if)
+(sb-simd-test-suite:define-simple-simd-test s16.16-and (s16.16) (&rest s16.16) s16-and)
+(sb-simd-test-suite:define-simple-simd-test s16.16-or (s16.16) (&rest s16.16) s16-or)
+(sb-simd-test-suite:define-simple-simd-test s16.16-xor (s16.16) (&rest s16.16) s16-xor)
+(sb-simd-test-suite:define-simple-simd-test s16.16-andc1 (s16.16) (s16.16 s16.16) s16-andc1)
+(sb-simd-test-suite:define-simple-simd-test s16.16-not (s16.16) (s16.16) s16-not)
+(sb-simd-test-suite:define-simple-simd-test s16.16+ (s16.16) (&rest s16.16) s16+)
+(sb-simd-test-suite:define-simple-simd-test s16.16- (s16.16) (s16.16 &rest s16.16) s16-)
+(sb-simd-test-suite:define-simple-simd-test s16.16= (u16.16) (s16.16 &rest s16.16) s16=)
+(sb-simd-test-suite:define-simple-simd-test s16.16/= (u16.16) (s16.16 &rest s16.16) s16/=)
+(sb-simd-test-suite:define-simple-simd-test s16.16< (u16.16) (s16.16 &rest s16.16) s16<)
+(sb-simd-test-suite:define-simple-simd-test s16.16<= (u16.16) (s16.16 &rest s16.16) s16<=)
+(sb-simd-test-suite:define-simple-simd-test s16.16> (u16.16) (s16.16 &rest s16.16) s16>)
+(sb-simd-test-suite:define-simple-simd-test s16.16>= (u16.16) (s16.16 &rest s16.16) s16>=)
+
+(sb-simd-test-suite:define-simple-simd-test s32.8-if (s32.8) (u32.8 s32.8 s32.8) s32-if)
+(sb-simd-test-suite:define-simple-simd-test s32.8-and (s32.8) (&rest s32.8) s32-and)
+(sb-simd-test-suite:define-simple-simd-test s32.8-or (s32.8) (&rest s32.8) s32-or)
+(sb-simd-test-suite:define-simple-simd-test s32.8-xor (s32.8) (&rest s32.8) s32-xor)
+(sb-simd-test-suite:define-simple-simd-test s32.8-andc1 (s32.8) (s32.8 s32.8) s32-andc1)
+(sb-simd-test-suite:define-simple-simd-test s32.8-not (s32.8) (s32.8) s32-not)
+(sb-simd-test-suite:define-simple-simd-test s32.8+ (s32.8) (&rest s32.8) s32+)
+(sb-simd-test-suite:define-simple-simd-test s32.8- (s32.8) (s32.8 &rest s32.8) s32-)
+(sb-simd-test-suite:define-simple-simd-test s32.8= (u32.8) (s32.8 &rest s32.8) s32=)
+(sb-simd-test-suite:define-simple-simd-test s32.8/= (u32.8) (s32.8 &rest s32.8) s32/=)
+(sb-simd-test-suite:define-simple-simd-test s32.8< (u32.8) (s32.8 &rest s32.8) s32<)
+(sb-simd-test-suite:define-simple-simd-test s32.8<= (u32.8) (s32.8 &rest s32.8) s32<=)
+(sb-simd-test-suite:define-simple-simd-test s32.8> (u32.8) (s32.8 &rest s32.8) s32>)
+(sb-simd-test-suite:define-simple-simd-test s32.8>= (u32.8) (s32.8 &rest s32.8) s32>=)
+
+(sb-simd-test-suite:define-simple-simd-test s64.4-if (s64.4) (u64.4 s64.4 s64.4) s64-if)
+(sb-simd-test-suite:define-simple-simd-test s64.4-and (s64.4) (&rest s64.4) s64-and)
+(sb-simd-test-suite:define-simple-simd-test s64.4-or (s64.4) (&rest s64.4) s64-or)
+(sb-simd-test-suite:define-simple-simd-test s64.4-xor (s64.4) (&rest s64.4) s64-xor)
+(sb-simd-test-suite:define-simple-simd-test s64.4-andc1 (s64.4) (s64.4 s64.4) s64-andc1)
+(sb-simd-test-suite:define-simple-simd-test s64.4-not (s64.4) (s64.4) s64-not)
+(sb-simd-test-suite:define-simple-simd-test s64.4+ (s64.4) (&rest s64.4) s64+)
+(sb-simd-test-suite:define-simple-simd-test s64.4- (s64.4) (s64.4 &rest s64.4) s64-)
+(sb-simd-test-suite:define-simple-simd-test s64.4= (u64.4) (s64.4 &rest s64.4) s64=)
+(sb-simd-test-suite:define-simple-simd-test s64.4< (u64.4) (s64.4 &rest s64.4) s64<)
+(sb-simd-test-suite:define-simple-simd-test s64.4<= (u64.4) (s64.4 &rest s64.4) s64<=)
+(sb-simd-test-suite:define-simple-simd-test s64.4> (u64.4) (s64.4 &rest s64.4) s64>)
+(sb-simd-test-suite:define-simple-simd-test s64.4>= (u64.4) (s64.4 &rest s64.4) s64>=)
+
+(in-package #:sb-simd-fma)
+
+(sb-simd-test-suite:define-simple-simd-test f32.4-fmadd  (f32.4) (f32.4 f32.4 f32.4) f32-fmadd)
+(sb-simd-test-suite:define-simple-simd-test f32.4-fnmadd (f32.4) (f32.4 f32.4 f32.4) f32-fnmadd)
+(sb-simd-test-suite:define-simple-simd-test f32.4-fmsub  (f32.4) (f32.4 f32.4 f32.4) f32-fmsub)
+
+(sb-simd-test-suite:define-simple-simd-test f32.8-fmadd  (f32.8) (f32.8 f32.8 f32.8) f32-fmadd)
+(sb-simd-test-suite:define-simple-simd-test f32.8-fnmadd (f32.8) (f32.8 f32.8 f32.8) f32-fnmadd)
+(sb-simd-test-suite:define-simple-simd-test f32.8-fmsub (f32.8) (f32.8 f32.8 f32.8) f32-fmsub)
+
+(sb-simd-test-suite:define-simple-simd-test f64.2-fmadd  (f64.2) (f64.2 f64.2 f64.2) f64-fmadd)
+(sb-simd-test-suite:define-simple-simd-test f64.2-fnmadd (f64.2) (f64.2 f64.2 f64.2) f64-fnmadd)
+(sb-simd-test-suite:define-simple-simd-test f64.2-fmsub (f64.2) (f64.2 f64.2 f64.2) f64-fmsub)
+
+(sb-simd-test-suite:define-simple-simd-test f64.4-fmadd  (f64.4) (f64.4 f64.4 f64.4) f64-fmadd)
+(sb-simd-test-suite:define-simple-simd-test f64.4-fnmadd (f64.4) (f64.4 f64.4 f64.4) f64-fnmadd)
+(sb-simd-test-suite:define-simple-simd-test f64.4-fmsub (f64.4) (f64.4 f64.4 f64.4) f64-fmsub)

contrib/sb-simd/test-suite/test-suite.lisp

@@ -0,0 +1,157 @@
+(in-package #:sb-simd-test-suite)
+
+;; A list of the names of all tests defined with the TEST macro.
+(defparameter *tests* '())
+
+;; The value of *RANDOM-STATE* when the test suite encountered the most
+;; recent error, or NIL, if there have been no recent errors.
+(defvar *failed-random-state* nil)
+
+;; The number of tests that have been run so far.
+(defvar *test-count*)
+
+;; The number of successful checks that have been performed so far.
+(defvar *pass-count*)
+
+;; The number of checks that have been performed in the current test.
+(defvar *check-count* 0)
+
+(defun call-with-random-state (thunk)
+  (let ((*random-state*
+          (if (not *failed-random-state*)
+              (load-time-value (make-random-state t))
+              *failed-random-state*)))
+    (setf *failed-random-state* (make-random-state *random-state*))
+    (multiple-value-prog1 (funcall thunk)
+      (setf *failed-random-state* nil))))
+
+(defmacro with-random-state (&body body)
+  `(call-with-random-state (lambda () ,@body)))
+
+(defun call-with-test-harness (thunk)
+  (if (and (boundp '*test-count*)
+           (boundp '*pass-count*))
+      (funcall thunk)
+      (let* ((*test-count* 0)
+             (*pass-count* 0))
+        (with-random-state (funcall thunk))
+        (report *test-count* *pass-count*))))
+
+(defmacro with-test-harness (&body body)
+  `(call-with-test-harness (lambda () ,@body)))
+
+(defun report (test-count pass-count)
+  (format t "~&Success: ~s test~:p, ~s check~:p.~%" test-count pass-count)
+  (finish-output))
+
+;;; Ensure that DEFINE-TEST never overwrites function names in other
+;;; packages.
+(defun intern-test-name (test-name)
+  (let ((test-suite-package #.*package*)
+        (name (symbol-name test-name))
+        (package (symbol-package test-name)))
+    (cond ((eq package test-suite-package)
+           test-name)
+          ((eq (nth-value 1 (find-symbol name package)) :external)
+           (intern (format nil "~A:~A" (package-name package) name) test-suite-package))
+          (t
+           (intern (format nil "~A:~A" (package-name package) name) test-suite-package)))))
+
+(defmacro define-test (test-name &body body)
+  "Define a test function and add it to *TESTS*."
+  (let ((name (intern-test-name test-name)))
+    `(prog1 ',name
+       (defun ,name ()
+         (declare (optimize (debug 3) (safety 3)))
+         (with-test-harness
+           (enter-test ',name)
+           ,@body))
+       (pushnew ',name *tests*))))
+
+(defun enter-test (test-name)
+  (incf *test-count*)
+  (setf *check-count* 0)
+  (format t "~&~A" (string test-name))
+  (finish-output))
+
+(defun pass ()
+  (incf *pass-count*)
+  (when (zerop (logand *check-count* (incf *check-count*)))
+    (write-char #\.))
+  (values))
+
+(defmacro is (test-form)
+  "Assert that TEST-FORM evaluates to true."
+  `(progn
+     (assert ,test-form)
+     (pass)))
+
+(defun expect-condition (expected thunk)
+  (block nil
+    (flet ((handler (condition)
+             (cond ((typep condition expected)
+                    (pass)
+                    (return))
+                   (t (error "Expected to signal ~s, but got ~s:~%~a"
+                             expected (type-of condition) condition)))))
+      (handler-bind ((condition #'handler))
+        (funcall thunk)))
+    (error "Expected to signal ~s, but got nothing." expected)))
+
+(defmacro signals (condition &body body)
+  "Assert that `body' signals a condition of type `condition'."
+  `(expect-condition ',condition (lambda () ,@body)))
+
+(defun all-tests ()
+  *tests*)
+
+;;; This function catches particularly hideous bugs - exported symbols with
+;;; no attached definition.  In case some exported symbols are
+;;; intentionally without definition, they can be excluded via the SKIP
+;;; argument.
+(defun check-package (package &key skip)
+  (let ((skip (loop for name in skip collect (intern (string name) package))))
+    (do-external-symbols (symbol package)
+      (unless (member symbol skip)
+        (unless (symbol-defined-p symbol)
+          (error "~@<The package ~A exports the symbol ~A, ~
+                     which does not have an attached definition.~:@>"
+                 (package-name package)
+                 (symbol-name symbol))))))
+  (pass))
+
+(defun symbol-defined-p (symbol)
+  (or (find-class symbol nil)
+      (boundp symbol)
+      (fboundp symbol)
+      (fboundp `(setf ,symbol))
+      (macro-function symbol)
+      (special-operator-p symbol)
+      (type-specifier-p symbol)))
+
+(defun type-specifier-p (object)
+  (ignore-errors (typep 42 object) t))
+
+(defun run-tests (&rest tests)
+  (with-test-harness (mapc #'funcall (shuffle tests)))
+  (values))
+
+(defun run-test-suite ()
+  (format t "== Testing SB-SIMD ==~%")
+  (format t "The library exports ~D functions."
+          (length (all-exported-functions)))
+  (apply #'run-tests (all-tests)))
+
+(defun all-instruction-sets ()
+  (remove-duplicates
+   (loop for iset being the hash-values of sb-simd-internals:*instruction-sets*
+         collect iset)))
+
+(defun all-exported-functions ()
+  (remove-duplicates
+   (loop for iset in (all-instruction-sets)
+         for package = (instruction-set-package iset)
+         append
+         (loop for symbol being the external-symbols of package
+               when (fboundp symbol)
+               collect symbol))))

contrib/sb-simd/test-suite/utilities.lisp

@@ -0,0 +1,174 @@
+(in-package #:sb-simd-test-suite)
+
+(defun shuffle (list)
+  (let ((result (copy-seq list)))
+    (loop for tail on result
+          for tail-length from (length result) downto 2
+          do (rotatef (first tail)
+                      (nth (random tail-length) tail)))
+    result))
+
+(defun simd-info (name)
+  "Returns, as list:
+
+ 1. The element type of the SIMD pack.
+
+ 2. The number of elements of the SIMD pack.
+
+ 3. The name of the function for creating the SIMD pack from individual
+    elements.
+
+ 4. The name of the function for returning the elements of the SIMD pack as
+    multiple values."
+  (with-accessors ((scalar-record simd-record-scalar-record)
+                   (width value-record-simd-width))
+      (find-value-record name)
+    (list
+     (value-record-name scalar-record)
+     width
+     (or (find-symbol (format nil "MAKE-~A" (symbol-name name))
+                      (symbol-package name))
+         (error "No constructor found for ~S." name))
+     (or (find-symbol (format nil "~A-VALUES" (symbol-name name))
+                      (symbol-package name))
+         (error "No unpacker found for ~S." name)))))
+
+(defun simd= (a b)
+  (typecase a
+    (sb-ext:simd-pack
+     (when (sb-ext:simd-pack-p b)
+       (multiple-value-bind (a0 a1) (sb-ext:%simd-pack-ub64s a)
+         (multiple-value-bind (b0 b1) (sb-ext:%simd-pack-ub64s b)
+           (and (= a0 b0) (= a1 b1))))))
+    (sb-ext:simd-pack-256
+     (when (sb-ext:simd-pack-256-p b)
+       (multiple-value-bind (a0 a1 a2 a3) (sb-ext:%simd-pack-256-ub64s a)
+         (multiple-value-bind (b0 b1 b2 b3) (sb-ext:%simd-pack-256-ub64s b)
+           (and (= a0 b0) (= a1 b1) (= a2 b2) (= a3 b3))))))
+    (otherwise nil)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;
+;;; Argument Type Specifications
+
+(defun parse-argtypes (argtypes)
+  "Returns, as multiple values:
+
+ 1. The list of mandatory argtypes.
+
+ 2. The list of optional argtypes.
+
+ 3. The argtype of the rest argument, or NIL if there is no &rest argtype."
+  (labels ((fail ()
+             (error "Malformed argtypes list: ~S" argtypes))
+           (process-mandatory (argtypes mandatory)
+             (if (null argtypes)
+                 (values (reverse mandatory) '() nil)
+                 (case (first argtypes)
+                   ((&optional)
+                    (process-optional (rest argtypes) mandatory '()))
+                   ((&rest)
+                    (process-rest (rest argtypes) mandatory '()))
+                   (#.(set-difference lambda-list-keywords '(&optional &rest))
+                    (fail))
+                   (otherwise
+                    (process-mandatory (rest argtypes) (cons (first argtypes) mandatory))))))
+           (process-optional (argtypes mandatory optional)
+             (if (null argtypes)
+                 (values (reverse mandatory) (reverse optional) nil)
+                 (case (first argtypes)
+                   ((&rest)
+                    (process-rest (rest argtypes) mandatory optional))
+                   (#.(set-difference lambda-list-keywords '(&rest))
+                    (fail))
+                   (otherwise
+                    (process-optional (rest argtypes) mandatory (cons (first argtypes) optional))))))
+           (process-rest (argtypes mandatory optional)
+             (if (or (endp argtypes)
+                     (not (endp (rest argtypes))))
+                 (fail)
+                 (values (reverse mandatory) (reverse optional) (first argtypes)))))
+    (process-mandatory argtypes '())))
+
+(defun argtypes-variants (argtypes)
+  "Returns a list of lists of type specifiers such that each list of type
+specifiers satisfies the argument type specification given by ARGTYPES."
+  (multiple-value-bind (mandatory optional rest)
+      (parse-argtypes argtypes)
+    (let ((result '()))
+      (loop for n-optional to (length optional) do
+        (loop for n-rest from 0 to (if (not rest) 0 3) do
+          (push (append mandatory
+                        (subseq optional 0 n-optional)
+                        (make-list n-rest :initial-element rest))
+                result)))
+      (reverse result))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;
+;;; Generators
+
+(defun find-generator (type)
+  (intern (format nil "RANDOM-~A" (symbol-name type))
+          #.*package*))
+
+(macrolet ((define-generators ()
+             `(progn
+                ,@(loop for (type name)
+                          in '((sb-simd:f32 random-f32)
+                               (sb-simd:f64 random-f64)
+                               (sb-simd:u8 random-u8)
+                               (sb-simd:u16 random-u16)
+                               (sb-simd:u32 random-u32)
+                               (sb-simd:u64 random-u64)
+                               (sb-simd:s8 random-s8)
+                               (sb-simd:s16 random-s16)
+                               (sb-simd:s32 random-s32)
+                               (sb-simd:s64 random-s64))
+                        collect
+                        (let ((numbers (numbers-of-type type)))
+                          `(defun ,name ()
+                             (aref ,(coerce numbers `(simple-array ,type (*)))
+                                   (random ,(length numbers)))))))))
+  (define-generators))
+
+(defun find-valid-simd-call (scalar-function input-generators simd-width
+                             input-constructors output-constructors)
+  (let ((inputs-list '())
+        (outputs-list '()))
+    (loop repeat simd-width do
+      (multiple-value-bind (inputs outputs)
+          (find-valid-scalar-call scalar-function input-generators)
+        (push inputs inputs-list)
+        (push outputs outputs-list)))
+    (values
+     (apply #'mapcar #'funcall input-constructors inputs-list)
+     (apply #'mapcar #'funcall output-constructors outputs-list))))
+
+(defun find-valid-scalar-call (scalar-function input-generators)
+  (let ((attempts 0))
+    (loop
+      (let ((inputs (mapcar #'funcall input-generators)))
+        (handler-case (return (values inputs (multiple-value-list (apply scalar-function inputs))))
+          (condition ()
+            (incf attempts)
+            (when (> attempts 1000)
+              (error "Failed to find a valid call to ~S." scalar-function))))))))
+
+(defun bitwise= (a b)
+  (etypecase a
+    (rational
+     (when (rationalp b)
+       (= a b)))
+    (single-float
+     (when (typep b 'single-float)
+       (= (sb-kernel:single-float-bits a)
+          (sb-kernel:single-float-bits b))))
+    (double-float
+     (when (typep b 'double-float)
+       (= (sb-kernel:double-float-bits a)
+          (sb-kernel:double-float-bits b))))
+    (complex
+     (when (typep b 'complex)
+       (bitwise= (realpart a) (realpart b))
+       (bitwise= (imagpart a) (imagpart b))))))

doc/manual/contrib-modules.texinfo

@@ -17,6 +17,7 @@ contributed modules.
 * sb-posix::
 * sb-queue::
 * sb-rotate-byte::
+* sb-simd::
 @end menu
 
 @page
@@ -43,4 +44,5 @@ contributed modules.
 @page
 @include sb-rotate-byte/sb-rotate-byte.texinfo
 
-
+@page
+@include sb-simd/sb-simd.texinfo

make-config.sh

@@ -271,7 +271,6 @@ find_gnumake
 
 echo "GNUMAKE=\"$GNUMAKE\"; export GNUMAKE" >> output/build-config
 echo "SBCL_XC_HOST=\"$SBCL_XC_HOST\"; export SBCL_XC_HOST" >> output/build-config
-echo "CONTRIB_BLOCKLIST=\"$CONTRIB_BLOCKLIST\"; export CONTRIB_BLOCKLIST" >> output/build-config
 if [ -n "$SBCL_HOST_LOCATION" ]; then
     echo "SBCL_HOST_LOCATION=\"$SBCL_HOST_LOCATION\"; export SBCL_HOST_LOCATION" >> output/build-config
 fi
@@ -492,6 +491,13 @@ echo ';;;; Please do not edit it by hand.' >> $ltf
 echo ';;;; See make-config.sh.' >> $ltf
 echo "(lambda (features) (set-difference (union features (list :${sbcl_arch}$WITH_FEATURES " >> $ltf
 
+# Automatically block sb-simd on non-x86 platforms, at least for now.
+case "$sbcl_arch" in
+    x86-64) ;; *) CONTRIB_BLOCKLIST="$CONTRIB_BLOCKLIST sb-simd" ;;
+esac
+
+echo "CONTRIB_BLOCKLIST=\"$CONTRIB_BLOCKLIST\"; export CONTRIB_BLOCKLIST" >> output/build-config
+
 echo //setting up OS-dependent information
 
 original_dir=`pwd`