EVEX: add V' and improve 5-bit register decoding

* Add +evex+ indicator to distinguish EVEX from VEX/legacy
  even for EVEX L'L=00 forms
* Decode ModRM.reg registers 16–31 using R'
* Stop incorrectly applying R' to ModRM.r/m operands
* Decode EVEX ModRM.r/m register-direct 16–31 using X as B'
* Decode EVEX vvvv registers 16–31 using V'

src/compiler/x86-64/avx2-insts.lisp

@@ -47,6 +47,12 @@
   :prefilter #'prefilter-xmmreg/mem
   :printer #'print-sized-xmmreg/mem-default-qword)
 
+;;; General indicator that we are decoding an EVEX instruction.
+;;; EVEX has 128-, 256-, and 512-bit forms; L'L=00 is 128-bit and sets
+;;; neither +vex-l+ nor +evex-l1+, so this separate bit is required for
+;;; code that needs to know "EVEX vs VEX/legacy".
+;;; (Bit 14 is currently unused by other dstate properties.)
+(defconstant +evex+ #b100000000000000)
 (defconstant +vex-l+ #b10000000000)
 ;; EVEX L'L=10 (512-bit) sets bit 11; L'L=01 (256-bit) sets bit 10 (=+vex-l+)
 (defconstant +evex-l1+ #b100000000000)
@@ -54,6 +60,10 @@
 (defconstant +evex-r-prime+ #b1000000000000)
 ;; EVEX X bit as B' (r/m bit 4, for registers 16-31 in ModR/M.r/m, reg-direct only)
 (defconstant +evex-b-prime+ #b10000000000000)
+;;; EVEX V' bit (vvvv bit 4, for registers 16-31 in the vvvv field).
+;;; The 4-bit vvvv field in the EVEX prefix is extended with this bit.
+;;; (Bit 15 is currently unused by other dstate properties.)
+(defconstant +evex-v-prime+ #b1000000000000000)
 
 (define-arg-type vex-l
   :prefilter  (lambda (dstate value)
@@ -95,13 +105,21 @@
   :prefilter (lambda (dstate value)
                (dstate-setprop dstate (if (zerop value) +evex-r-prime+ 0))))
 
+;;; Marks that the instruction being decoded uses an EVEX prefix.
+;;; This is set by the EVEX fixed-bit field and is independent of L'L.
+(define-arg-type evex-fixed
+  :prefilter (lambda (dstate value)
+               (declare (ignore value))
+               (dstate-setprop dstate +evex+)))
+
 ;; EVEX V' extends vvvv bit 4 (inverted in prefix)
 ;; V'=0 means bit4=1 (register 16-31 in vvvv)
 ;; Note: the printer for vvvv (print-ymmreg via ymm-vvvv-reg) gets
 ;; a 4-bit value from the invert-4 prefilter. V' provides the 5th bit.
 (define-arg-type evex-v-prime
   :prefilter (lambda (dstate value)
-               (declare (ignore dstate value))))
+               (dstate-setprop dstate
+                               (if (zerop value) +evex-v-prime+ 0))))
 
 ;; EVEX L'L: 2-bit vector length (00=128, 01=256, 10=512)
 ;; Stores into dstate bits 10-11: L'L=01 sets bit 10 (+vex-l+),
@@ -115,6 +133,12 @@
   :prefilter (lambda (dstate value)
                (dstate-setprop dstate (if (plusp value) +rex-w+ 0))))
 
+;;; EVEX vvvv register operand.
+;;; The 4-bit value is inverted and extended with V' by the printer.
+(define-arg-type evex-ymm-vvvv-reg
+  :prefilter #'invert-4
+  :printer #'print-ymmreg-vvvv)
+
 ;; Opmask register k0-k7
 (define-arg-type opmask-reg
   :printer #'print-opmask-register)
@@ -293,8 +317,8 @@
   (mm       :field (byte 2 8))
   ;; Byte 2
   (w          :field (byte 1 23) :type 'evex-w)
-  (vvvv       :field (byte 4 19) :type 'ymm-vvvv-reg)
-  (evex-fixed :field (byte 1 18) :value 1) ; must be 1 for EVEX
+  (vvvv       :field (byte 4 19) :type 'evex-ymm-vvvv-reg)
+  (evex-fixed :field (byte 1 18) :value 1 :type 'evex-fixed) ; must be 1 for EVEX
   (pp         :field (byte 2 16))
   ;; Byte 3
   (z-bit   :field (byte 1 31))

src/compiler/x86-64/avx512-insts.lisp

@@ -1171,15 +1171,25 @@
 ;;; mask-reg-number is 1-7 (k1-k7; k0 means no masking).
 ;;; Merge-masking: destination elements not selected by mask are preserved.
 (macrolet ((def (name prefix opcode w &optional (opcode-prefix #x0f))
-             `(define-instruction ,name (segment dst src1 src2 mask)
-                ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode
-                                            :opcode-prefix opcode-prefix :w w :nds t)
-                (:emitter
-                 (emit-avx512-inst segment src2 dst ,prefix ,opcode
-                                   :opcode-prefix ,opcode-prefix
-                                   :vvvv src1
-                                   :w ,w
-                                   :aaa mask)))))
+             (let ((mask-printer
+                    '(:name :tab reg ", " vvvv ", " reg/mem
+                      " {" aaa "}")))
+               `(define-instruction ,name (segment dst src1 src2 mask)
+                  ,@(loop for k from 1 to 7
+                          append
+                          (avx512-inst-printer-list
+                           'ymm-ymm/mem prefix opcode
+                           :opcode-prefix opcode-prefix
+                           :w w
+                           :nds t
+                           :more-fields `((aaa ,k))
+                           :printer mask-printer))
+                  (:emitter
+                   (emit-avx512-inst segment src2 dst ,prefix ,opcode
+                                     :opcode-prefix ,opcode-prefix
+                                     :vvvv src1
+                                     :w ,w
+                                     :aaa mask))))))
   ;; Integer arithmetic (qword)
   (def vpaddq-masked  #x66 #xd4 1)
   (def vpsubq-masked  #x66 #xfb 1)

src/compiler/x86-64/target-avx2-insts.lisp

@@ -11,6 +11,38 @@
 
 (in-package "SB-X86-64-ASM")
 
+;;; Printer for EVEX/VEX ModRM.r/m register operands.
+;;; Does NOT use EVEX R' — R' belongs only to the ModRM.reg field.
+(defun print-ymmreg-rm (value stream dstate)
+  (let* ((offset (etypecase value
+                   ((unsigned-byte 4) value)
+                   (reg (reg-num value))))
+         (reg (get-fpr (cond ((dstate-getprop dstate +evex-l1+) :zmm)
+                             ((dstate-getprop dstate +vex-l+) :ymm)
+                             (t :xmm))
+                       offset))
+         (name (reg-name reg)))
+    (if stream
+        (write-string name stream)
+        (operand name dstate))))
+
+;;; Uses EVEX V' to form a 5-bit register number.
+(defun print-ymmreg-vvvv (value stream dstate)
+  (let* ((offset (etypecase value
+                   ((unsigned-byte 4) value)
+                   (reg (reg-num value))))
+         (offset (if (dstate-getprop dstate +evex-v-prime+)
+                     (+ offset 16)
+                     offset))
+         (reg (get-fpr (cond ((dstate-getprop dstate +evex-l1+) :zmm)
+                             ((dstate-getprop dstate +vex-l+) :ymm)
+                             (t :xmm))
+                       offset))
+         (name (reg-name reg)))
+    (if stream
+        (write-string name stream)
+        (operand name dstate))))
+
 (defun print-ymmreg (value stream dstate)
   (let* ((offset (etypecase value
                    ((unsigned-byte 4) value)
@@ -47,7 +79,7 @@
 (defun print-ymmreg/mem (value stream dstate)
   (if (machine-ea-p value)
       (print-mem-ref :ref value nil stream dstate)
-      (print-ymmreg value stream dstate)))
+      (print-ymmreg-rm value stream dstate)))
 
 (defun invert-4 (dstate value)
   (declare (ignore dstate))

src/compiler/x86-64/target-insts.lisp

@@ -365,7 +365,11 @@
       (cond ((= mod #b11) ; register direct mode
              (case regclass
               (gpr (get-gpr :qword full-reg)) ; size is not really known here
-              (fpr (get-fpr :xmm full-reg))))
+              (fpr (get-fpr :xmm
+              (if (and (dstate-getprop dstate +evex+)
+                       (dstate-getprop dstate +rex-x+))
+                  (+ full-reg 16)
+                  full-reg)))))
             ((= r/m #b100) ; SIB byte - rex.b is "don't care"
              (let* ((sib (the (unsigned-byte 8) (read-suffix 8 dstate)))
                     (index-reg (extend +rex-x+ (ldb (byte 3 3) sib)))

tests/simd-pack-512.pure.lisp

@@ -284,3 +284,41 @@
                    1d0 0f0 0f0 0f0 0f0 0f0 0f0 0f0
                    0f0 0f0 0f0 0f0 0f0 0f0 0f0 0f0)
                   type-error)))
+
+;; evex patch
+(cl:in-package "SB-VM")
+
+(sb-c::defknown %test-evex-high-regs ()
+    (unsigned-byte 64)
+  (sb-c::flushable sb-c::movable))
+
+(defun %test-evex-high-regs ()
+  (error "%test-evex-high-regs stub"))
+
+(define-vop (%test-evex-high-regs)
+  (:translate %test-evex-high-regs)
+  (:policy :fast-safe)
+  (:temporary (:sc single-avx512-reg :offset 16) z16)
+  (:temporary (:sc single-avx512-reg :offset 17) z17)
+  (:temporary (:sc single-avx512-reg :offset 18) z18)
+  (:results (res :scs (unsigned-reg)))
+  (:result-types unsigned-num)
+  (:generator 1
+    (inst vaddps z16 z17 z18)
+    (inst xor :dword res res)))
+
+(cl:in-package :test-util)
+
+(with-test (:name :evex-high-register-disassembly)
+  (let* ((fun (compile nil
+                       '(lambda ()
+                          (sb-vm::%test-evex-high-regs))))
+         (text (with-output-to-string (s)
+                 (disassemble fun :stream s))))
+    ;; These names can only appear if the disassembler correctly
+    ;; handles EVEX R', V', and X-as-B'.
+    (assert (search "ZMM16" text))
+    (assert (search "ZMM17" text))
+    (assert (search "ZMM18" text))
+    ;; Ideally we see a decoded instruction, not raw EVEX bytes.
+    (assert (search "VADDPS" text))))