common-lisp/sbcl.sbcl
1
0
Fork 0
mirror of git://git.code.sf.net/p/sbcl/sbcl synced 2026-09-10 07:26:40 -04:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

x86-64: implement AVX-512_FP16 instruction set and CPUID detection

Browse source 
- avx2-insts.lisp: expand EVEX prefix encoder to support 3-bit mmm
  opcode map selection, enabling Map 5 (#b101) and Map 6 (#b110).
- x86-64-arch.c: add runtime CPUID detection for AVX-512_FP16
  (leaf 7, subleaf 0, EDX bit 23) guarded by OS ZMM state enablement.
- avx512-insts.lisp: implement complete AVX-512_FP16 instruction suite:
  * vector and scalar arithmetic (vaddph/sh, vsubph/sh, vmulph/sh,
    vdivph/sh, vminph/sh, vmaxph/sh, vsqrtph/sh, vrcpph/sh, vrsqrtph/sh,
    vscalefph/sh)
  * fused multiply-add (vfmadd*ph/sh, vfmsub*ph/sh, vfnmadd*ph/sh,
    vfnmsub*ph/sh, vfmaddsub*ph, vfmsubadd*ph, vfcmaddcph, vfmaddcph)
  * comparisons and classification (vcmpph/sh, vcomish, vucomish,
    vfpclassph/sh, vrndscaleph/sh)
  * conversions (vcvtph2psx, vcvtps2phx, vcvtdq2ph, vcvtph2dq,
    vcvttph2dq, vcvtuw2ph, vcvtw2ph, vcvtph2w, vcvtph2uw, vcvttph2w,
    vcvttph2uw, vcvtsd2sh, vcvtss2sh, vcvtsi2sh, vcvtsh2si, vcvttsh2si)
  * scalar and GPR moves (vmovsh, vmovw)
  * masked instruction variants with {k} and {z} zeroing
- tests/assembler.pure.lisp: verify bit-exact binary emission and
  round-trip disassembly across all FP16 instruction categories.
This commit is contained in:
Aaron Estrada 2026-09-06 11:05:31 -06:00 committed by Stas Boukarev
parent ce548a2163
commit 2428c1efd0
4 changed files with 694 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/compiler/x86-64/avx2-insts.lisp
View file

@ -326,8 +326,8 @@
(x :field (byte 1 14) :type 'vex-x) (x :field (byte 1 14) :type 'vex-x)
(b :field (byte 1 13) :type 'vex-b) (b :field (byte 1 13) :type 'vex-b)
(r-prime :field (byte 1 12) :type 'evex-r-prime) (r-prime :field (byte 1 12) :type 'evex-r-prime)
(reserved :field (byte 2 10) :value #b00) ; distinguishes EVEX from BOUND (reserved :field (byte 1 11) :value 0) ; bit 11 reserved (0 in EVEX)
(mm :field (byte 2 8)) (mm :field (byte 3 8)) ; bits 10:8 are mmm (maps 1, 2, 3, 5, 6)
;; Byte 2 ;; Byte 2
(w :field (byte 1 23) :type 'evex-w) (w :field (byte 1 23) :type 'evex-w)
(vvvv :field (byte 4 19) :type 'evex-ymm-vvvv-reg) (vvvv :field (byte 4 19) :type 'evex-ymm-vvvv-reg)
@ -435,9 +435,11 @@
(defun evex-encode-mm (m-mmmm) (defun evex-encode-mm (m-mmmm)
(ecase m-mmmm (ecase m-mmmm
(#x0F #b01) (#x0F #b001)
(#x0F38 #b10) (#x0F38 #b010)
(#x0F3A #b11)))) (#x0F3A #b011)
((:map5 5 #x5) #b101)
((:map6 6 #x6) #b110))))
(defun emit-two-byte-vex (segment r vvvv l pp) (defun emit-two-byte-vex (segment r vvvv l pp)
(emit-bytes segment (emit-bytes segment
@ -518,7 +520,7 @@
;;; EVEX prefix encoding for AVX-512 ;;; EVEX prefix encoding for AVX-512
;;; EVEX is a 4-byte prefix: 62h | P1 | P2 | P3 ;;; EVEX is a 4-byte prefix: 62h | P1 | P2 | P3
;;; P1: R(7) X(6) B(5) R'(4) 00(3:2) mm(1:0) ;;; P1: R(7) X(6) B(5) R'(4) 0(3) mmm(2:0)
;;; P2: W(7) vvvv(6:3) 1(2) pp(1:0) ;;; P2: W(7) vvvv(6:3) 1(2) pp(1:0)
;;; P3: z(7) L'(6) L(5) b(4) V'(3) aaa(2:0) ;;; P3: z(7) L'(6) L(5) b(4) V'(3) aaa(2:0)
;;; R, X, B, R', V' are inverted. vvvv is inverted. ;;; R, X, B, R', V' are inverted. vvvv is inverted.
@ -526,12 +528,12 @@
(defun emit-evex (segment r x b r-prime opcode-prefix w vvvv pp z ll evex-b v-prime aaa) (defun emit-evex (segment r x b r-prime opcode-prefix w vvvv pp z ll evex-b v-prime aaa)
(emit-bytes segment (emit-bytes segment
#x62 #x62
;; P1: R X B R' 00 mm ;; P1: R X B R' 0 mmm
(logior (ash (logxor 1 r) 7) (logior (ash (logxor 1 r) 7)
(ash (logxor 1 x) 6) (ash (logxor 1 x) 6)
(ash (logxor 1 b) 5) (ash (logxor 1 b) 5)
(ash (logxor 1 r-prime) 4) (ash (logxor 1 r-prime) 4)
;; bits 3:2 are reserved (0) ;; bit 3 is reserved (0), bits 2:0 are mmm (maps 1, 2, 3, 5, 6)
(evex-encode-mm opcode-prefix)) (evex-encode-mm opcode-prefix))
;; P2: W vvvv 1 pp ;; P2: W vvvv 1 pp
(logior (ash w 7) (logior (ash w 7)

573
src/compiler/x86-64/avx512-insts.lisp
View file

@ -5,7 +5,7 @@
;;;; Implemented subsets: ;;;; Implemented subsets:
;;;; AVX-512F, AVX-512BW, AVX-512DQ, AVX-512CD, AVX-512IFMA, ;;;; AVX-512F, AVX-512BW, AVX-512DQ, AVX-512CD, AVX-512IFMA,
;;;; AVX-512VBMI, AVX-512VBMI2, AVX-512VPOPCNTDQ, AVX-512BITALG, ;;;; AVX-512VBMI, AVX-512VBMI2, AVX-512VPOPCNTDQ, AVX-512BITALG,
;;;; AVX-512VNNI, AVX-512BF16, ;;;; AVX-512VNNI, AVX-512BF16, AVX-512FP16,
;;;; GFNI (in avx2-insts.lisp), ;;;; GFNI (in avx2-insts.lisp),
;;;; VPCLMULQDQ-256/512 (via VEX auto-promotion to EVEX), ;;;; VPCLMULQDQ-256/512 (via VEX auto-promotion to EVEX),
;;;; EVEX Compare-to-Opmask (vcmpps, vcmppd, vcmpss, vcmpsd), ;;;; EVEX Compare-to-Opmask (vcmpps, vcmppd, vcmpss, vcmpsd),
@ -18,7 +18,6 @@
;;;; AVX-512VL - Explicit EVEX 128/256-bit forms with masking/broadcast ;;;; AVX-512VL - Explicit EVEX 128/256-bit forms with masking/broadcast
;;;; (auto-promotion handles basic ZMM; full VL needs ;;;; (auto-promotion handles basic ZMM; full VL needs
;;;; explicit EVEX for XMM/YMM with masking) ;;;; explicit EVEX for XMM/YMM with masking)
;;;; AVX-512FP16 - Half-precision FP16 arithmetic (~100 instructions)
;;;; VAES-256/512 - Wide forms of vaesenc/vaesdec ;;;; VAES-256/512 - Wide forms of vaesenc/vaesdec
;;;; VP2INTERSECT - vp2intersectd/q ;;;; VP2INTERSECT - vp2intersectd/q
;;;; AVX-512ER/PF - vexp2ps/pd, prefetch (Knights Landing Xeon Phi, deprecated) ;;;; AVX-512ER/PF - vexp2ps/pd, prefetch (Knights Landing Xeon Phi, deprecated)
@ -1546,3 +1545,573 @@
:z z-num :z z-num
:disp-n disp-n)))))) :disp-n disp-n))))))
(def)) (def))
;;;; ---- AVX-512_FP16 instructions ----
;;; 3-operand vector arithmetic (Map 5 & Map 6)
(macrolet ((def (name opcode &optional (opcode-prefix :map5) (prefix nil) (w 0))
(let ((masked-name (symbolicate name "-MASKED")))
`(progn
(define-instruction ,name (segment dst src1 src2 &optional mask (zeroing 0))
,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w :nds t)
,@(loop for k from 1 to 7
append
(avx512-inst-printer-list
'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w :nds t
:more-fields `((aaa ,k) (z-bit 0))
:printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "}"))
append
(avx512-inst-printer-list
'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w :nds t
:more-fields `((aaa ,k) (z-bit 1))
:printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "} {z}")))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t (error "Invalid mask ~S" mask))))
(z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)))
(emit-avx512-inst segment src2 dst ,prefix ,opcode
:opcode-prefix ,opcode-prefix
:vvvv src1
:w ,w
:aaa mask-num
:z z-num
:disp-n (full-vector-disp-n dst)))))
(define-instruction ,masked-name (segment dst src1 src2 mask &optional (zeroing 0))
(:emitter
(let ((mask-num (cond ((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t (error "Invalid mask ~S" mask))))
(z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)))
(emit-avx512-inst segment src2 dst ,prefix ,opcode
:opcode-prefix ,opcode-prefix
:vvvv src1
:w ,w
:aaa mask-num
:z z-num
:disp-n (full-vector-disp-n dst)))))))))
;; Map 5 arithmetic
(def vaddph #x58)
(def vsubph #x5c)
(def vmulph #x59)
(def vdivph #x5e)
(def vminph #x5d)
(def vmaxph #x5f)
;; Map 6 scalef
(def vscalefph #x2c :map6 #x66)
;; Map 6 FMA (vector)
(def vfmadd132ph #x98 :map6 #x66)
(def vfmadd213ph #xa8 :map6 #x66)
(def vfmadd231ph #xb8 :map6 #x66)
(def vfmsub132ph #x9a :map6 #x66)
(def vfmsub213ph #xaa :map6 #x66)
(def vfmsub231ph #xba :map6 #x66)
(def vfnmadd132ph #x9c :map6 #x66)
(def vfnmadd213ph #xac :map6 #x66)
(def vfnmadd231ph #xbc :map6 #x66)
(def vfnmsub132ph #x9e :map6 #x66)
(def vfnmsub213ph #xae :map6 #x66)
(def vfnmsub231ph #xbe :map6 #x66)
(def vfmaddsub132ph #x96 :map6 #x66)
(def vfmaddsub213ph #xa6 :map6 #x66)
(def vfmaddsub231ph #xb6 :map6 #x66)
(def vfmsubadd132ph #x97 :map6 #x66)
(def vfmsubadd213ph #xa7 :map6 #x66)
(def vfmsubadd231ph #xb7 :map6 #x66)
;; Map 6 Complex FMA
(def vfcmaddcph #x56 :map6 #xf2)
(def vfmaddcph #x56 :map6 #xf3))
;;; 2-operand vector arithmetic (sqrt, rcp, rsqrt)
(macrolet ((def (name opcode &optional (opcode-prefix :map5) (prefix nil) (w 0))
(let ((masked-name (symbolicate name "-MASKED")))
`(progn
(define-instruction ,name (segment dst src &optional mask (zeroing 0))
,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w)
,@(loop for k from 1 to 7
append
(avx512-inst-printer-list
'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w
:more-fields `((aaa ,k) (z-bit 0))
:printer '(:name :tab reg ", " reg/mem " {" aaa "}"))
append
(avx512-inst-printer-list
'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w
:more-fields `((aaa ,k) (z-bit 1))
:printer '(:name :tab reg ", " reg/mem " {" aaa "} {z}")))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t (error "Invalid mask ~S" mask))))
(z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)))
(emit-avx512-inst segment src dst ,prefix ,opcode
:opcode-prefix ,opcode-prefix
:w ,w
:aaa mask-num
:z z-num
:disp-n (full-vector-disp-n dst)))))
(define-instruction ,masked-name (segment dst src mask &optional (zeroing 0))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t (error "Invalid mask ~S" mask))))
(z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)))
(emit-avx512-inst segment src dst ,prefix ,opcode
:opcode-prefix ,opcode-prefix
:w ,w
:aaa mask-num
:z z-num
:disp-n (full-vector-disp-n dst)))))))))
(def vsqrtph #x51 :map5 nil)
(def vrcpph #x4c :map6 #x66)
(def vrsqrtph #x4e :map6 #x66))
;;; Scalar arithmetic (Map 5 & Map 6, 3-operand, ll=0, disp-n=2)
(macrolet ((def (name opcode &optional (opcode-prefix :map5) (prefix #xf3) (w 0))
(let ((masked-name (symbolicate name "-MASKED")))
`(progn
(define-instruction ,name (segment dst src1 src2 &optional mask (zeroing 0))
,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w :ll 0 :nds t)
,@(loop for k from 1 to 7
append
(avx512-inst-printer-list
'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w :ll 0 :nds t
:more-fields `((aaa ,k) (z-bit 0))
:printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "}"))
append
(avx512-inst-printer-list
'ymm-ymm/mem prefix opcode
:opcode-prefix opcode-prefix :w w :ll 0 :nds t
:more-fields `((aaa ,k) (z-bit 1))
:printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "} {z}")))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t (error "Invalid mask ~S" mask))))
(z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)))
(emit-avx512-inst segment src2 dst ,prefix ,opcode
:opcode-prefix ,opcode-prefix
:vvvv src1
:w ,w
:ll 0
:aaa mask-num
:z z-num
:disp-n 2))))
(define-instruction ,masked-name (segment dst src1 src2 mask &optional (zeroing 0))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t (error "Invalid mask ~S" mask))))
(z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)))
(emit-avx512-inst segment src2 dst ,prefix ,opcode
:opcode-prefix ,opcode-prefix
:vvvv src1
:w ,w
:ll 0
:aaa mask-num
:z z-num
:disp-n 2))))))))
(def vaddsh #x58)
(def vsubsh #x5c)
(def vmulsh #x59)
(def vdivsh #x5e)
(def vminsh #x5d)
(def vmaxsh #x5f)
(def vsqrtsh #x51)
(def vrcpsh #x4d :map6 #x66)
(def vrsqrtsh #x4f :map6 #x66)
(def vscalefsh #x2d :map6 #x66)
;; Scalar FMA
(def vfmadd132sh #x99 :map6 #x66)
(def vfmadd213sh #xa9 :map6 #x66)
(def vfmadd231sh #xb9 :map6 #x66)
(def vfmsub132sh #x9b :map6 #x66)
(def vfmsub213sh #xab :map6 #x66)
(def vfmsub231sh #xbb :map6 #x66)
(def vfnmadd132sh #x9d :map6 #x66)
(def vfnmadd213sh #xad :map6 #x66)
(def vfnmadd231sh #xbd :map6 #x66)
(def vfnmsub132sh #x9f :map6 #x66)
(def vfnmsub213sh #xaf :map6 #x66)
(def vfnmsub231sh #xbf :map6 #x66))
;;; Comparisons & Classification
(macrolet ((def-cmp (name prefix name-suffix &key scalar)
`(define-instruction ,name (segment condition dst src src2 &optional mask)
,@(avx512-inst-printer-list 'ymm-ymm/mem-imm prefix #xc2
:opcode-prefix #x0f3a
:w 0
:ll (if scalar 0 nil)
:more-fields `((reg nil :type 'opmask-reg)
(imm nil :type 'avx-condition-code))
:printer `("VCMP" imm ,name-suffix
:tab reg ", " vvvv ", " reg/mem))
,@(loop for k from 1 to 7
append
(avx512-inst-printer-list
'ymm-ymm/mem-imm prefix #xc2
:opcode-prefix #x0f3a
:w 0
:ll (if scalar 0 nil)
:more-fields `((reg nil :type 'opmask-reg)
(imm nil :type 'avx-condition-code)
(aaa ,k))
:printer `("VCMP" imm ,name-suffix
:tab reg " {" aaa "}, " vvvv ", " reg/mem)))
(:emitter
(multiple-value-bind (cond-arg dst-reg src1 src2-arg mask-val)
(if (register-p condition)
(values src2 condition dst src (or mask 0))
(values condition dst src src2 (or mask 0)))
(let ((imm (or (position cond-arg +avx-conditions+)
(and (integerp cond-arg) (<= 0 cond-arg 31) cond-arg)
(error "~s not one of ~s or 0..31"
cond-arg
+avx-conditions+)))
(mask-num (cond ((null mask-val) 0)
((integerp mask-val) mask-val)
((k-register-p mask-val) (reg-id-num (reg-id mask-val)))
(t 0))))
(aver (k-register-p dst-reg))
(let ((disp-n ,(if scalar
2
`(cond ((zmm-register-p src1) 64)
((ymm-register-p src1) 32)
(t 16)))))
(emit-avx512-inst segment src2-arg dst-reg ,prefix #xc2
:opcode-prefix #x0f3a
:vvvv src1
:w 0
:ll ,(if scalar 0 nil)
:aaa mask-num
:disp-n disp-n
:remaining-bytes 1)
(emit-byte segment imm))))))))
(def-cmp vcmpph nil "PH")
(def-cmp vcmpsh #xf3 "SH" :scalar t))
(macrolet ((def ()
`(progn
(define-instruction vcomish (segment dst src)
,@(avx512-inst-printer-list 'ymm-ymm/mem nil #x2f :opcode-prefix :map5 :w 0 :ll 0)
(:emitter
(emit-avx512-inst segment src dst nil #x2f
:opcode-prefix :map5 :w 0 :ll 0 :disp-n 2)))
(define-instruction vucomish (segment dst src)
,@(avx512-inst-printer-list 'ymm-ymm/mem nil #x2e :opcode-prefix :map5 :w 0 :ll 0)
(:emitter
(emit-avx512-inst segment src dst nil #x2e
:opcode-prefix :map5 :w 0 :ll 0 :disp-n 2)))
(define-instruction vfpclassph (segment dst src imm &optional mask)
,@(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x66 :opcode-prefix #x0f3a :w 0
:more-fields '((reg nil :type 'opmask-reg))
:printer '(:name :tab reg ", " reg/mem ", " imm))
,@(loop for k from 1 to 7
append
(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x66 :opcode-prefix #x0f3a :w 0
:more-fields `((reg nil :type 'opmask-reg) (aaa ,k))
:printer '(:name :tab reg " {" aaa "}, " reg/mem ", " imm)))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t 0))))
(emit-avx512-inst segment src dst nil #x66
:opcode-prefix #x0f3a
:w 0
:aaa mask-num
:disp-n (full-vector-disp-n src)
:remaining-bytes 1)
(emit-byte segment imm))))
(define-instruction vfpclasssh (segment dst src imm &optional mask)
,@(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x67 :opcode-prefix #x0f3a :w 0 :ll 0
:more-fields '((reg nil :type 'opmask-reg))
:printer '(:name :tab reg ", " reg/mem ", " imm))
,@(loop for k from 1 to 7
append
(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x67 :opcode-prefix #x0f3a :w 0 :ll 0
:more-fields `((reg nil :type 'opmask-reg) (aaa ,k))
:printer '(:name :tab reg " {" aaa "}, " reg/mem ", " imm)))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t 0))))
(emit-avx512-inst segment src dst nil #x67
:opcode-prefix #x0f3a
:w 0
:ll 0
:aaa mask-num
:disp-n 2
:remaining-bytes 1)
(emit-byte segment imm))))
(define-instruction vrndscaleph (segment dst src imm &optional mask (zeroing 0))
,@(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x08 :opcode-prefix #x0f3a :w 0
:printer '(:name :tab reg ", " reg/mem ", " imm))
,@(loop for k from 1 to 7
append
(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x08 :opcode-prefix #x0f3a :w 0
:more-fields `((aaa ,k) (z-bit 0))
:printer '(:name :tab reg ", " reg/mem ", " imm " {" aaa "}"))
append
(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x08 :opcode-prefix #x0f3a :w 0
:more-fields `((aaa ,k) (z-bit 1))
:printer '(:name :tab reg ", " reg/mem ", " imm " {" aaa "} {z}")))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t 0)))
(z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)))
(emit-avx512-inst segment src dst nil #x08
:opcode-prefix #x0f3a
:w 0
:aaa mask-num
:z z-num
:disp-n (full-vector-disp-n dst)
:remaining-bytes 1)
(emit-byte segment imm))))
(define-instruction vrndscalesh (segment dst src1 src2 imm &optional mask (zeroing 0))
,@(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x0a :opcode-prefix #x0f3a :w 0 :ll 0
:printer '(:name :tab reg ", " vvvv ", " reg/mem ", " imm))
,@(loop for k from 1 to 7
append
(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x0a :opcode-prefix #x0f3a :w 0 :ll 0
:more-fields `((aaa ,k) (z-bit 0))
:printer '(:name :tab reg ", " vvvv ", " reg/mem ", " imm " {" aaa "}"))
append
(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x0a :opcode-prefix #x0f3a :w 0 :ll 0
:more-fields `((aaa ,k) (z-bit 1))
:printer '(:name :tab reg ", " vvvv ", " reg/mem ", " imm " {" aaa "} {z}")))
(:emitter
(let ((mask-num (cond ((null mask) 0)
((integerp mask) mask)
((k-register-p mask) (reg-id-num (reg-id mask)))
(t 0)))
(z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)))
(emit-avx512-inst segment src2 dst nil #x0a
:opcode-prefix #x0f3a
:vvvv src1
:w 0
:ll 0
:aaa mask-num
:z z-num
:disp-n 2
:remaining-bytes 1)
(emit-byte segment imm)))))))
(def))
;;; Conversions between FP16 and single-precision float (vcvtph2psx, vcvtps2phx)
(macrolet ((def ()
`(progn
(define-instruction vcvtph2psx (segment dst src)
,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 #x13 :opcode-prefix :map6 :w 0)
(:emitter
(let ((ll (cond ((zmm-register-p dst) #b10)
((ymm-register-p dst) #b01)
(t #b00)))
(disp-n (cond ((zmm-register-p dst) 32)
((ymm-register-p dst) 16)
(t 8))))
(emit-avx512-inst segment src dst #x66 #x13
:opcode-prefix :map6
:w 0
:ll ll
:disp-n disp-n))))
(define-instruction vcvtps2phx (segment dst src)
,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 #x1d :opcode-prefix :map5 :w 0)
(:emitter
(let ((ll (cond ((zmm-register-p src) #b10)
((ymm-register-p src) #b01)
(t #b00)))
(disp-n (cond ((zmm-register-p src) 64)
((ymm-register-p src) 32)
(t 16))))
(emit-avx512-inst segment src dst #x66 #x1d
:opcode-prefix :map5
:w 0
:ll ll
:disp-n disp-n))))
(define-instruction vcvtdq2ph (segment dst src)
,@(avx512-inst-printer-list 'ymm-ymm/mem nil #x5b :opcode-prefix :map5 :w 0)
(:emitter
(let ((ll (cond ((zmm-register-p src) #b10)
((ymm-register-p src) #b01)
(t #b00)))
(disp-n (cond ((zmm-register-p src) 64)
((ymm-register-p src) 32)
(t 16))))
(emit-avx512-inst segment src dst nil #x5b
:opcode-prefix :map5
:w 0
:ll ll
:disp-n disp-n)))))))
(def))
(macrolet ((def (name prefix opcode)
`(define-instruction ,name (segment dst src)
,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode :opcode-prefix :map5 :w 0)
(:emitter
(let ((ll (cond ((zmm-register-p dst) #b10)
((ymm-register-p dst) #b01)
(t #b00)))
(disp-n (cond ((zmm-register-p dst) 32)
((ymm-register-p dst) 16)
(t 8))))
(emit-avx512-inst segment src dst ,prefix ,opcode
:opcode-prefix :map5
:w 0
:ll ll
:disp-n disp-n))))))
(def vcvtph2dq #x66 #x5b)
(def vcvttph2dq #xf3 #x5b))
;;; Conversions between FP16 and 16-bit integers (1:1 width)
(macrolet ((def (name prefix opcode)
`(define-instruction ,name (segment dst src)
,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode :opcode-prefix :map5 :w 0)
(:emitter
(emit-avx512-inst segment src dst ,prefix ,opcode
:opcode-prefix :map5
:w 0
:disp-n (full-vector-disp-n dst))))))
(def vcvtuw2ph #xf2 #x7d)
(def vcvtw2ph #xf3 #x7d)
(def vcvtph2w #x66 #x7d)
(def vcvtph2uw nil #x7d)
(def vcvttph2w #x66 #x7c)
(def vcvttph2uw nil #x7c))
;;; Scalar conversions
(macrolet ((def ()
`(progn
(define-instruction vcvtsd2sh (segment dst src1 src2)
,@(avx512-inst-printer-list 'ymm-ymm/mem #xf2 #x5a :opcode-prefix :map5 :w 1 :ll 0 :nds t)
(:emitter
(aver (xmm-register-p dst))
(emit-avx512-inst segment src2 dst #xf2 #x5a
:opcode-prefix :map5
:vvvv src1
:w 1
:ll 0
:disp-n 8)))
(define-instruction vcvtss2sh (segment dst src1 src2)
,@(avx512-inst-printer-list 'ymm-ymm/mem nil #x1d :opcode-prefix :map5 :w 0 :ll 0 :nds t)
(:emitter
(aver (xmm-register-p dst))
(emit-avx512-inst segment src2 dst nil #x1d
:opcode-prefix :map5
:vvvv src1
:w 0
:ll 0
:disp-n 4)))
(define-instruction vcvtsi2sh (segment dst src1 src2)
,@(avx512-inst-printer-list 'ymm-ymm/mem #xf3 #x2a
:opcode-prefix :map5
:reg-mem-size :sized
:nds t :ll 0)
(:emitter
(aver (xmm-register-p dst))
(let ((src-size (operand-size src2)))
(emit-avx512-inst segment src2 dst #xf3 #x2a
:opcode-prefix :map5
:ll 0
:vvvv src1
:w (case src-size
(:qword 1)
(:dword 0)
(t 1))
:disp-n (case src-size
(:qword 8)
(t 4)))))))))
(def))
(macrolet ((def (name opcode)
`(define-instruction ,name (segment dst src)
,@(avx512-inst-printer-list 'reg-ymm/mem #xf3 opcode
:opcode-prefix :map5 :ll 0)
(:emitter
(aver (gpr-p dst))
(let ((dst-size (operand-size dst)))
(aver (or (eq dst-size :qword) (eq dst-size :dword)))
(emit-avx512-inst segment src dst #xf3 ,opcode
:opcode-prefix :map5
:ll 0
:w (ecase dst-size
(:qword 1)
(:dword 0))
:disp-n 2))))))
(def vcvtsh2si #x2d)
(def vcvttsh2si #x2c))
;;; FP16 moves (vmovsh, vmovw)
(macrolet ((def ()
`(progn
(define-instruction vmovsh (segment dst src &optional src2)
,@(avx512-inst-printer-list 'ymm-ymm/mem-dir #xf3 #b0001000 :opcode-prefix :map5 :w 0 :ll 0)
(:emitter
(cond ((ea-p src)
(aver (xmm-register-p dst))
(emit-avx512-inst segment src dst #xf3 #x10
:opcode-prefix :map5 :w 0 :ll 0 :disp-n 2))
((ea-p dst)
(aver (xmm-register-p src))
(emit-avx512-inst segment dst src #xf3 #x11
:opcode-prefix :map5 :w 0 :ll 0 :disp-n 2))
(src2
(aver (and (xmm-register-p dst) (xmm-register-p src) (xmm-register-p src2)))
(emit-avx512-inst segment src2 dst #xf3 #x10
:opcode-prefix :map5 :w 0 :ll 0 :vvvv src))
(t
(aver (and (xmm-register-p dst) (xmm-register-p src)))
(emit-avx512-inst segment src dst #xf3 #x10
:opcode-prefix :map5 :w 0 :ll 0 :vvvv dst)))))
(define-instruction vmovw (segment dst src)
,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 #x6e :opcode-prefix :map5 :w 0 :ll 0
:reg-mem-size :dword)
,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 #x7e :opcode-prefix :map5 :w 0 :ll 0
:reg-mem-size :dword
:printer '(:name :tab reg/mem ", " reg))
(:emitter
(cond ((gpr-p dst)
(aver (xmm-register-p src))
(emit-avx512-inst segment dst src #x66 #x7e
:opcode-prefix :map5 :w 0 :ll 0))
((gpr-p src)
(aver (xmm-register-p dst))
(emit-avx512-inst segment src dst #x66 #x6e
:opcode-prefix :map5 :w 0 :ll 0))
((ea-p dst)
(aver (xmm-register-p src))
(emit-avx512-inst segment dst src #x66 #x7e
:opcode-prefix :map5 :w 0 :ll 0 :disp-n 2))
((ea-p src)
(aver (xmm-register-p dst))
(emit-avx512-inst segment src dst #x66 #x6e
:opcode-prefix :map5 :w 0 :ll 0 :disp-n 2))
(t
(error "Unsupported operands for VMOVW: ~S, ~S" dst src))))))))
(def))

5
src/runtime/x86-64-arch.c
View file

@ -43,7 +43,7 @@
#define UD2_INST 0x0b0f #define UD2_INST 0x0b0f
#define BREAKPOINT_WIDTH 1 #define BREAKPOINT_WIDTH 1
int avx_supported = 0, avx2_supported = 0, avx512_supported = 0; int avx_supported = 0, avx2_supported = 0, avx512_supported = 0, avx512fp16_supported = 0;
static void cpuid(unsigned info, unsigned subinfo, static void cpuid(unsigned info, unsigned subinfo,
unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx)
@ -131,6 +131,9 @@ void tune_asm_routines_for_microarch(void)
if ((ebx & (1u << 16)) && // AVX512F if ((ebx & (1u << 16)) && // AVX512F
((xcr0 & 0xE6) == 0xE6)) { // OS supports ZMM ((xcr0 & 0xE6) == 0xE6)) { // OS supports ZMM
avx512_supported = 1; avx512_supported = 1;
if (edx & (1u << 23)) { // AVX512_FP16
avx512fp16_supported = 1;
}
} }
} }
} }

109
tests/assembler.pure.lisp
View file

@ -418,3 +418,112 @@
(test-assemble `(vdpbf16ps-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z) (test-assemble `(vdpbf16ps-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z)
"62F276C952C2 VDPBF16PS-MASKED ZMM0, ZMM1, ZMM2 {K1} {z}"))) "62F276C952C2 VDPBF16PS-MASKED ZMM0, ZMM1, ZMM2 {K1} {z}")))
#+x86-64
(test-util:with-test (:name :avx512-fp16-instructions)
(let ((k1 (sb-x86-64-asm::get-fpr :kreg 1))
(k2 (sb-x86-64-asm::get-fpr :kreg 2))
(zmm0 (sb-x86-64-asm::get-fpr :zmm 0))
(zmm1 (sb-x86-64-asm::get-fpr :zmm 1))
(zmm2 (sb-x86-64-asm::get-fpr :zmm 2))
(ymm0 (sb-x86-64-asm::get-fpr :ymm 0))
(ymm1 (sb-x86-64-asm::get-fpr :ymm 1))
(xmm0 (sb-x86-64-asm::get-fpr :xmm 0))
(xmm1 (sb-x86-64-asm::get-fpr :xmm 1))
(xmm2 (sb-x86-64-asm::get-fpr :xmm 2)))
;; Vector arithmetic (Map 5, W0)
(test-assemble `(vaddph ,zmm0 ,zmm1 ,zmm2)
"62F5744858C2 VADDPH ZMM0, ZMM1, ZMM2")
(test-assemble `(vaddph-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z)
"62F574C958C2 VADDPH ZMM0, ZMM1, ZMM2 {K1} {z}")
(test-assemble `(vsubph ,zmm0 ,zmm1 ,zmm2)
"62F574485CC2 VSUBPH ZMM0, ZMM1, ZMM2")
(test-assemble `(vmulph ,zmm0 ,zmm1 ,zmm2)
"62F5744859C2 VMULPH ZMM0, ZMM1, ZMM2")
(test-assemble `(vdivph ,zmm0 ,zmm1 ,zmm2)
"62F574485EC2 VDIVPH ZMM0, ZMM1, ZMM2")
(test-assemble `(vminph ,zmm0 ,zmm1 ,zmm2)
"62F574485DC2 VMINPH ZMM0, ZMM1, ZMM2")
(test-assemble `(vmaxph ,zmm0 ,zmm1 ,zmm2)
"62F574485FC2 VMAXPH ZMM0, ZMM1, ZMM2")
(test-assemble `(vsqrtph ,zmm0 ,zmm1)
"62F57C4851C1 VSQRTPH ZMM0, ZMM1")
;; Scalar arithmetic (Map 5, #xF3, W0)
(test-assemble `(vaddsh ,xmm0 ,xmm1 ,xmm2)
"62F5760858C2 VADDSH XMM0, XMM1, XMM2")
(test-assemble `(vsubsh ,xmm0 ,xmm1 ,xmm2)
"62F576085CC2 VSUBSH XMM0, XMM1, XMM2")
(test-assemble `(vmulsh ,xmm0 ,xmm1 ,xmm2)
"62F5760859C2 VMULSH XMM0, XMM1, XMM2")
(test-assemble `(vdivsh ,xmm0 ,xmm1 ,xmm2)
"62F576085EC2 VDIVSH XMM0, XMM1, XMM2")
(test-assemble `(vsqrtsh ,xmm0 ,xmm1 ,xmm2)
"62F5760851C2 VSQRTSH XMM0, XMM1, XMM2")
;; Map 6 instructions (vrcpph, vrsqrtph, vscalefph, FMA)
(test-assemble `(vrcpph ,zmm0 ,zmm1)
"62F67D484CC1 VRCPPH ZMM0, ZMM1")
(test-assemble `(vrsqrtph ,zmm0 ,zmm1)
"62F67D484EC1 VRSQRTPH ZMM0, ZMM1")
(test-assemble `(vscalefph ,zmm0 ,zmm1 ,zmm2)
"62F675482CC2 VSCALEFPH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfmadd132ph ,zmm0 ,zmm1 ,zmm2)
"62F6754898C2 VFMADD132PH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfmadd213ph ,zmm0 ,zmm1 ,zmm2)
"62F67548A8C2 VFMADD213PH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfmadd231ph ,zmm0 ,zmm1 ,zmm2)
"62F67548B8C2 VFMADD231PH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfmsub132ph ,zmm0 ,zmm1 ,zmm2)
"62F675489AC2 VFMSUB132PH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfnmadd132ph ,zmm0 ,zmm1 ,zmm2)
"62F675489CC2 VFNMADD132PH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfnmsub132ph ,zmm0 ,zmm1 ,zmm2)
"62F675489EC2 VFNMSUB132PH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfmaddsub132ph ,zmm0 ,zmm1 ,zmm2)
"62F6754896C2 VFMADDSUB132PH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfmsubadd132ph ,zmm0 ,zmm1 ,zmm2)
"62F6754897C2 VFMSUBADD132PH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfcmaddcph ,zmm0 ,zmm1 ,zmm2)
"62F6774856C2 VFCMADDCPH ZMM0, ZMM1, ZMM2")
(test-assemble `(vfmaddcph ,zmm0 ,zmm1 ,zmm2)
"62F6764856C2 VFMADDCPH ZMM0, ZMM1, ZMM2")
;; Comparisons (vcmpph, vcmpsh)
(try `(vcmpph :eq ,k1 ,zmm1 ,zmm2))
(try `(vcmpph :eq ,k1 ,zmm1 ,zmm2 ,k2))
(try `(vcmpph ,k1 ,zmm1 ,zmm2 :eq))
(try `(vcmpsh :eq ,k1 ,xmm1 ,xmm2))
(try `(vcomish ,xmm0 ,xmm1))
(try `(vucomish ,xmm0 ,xmm1))
(try `(vfpclassph ,k1 ,zmm1 0))
(try `(vfpclasssh ,k1 ,xmm1 0))
(try `(vrndscaleph ,zmm0 ,zmm1 0))
(try `(vrndscalesh ,xmm0 ,xmm1 ,xmm2 0))
;; Conversions
(test-assemble `(vcvtph2psx ,zmm0 ,ymm1)
"62F67D4813C1 VCVTPH2PSX ZMM0, ZMM1")
(test-assemble `(vcvtps2phx ,ymm0 ,zmm1)
"62F57D481DC1 VCVTPS2PHX ZMM0, ZMM1")
(test-assemble `(vcvtdq2ph ,ymm0 ,zmm1)
"62F57C485BC1 VCVTDQ2PH ZMM0, ZMM1")
(test-assemble `(vcvtph2dq ,zmm0 ,ymm1)
"62F57D485BC1 VCVTPH2DQ ZMM0, ZMM1")
(test-assemble `(vcvtuw2ph ,zmm0 ,zmm1)
"62F57F487DC1 VCVTUW2PH ZMM0, ZMM1")
(test-assemble `(vcvtw2ph ,zmm0 ,zmm1)
"62F57E487DC1 VCVTW2PH ZMM0, ZMM1")
(test-assemble `(vcvtph2w ,zmm0 ,zmm1)
"62F57D487DC1 VCVTPH2W ZMM0, ZMM1")
(test-assemble `(vcvtph2uw ,zmm0 ,zmm1)
"62F57C487DC1 VCVTPH2UW ZMM0, ZMM1")
;; Moves
(test-assemble `(vmovw ,xmm0 ,eax)
"62F57D086EC0 VMOVW XMM0, EAX")
(test-assemble `(vmovw ,eax ,xmm0)
"62F57D087EC0 VMOVW EAX, XMM0")
(test-assemble `(vmovsh ,xmm0 ,xmm1 ,xmm2)
"62F5760810C2 VMOVSH XMM0, XMM1, XMM2")))