diff --git a/src/compiler/x86-64/avx2-insts.lisp b/src/compiler/x86-64/avx2-insts.lisp index 9ce9283f8..4c839f52a 100644 --- a/src/compiler/x86-64/avx2-insts.lisp +++ b/src/compiler/x86-64/avx2-insts.lisp @@ -326,8 +326,8 @@ (x :field (byte 1 14) :type 'vex-x) (b :field (byte 1 13) :type 'vex-b) (r-prime :field (byte 1 12) :type 'evex-r-prime) - (reserved :field (byte 2 10) :value #b00) ; distinguishes EVEX from BOUND - (mm :field (byte 2 8)) + (reserved :field (byte 1 11) :value 0) ; bit 11 reserved (0 in EVEX) + (mm :field (byte 3 8)) ; bits 10:8 are mmm (maps 1, 2, 3, 5, 6) ;; Byte 2 (w :field (byte 1 23) :type 'evex-w) (vvvv :field (byte 4 19) :type 'evex-ymm-vvvv-reg) @@ -435,9 +435,11 @@ (defun evex-encode-mm (m-mmmm) (ecase m-mmmm - (#x0F #b01) - (#x0F38 #b10) - (#x0F3A #b11)))) + (#x0F #b001) + (#x0F38 #b010) + (#x0F3A #b011) + ((:map5 5 #x5) #b101) + ((:map6 6 #x6) #b110)))) (defun emit-two-byte-vex (segment r vvvv l pp) (emit-bytes segment @@ -518,7 +520,7 @@ ;;; EVEX prefix encoding for AVX-512 ;;; EVEX is a 4-byte prefix: 62h | P1 | P2 | P3 -;;; P1: R(7) X(6) B(5) R'(4) 00(3:2) mm(1:0) +;;; P1: R(7) X(6) B(5) R'(4) 0(3) mmm(2:0) ;;; P2: W(7) vvvv(6:3) 1(2) pp(1:0) ;;; P3: z(7) L'(6) L(5) b(4) V'(3) aaa(2:0) ;;; R, X, B, R', V' are inverted. vvvv is inverted. @@ -526,12 +528,12 @@ (defun emit-evex (segment r x b r-prime opcode-prefix w vvvv pp z ll evex-b v-prime aaa) (emit-bytes segment #x62 - ;; P1: R X B R' 00 mm + ;; P1: R X B R' 0 mmm (logior (ash (logxor 1 r) 7) (ash (logxor 1 x) 6) (ash (logxor 1 b) 5) (ash (logxor 1 r-prime) 4) - ;; bits 3:2 are reserved (0) + ;; bit 3 is reserved (0), bits 2:0 are mmm (maps 1, 2, 3, 5, 6) (evex-encode-mm opcode-prefix)) ;; P2: W vvvv 1 pp (logior (ash w 7) diff --git a/src/compiler/x86-64/avx512-insts.lisp b/src/compiler/x86-64/avx512-insts.lisp index 7879fb9c6..9495854a1 100644 --- a/src/compiler/x86-64/avx512-insts.lisp +++ b/src/compiler/x86-64/avx512-insts.lisp @@ -5,7 +5,7 @@ ;;;; Implemented subsets: ;;;; AVX-512F, AVX-512BW, AVX-512DQ, AVX-512CD, AVX-512IFMA, ;;;; AVX-512VBMI, AVX-512VBMI2, AVX-512VPOPCNTDQ, AVX-512BITALG, -;;;; AVX-512VNNI, AVX-512BF16, +;;;; AVX-512VNNI, AVX-512BF16, AVX-512FP16, ;;;; GFNI (in avx2-insts.lisp), ;;;; VPCLMULQDQ-256/512 (via VEX auto-promotion to EVEX), ;;;; EVEX Compare-to-Opmask (vcmpps, vcmppd, vcmpss, vcmpsd), @@ -18,7 +18,6 @@ ;;;; AVX-512VL - Explicit EVEX 128/256-bit forms with masking/broadcast ;;;; (auto-promotion handles basic ZMM; full VL needs ;;;; explicit EVEX for XMM/YMM with masking) -;;;; AVX-512FP16 - Half-precision FP16 arithmetic (~100 instructions) ;;;; VAES-256/512 - Wide forms of vaesenc/vaesdec ;;;; VP2INTERSECT - vp2intersectd/q ;;;; AVX-512ER/PF - vexp2ps/pd, prefetch (Knights Landing Xeon Phi, deprecated) @@ -1546,3 +1545,573 @@ :z z-num :disp-n disp-n)))))) (def)) + +;;;; ---- AVX-512_FP16 instructions ---- + +;;; 3-operand vector arithmetic (Map 5 & Map 6) +(macrolet ((def (name opcode &optional (opcode-prefix :map5) (prefix nil) (w 0)) + (let ((masked-name (symbolicate name "-MASKED"))) + `(progn + (define-instruction ,name (segment dst src1 src2 &optional mask (zeroing 0)) + ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w :nds t) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list + 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w :nds t + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "}")) + append + (avx512-inst-printer-list + 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w :nds t + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src2 dst ,prefix ,opcode + :opcode-prefix ,opcode-prefix + :vvvv src1 + :w ,w + :aaa mask-num + :z z-num + :disp-n (full-vector-disp-n dst))))) + (define-instruction ,masked-name (segment dst src1 src2 mask &optional (zeroing 0)) + (:emitter + (let ((mask-num (cond ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src2 dst ,prefix ,opcode + :opcode-prefix ,opcode-prefix + :vvvv src1 + :w ,w + :aaa mask-num + :z z-num + :disp-n (full-vector-disp-n dst))))))))) + ;; Map 5 arithmetic + (def vaddph #x58) + (def vsubph #x5c) + (def vmulph #x59) + (def vdivph #x5e) + (def vminph #x5d) + (def vmaxph #x5f) + ;; Map 6 scalef + (def vscalefph #x2c :map6 #x66) + ;; Map 6 FMA (vector) + (def vfmadd132ph #x98 :map6 #x66) + (def vfmadd213ph #xa8 :map6 #x66) + (def vfmadd231ph #xb8 :map6 #x66) + (def vfmsub132ph #x9a :map6 #x66) + (def vfmsub213ph #xaa :map6 #x66) + (def vfmsub231ph #xba :map6 #x66) + (def vfnmadd132ph #x9c :map6 #x66) + (def vfnmadd213ph #xac :map6 #x66) + (def vfnmadd231ph #xbc :map6 #x66) + (def vfnmsub132ph #x9e :map6 #x66) + (def vfnmsub213ph #xae :map6 #x66) + (def vfnmsub231ph #xbe :map6 #x66) + (def vfmaddsub132ph #x96 :map6 #x66) + (def vfmaddsub213ph #xa6 :map6 #x66) + (def vfmaddsub231ph #xb6 :map6 #x66) + (def vfmsubadd132ph #x97 :map6 #x66) + (def vfmsubadd213ph #xa7 :map6 #x66) + (def vfmsubadd231ph #xb7 :map6 #x66) + ;; Map 6 Complex FMA + (def vfcmaddcph #x56 :map6 #xf2) + (def vfmaddcph #x56 :map6 #xf3)) + +;;; 2-operand vector arithmetic (sqrt, rcp, rsqrt) +(macrolet ((def (name opcode &optional (opcode-prefix :map5) (prefix nil) (w 0)) + (let ((masked-name (symbolicate name "-MASKED"))) + `(progn + (define-instruction ,name (segment dst src &optional mask (zeroing 0)) + ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list + 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " reg/mem " {" aaa "}")) + append + (avx512-inst-printer-list + 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " reg/mem " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src dst ,prefix ,opcode + :opcode-prefix ,opcode-prefix + :w ,w + :aaa mask-num + :z z-num + :disp-n (full-vector-disp-n dst))))) + (define-instruction ,masked-name (segment dst src mask &optional (zeroing 0)) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src dst ,prefix ,opcode + :opcode-prefix ,opcode-prefix + :w ,w + :aaa mask-num + :z z-num + :disp-n (full-vector-disp-n dst))))))))) + (def vsqrtph #x51 :map5 nil) + (def vrcpph #x4c :map6 #x66) + (def vrsqrtph #x4e :map6 #x66)) + +;;; Scalar arithmetic (Map 5 & Map 6, 3-operand, ll=0, disp-n=2) +(macrolet ((def (name opcode &optional (opcode-prefix :map5) (prefix #xf3) (w 0)) + (let ((masked-name (symbolicate name "-MASKED"))) + `(progn + (define-instruction ,name (segment dst src1 src2 &optional mask (zeroing 0)) + ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w :ll 0 :nds t) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list + 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w :ll 0 :nds t + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "}")) + append + (avx512-inst-printer-list + 'ymm-ymm/mem prefix opcode + :opcode-prefix opcode-prefix :w w :ll 0 :nds t + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src2 dst ,prefix ,opcode + :opcode-prefix ,opcode-prefix + :vvvv src1 + :w ,w + :ll 0 + :aaa mask-num + :z z-num + :disp-n 2)))) + (define-instruction ,masked-name (segment dst src1 src2 mask &optional (zeroing 0)) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src2 dst ,prefix ,opcode + :opcode-prefix ,opcode-prefix + :vvvv src1 + :w ,w + :ll 0 + :aaa mask-num + :z z-num + :disp-n 2)))))))) + (def vaddsh #x58) + (def vsubsh #x5c) + (def vmulsh #x59) + (def vdivsh #x5e) + (def vminsh #x5d) + (def vmaxsh #x5f) + (def vsqrtsh #x51) + (def vrcpsh #x4d :map6 #x66) + (def vrsqrtsh #x4f :map6 #x66) + (def vscalefsh #x2d :map6 #x66) + ;; Scalar FMA + (def vfmadd132sh #x99 :map6 #x66) + (def vfmadd213sh #xa9 :map6 #x66) + (def vfmadd231sh #xb9 :map6 #x66) + (def vfmsub132sh #x9b :map6 #x66) + (def vfmsub213sh #xab :map6 #x66) + (def vfmsub231sh #xbb :map6 #x66) + (def vfnmadd132sh #x9d :map6 #x66) + (def vfnmadd213sh #xad :map6 #x66) + (def vfnmadd231sh #xbd :map6 #x66) + (def vfnmsub132sh #x9f :map6 #x66) + (def vfnmsub213sh #xaf :map6 #x66) + (def vfnmsub231sh #xbf :map6 #x66)) + +;;; Comparisons & Classification +(macrolet ((def-cmp (name prefix name-suffix &key scalar) + `(define-instruction ,name (segment condition dst src src2 &optional mask) + ,@(avx512-inst-printer-list 'ymm-ymm/mem-imm prefix #xc2 + :opcode-prefix #x0f3a + :w 0 + :ll (if scalar 0 nil) + :more-fields `((reg nil :type 'opmask-reg) + (imm nil :type 'avx-condition-code)) + :printer `("VCMP" imm ,name-suffix + :tab reg ", " vvvv ", " reg/mem)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list + 'ymm-ymm/mem-imm prefix #xc2 + :opcode-prefix #x0f3a + :w 0 + :ll (if scalar 0 nil) + :more-fields `((reg nil :type 'opmask-reg) + (imm nil :type 'avx-condition-code) + (aaa ,k)) + :printer `("VCMP" imm ,name-suffix + :tab reg " {" aaa "}, " vvvv ", " reg/mem))) + (:emitter + (multiple-value-bind (cond-arg dst-reg src1 src2-arg mask-val) + (if (register-p condition) + (values src2 condition dst src (or mask 0)) + (values condition dst src src2 (or mask 0))) + (let ((imm (or (position cond-arg +avx-conditions+) + (and (integerp cond-arg) (<= 0 cond-arg 31) cond-arg) + (error "~s not one of ~s or 0..31" + cond-arg + +avx-conditions+))) + (mask-num (cond ((null mask-val) 0) + ((integerp mask-val) mask-val) + ((k-register-p mask-val) (reg-id-num (reg-id mask-val))) + (t 0)))) + (aver (k-register-p dst-reg)) + (let ((disp-n ,(if scalar + 2 + `(cond ((zmm-register-p src1) 64) + ((ymm-register-p src1) 32) + (t 16))))) + (emit-avx512-inst segment src2-arg dst-reg ,prefix #xc2 + :opcode-prefix #x0f3a + :vvvv src1 + :w 0 + :ll ,(if scalar 0 nil) + :aaa mask-num + :disp-n disp-n + :remaining-bytes 1) + (emit-byte segment imm)))))))) + (def-cmp vcmpph nil "PH") + (def-cmp vcmpsh #xf3 "SH" :scalar t)) + +(macrolet ((def () + `(progn + (define-instruction vcomish (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem nil #x2f :opcode-prefix :map5 :w 0 :ll 0) + (:emitter + (emit-avx512-inst segment src dst nil #x2f + :opcode-prefix :map5 :w 0 :ll 0 :disp-n 2))) + + (define-instruction vucomish (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem nil #x2e :opcode-prefix :map5 :w 0 :ll 0) + (:emitter + (emit-avx512-inst segment src dst nil #x2e + :opcode-prefix :map5 :w 0 :ll 0 :disp-n 2))) + + (define-instruction vfpclassph (segment dst src imm &optional mask) + ,@(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x66 :opcode-prefix #x0f3a :w 0 + :more-fields '((reg nil :type 'opmask-reg)) + :printer '(:name :tab reg ", " reg/mem ", " imm)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x66 :opcode-prefix #x0f3a :w 0 + :more-fields `((reg nil :type 'opmask-reg) (aaa ,k)) + :printer '(:name :tab reg " {" aaa "}, " reg/mem ", " imm))) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t 0)))) + (emit-avx512-inst segment src dst nil #x66 + :opcode-prefix #x0f3a + :w 0 + :aaa mask-num + :disp-n (full-vector-disp-n src) + :remaining-bytes 1) + (emit-byte segment imm)))) + + (define-instruction vfpclasssh (segment dst src imm &optional mask) + ,@(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x67 :opcode-prefix #x0f3a :w 0 :ll 0 + :more-fields '((reg nil :type 'opmask-reg)) + :printer '(:name :tab reg ", " reg/mem ", " imm)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x67 :opcode-prefix #x0f3a :w 0 :ll 0 + :more-fields `((reg nil :type 'opmask-reg) (aaa ,k)) + :printer '(:name :tab reg " {" aaa "}, " reg/mem ", " imm))) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t 0)))) + (emit-avx512-inst segment src dst nil #x67 + :opcode-prefix #x0f3a + :w 0 + :ll 0 + :aaa mask-num + :disp-n 2 + :remaining-bytes 1) + (emit-byte segment imm)))) + + (define-instruction vrndscaleph (segment dst src imm &optional mask (zeroing 0)) + ,@(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x08 :opcode-prefix #x0f3a :w 0 + :printer '(:name :tab reg ", " reg/mem ", " imm)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x08 :opcode-prefix #x0f3a :w 0 + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " reg/mem ", " imm " {" aaa "}")) + append + (avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x08 :opcode-prefix #x0f3a :w 0 + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " reg/mem ", " imm " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t 0))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src dst nil #x08 + :opcode-prefix #x0f3a + :w 0 + :aaa mask-num + :z z-num + :disp-n (full-vector-disp-n dst) + :remaining-bytes 1) + (emit-byte segment imm)))) + + (define-instruction vrndscalesh (segment dst src1 src2 imm &optional mask (zeroing 0)) + ,@(avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x0a :opcode-prefix #x0f3a :w 0 :ll 0 + :printer '(:name :tab reg ", " vvvv ", " reg/mem ", " imm)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x0a :opcode-prefix #x0f3a :w 0 :ll 0 + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem ", " imm " {" aaa "}")) + append + (avx512-inst-printer-list 'ymm-ymm/mem-imm nil #x0a :opcode-prefix #x0f3a :w 0 :ll 0 + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem ", " imm " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((null mask) 0) + ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t 0))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src2 dst nil #x0a + :opcode-prefix #x0f3a + :vvvv src1 + :w 0 + :ll 0 + :aaa mask-num + :z z-num + :disp-n 2 + :remaining-bytes 1) + (emit-byte segment imm))))))) + (def)) + +;;; Conversions between FP16 and single-precision float (vcvtph2psx, vcvtps2phx) +(macrolet ((def () + `(progn + (define-instruction vcvtph2psx (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 #x13 :opcode-prefix :map6 :w 0) + (:emitter + (let ((ll (cond ((zmm-register-p dst) #b10) + ((ymm-register-p dst) #b01) + (t #b00))) + (disp-n (cond ((zmm-register-p dst) 32) + ((ymm-register-p dst) 16) + (t 8)))) + (emit-avx512-inst segment src dst #x66 #x13 + :opcode-prefix :map6 + :w 0 + :ll ll + :disp-n disp-n)))) + + (define-instruction vcvtps2phx (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 #x1d :opcode-prefix :map5 :w 0) + (:emitter + (let ((ll (cond ((zmm-register-p src) #b10) + ((ymm-register-p src) #b01) + (t #b00))) + (disp-n (cond ((zmm-register-p src) 64) + ((ymm-register-p src) 32) + (t 16)))) + (emit-avx512-inst segment src dst #x66 #x1d + :opcode-prefix :map5 + :w 0 + :ll ll + :disp-n disp-n)))) + + (define-instruction vcvtdq2ph (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem nil #x5b :opcode-prefix :map5 :w 0) + (:emitter + (let ((ll (cond ((zmm-register-p src) #b10) + ((ymm-register-p src) #b01) + (t #b00))) + (disp-n (cond ((zmm-register-p src) 64) + ((ymm-register-p src) 32) + (t 16)))) + (emit-avx512-inst segment src dst nil #x5b + :opcode-prefix :map5 + :w 0 + :ll ll + :disp-n disp-n))))))) + (def)) + +(macrolet ((def (name prefix opcode) + `(define-instruction ,name (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode :opcode-prefix :map5 :w 0) + (:emitter + (let ((ll (cond ((zmm-register-p dst) #b10) + ((ymm-register-p dst) #b01) + (t #b00))) + (disp-n (cond ((zmm-register-p dst) 32) + ((ymm-register-p dst) 16) + (t 8)))) + (emit-avx512-inst segment src dst ,prefix ,opcode + :opcode-prefix :map5 + :w 0 + :ll ll + :disp-n disp-n)))))) + (def vcvtph2dq #x66 #x5b) + (def vcvttph2dq #xf3 #x5b)) + +;;; Conversions between FP16 and 16-bit integers (1:1 width) +(macrolet ((def (name prefix opcode) + `(define-instruction ,name (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode :opcode-prefix :map5 :w 0) + (:emitter + (emit-avx512-inst segment src dst ,prefix ,opcode + :opcode-prefix :map5 + :w 0 + :disp-n (full-vector-disp-n dst)))))) + (def vcvtuw2ph #xf2 #x7d) + (def vcvtw2ph #xf3 #x7d) + (def vcvtph2w #x66 #x7d) + (def vcvtph2uw nil #x7d) + (def vcvttph2w #x66 #x7c) + (def vcvttph2uw nil #x7c)) + +;;; Scalar conversions +(macrolet ((def () + `(progn + (define-instruction vcvtsd2sh (segment dst src1 src2) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #xf2 #x5a :opcode-prefix :map5 :w 1 :ll 0 :nds t) + (:emitter + (aver (xmm-register-p dst)) + (emit-avx512-inst segment src2 dst #xf2 #x5a + :opcode-prefix :map5 + :vvvv src1 + :w 1 + :ll 0 + :disp-n 8))) + + (define-instruction vcvtss2sh (segment dst src1 src2) + ,@(avx512-inst-printer-list 'ymm-ymm/mem nil #x1d :opcode-prefix :map5 :w 0 :ll 0 :nds t) + (:emitter + (aver (xmm-register-p dst)) + (emit-avx512-inst segment src2 dst nil #x1d + :opcode-prefix :map5 + :vvvv src1 + :w 0 + :ll 0 + :disp-n 4))) + + (define-instruction vcvtsi2sh (segment dst src1 src2) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #xf3 #x2a + :opcode-prefix :map5 + :reg-mem-size :sized + :nds t :ll 0) + (:emitter + (aver (xmm-register-p dst)) + (let ((src-size (operand-size src2))) + (emit-avx512-inst segment src2 dst #xf3 #x2a + :opcode-prefix :map5 + :ll 0 + :vvvv src1 + :w (case src-size + (:qword 1) + (:dword 0) + (t 1)) + :disp-n (case src-size + (:qword 8) + (t 4))))))))) + (def)) + +(macrolet ((def (name opcode) + `(define-instruction ,name (segment dst src) + ,@(avx512-inst-printer-list 'reg-ymm/mem #xf3 opcode + :opcode-prefix :map5 :ll 0) + (:emitter + (aver (gpr-p dst)) + (let ((dst-size (operand-size dst))) + (aver (or (eq dst-size :qword) (eq dst-size :dword))) + (emit-avx512-inst segment src dst #xf3 ,opcode + :opcode-prefix :map5 + :ll 0 + :w (ecase dst-size + (:qword 1) + (:dword 0)) + :disp-n 2)))))) + (def vcvtsh2si #x2d) + (def vcvttsh2si #x2c)) + +;;; FP16 moves (vmovsh, vmovw) +(macrolet ((def () + `(progn + (define-instruction vmovsh (segment dst src &optional src2) + ,@(avx512-inst-printer-list 'ymm-ymm/mem-dir #xf3 #b0001000 :opcode-prefix :map5 :w 0 :ll 0) + (:emitter + (cond ((ea-p src) + (aver (xmm-register-p dst)) + (emit-avx512-inst segment src dst #xf3 #x10 + :opcode-prefix :map5 :w 0 :ll 0 :disp-n 2)) + ((ea-p dst) + (aver (xmm-register-p src)) + (emit-avx512-inst segment dst src #xf3 #x11 + :opcode-prefix :map5 :w 0 :ll 0 :disp-n 2)) + (src2 + (aver (and (xmm-register-p dst) (xmm-register-p src) (xmm-register-p src2))) + (emit-avx512-inst segment src2 dst #xf3 #x10 + :opcode-prefix :map5 :w 0 :ll 0 :vvvv src)) + (t + (aver (and (xmm-register-p dst) (xmm-register-p src))) + (emit-avx512-inst segment src dst #xf3 #x10 + :opcode-prefix :map5 :w 0 :ll 0 :vvvv dst))))) + + (define-instruction vmovw (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 #x6e :opcode-prefix :map5 :w 0 :ll 0 + :reg-mem-size :dword) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 #x7e :opcode-prefix :map5 :w 0 :ll 0 + :reg-mem-size :dword + :printer '(:name :tab reg/mem ", " reg)) + (:emitter + (cond ((gpr-p dst) + (aver (xmm-register-p src)) + (emit-avx512-inst segment dst src #x66 #x7e + :opcode-prefix :map5 :w 0 :ll 0)) + ((gpr-p src) + (aver (xmm-register-p dst)) + (emit-avx512-inst segment src dst #x66 #x6e + :opcode-prefix :map5 :w 0 :ll 0)) + ((ea-p dst) + (aver (xmm-register-p src)) + (emit-avx512-inst segment dst src #x66 #x7e + :opcode-prefix :map5 :w 0 :ll 0 :disp-n 2)) + ((ea-p src) + (aver (xmm-register-p dst)) + (emit-avx512-inst segment src dst #x66 #x6e + :opcode-prefix :map5 :w 0 :ll 0 :disp-n 2)) + (t + (error "Unsupported operands for VMOVW: ~S, ~S" dst src)))))))) + (def)) diff --git a/src/runtime/x86-64-arch.c b/src/runtime/x86-64-arch.c index e87813305..75efe8fdb 100644 --- a/src/runtime/x86-64-arch.c +++ b/src/runtime/x86-64-arch.c @@ -43,7 +43,7 @@ #define UD2_INST 0x0b0f #define BREAKPOINT_WIDTH 1 -int avx_supported = 0, avx2_supported = 0, avx512_supported = 0; +int avx_supported = 0, avx2_supported = 0, avx512_supported = 0, avx512fp16_supported = 0; static void cpuid(unsigned info, unsigned subinfo, unsigned *eax, unsigned *ebx, unsigned *ecx, unsigned *edx) @@ -131,6 +131,9 @@ void tune_asm_routines_for_microarch(void) if ((ebx & (1u << 16)) && // AVX512F ((xcr0 & 0xE6) == 0xE6)) { // OS supports ZMM avx512_supported = 1; + if (edx & (1u << 23)) { // AVX512_FP16 + avx512fp16_supported = 1; + } } } } diff --git a/tests/assembler.pure.lisp b/tests/assembler.pure.lisp index 6d9d68215..b6809a2ae 100644 --- a/tests/assembler.pure.lisp +++ b/tests/assembler.pure.lisp @@ -418,3 +418,112 @@ (test-assemble `(vdpbf16ps-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z) "62F276C952C2 VDPBF16PS-MASKED ZMM0, ZMM1, ZMM2 {K1} {z}"))) +#+x86-64 +(test-util:with-test (:name :avx512-fp16-instructions) + (let ((k1 (sb-x86-64-asm::get-fpr :kreg 1)) + (k2 (sb-x86-64-asm::get-fpr :kreg 2)) + (zmm0 (sb-x86-64-asm::get-fpr :zmm 0)) + (zmm1 (sb-x86-64-asm::get-fpr :zmm 1)) + (zmm2 (sb-x86-64-asm::get-fpr :zmm 2)) + (ymm0 (sb-x86-64-asm::get-fpr :ymm 0)) + (ymm1 (sb-x86-64-asm::get-fpr :ymm 1)) + (xmm0 (sb-x86-64-asm::get-fpr :xmm 0)) + (xmm1 (sb-x86-64-asm::get-fpr :xmm 1)) + (xmm2 (sb-x86-64-asm::get-fpr :xmm 2))) + ;; Vector arithmetic (Map 5, W0) + (test-assemble `(vaddph ,zmm0 ,zmm1 ,zmm2) + "62F5744858C2 VADDPH ZMM0, ZMM1, ZMM2") + (test-assemble `(vaddph-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z) + "62F574C958C2 VADDPH ZMM0, ZMM1, ZMM2 {K1} {z}") + (test-assemble `(vsubph ,zmm0 ,zmm1 ,zmm2) + "62F574485CC2 VSUBPH ZMM0, ZMM1, ZMM2") + (test-assemble `(vmulph ,zmm0 ,zmm1 ,zmm2) + "62F5744859C2 VMULPH ZMM0, ZMM1, ZMM2") + (test-assemble `(vdivph ,zmm0 ,zmm1 ,zmm2) + "62F574485EC2 VDIVPH ZMM0, ZMM1, ZMM2") + (test-assemble `(vminph ,zmm0 ,zmm1 ,zmm2) + "62F574485DC2 VMINPH ZMM0, ZMM1, ZMM2") + (test-assemble `(vmaxph ,zmm0 ,zmm1 ,zmm2) + "62F574485FC2 VMAXPH ZMM0, ZMM1, ZMM2") + (test-assemble `(vsqrtph ,zmm0 ,zmm1) + "62F57C4851C1 VSQRTPH ZMM0, ZMM1") + + ;; Scalar arithmetic (Map 5, #xF3, W0) + (test-assemble `(vaddsh ,xmm0 ,xmm1 ,xmm2) + "62F5760858C2 VADDSH XMM0, XMM1, XMM2") + (test-assemble `(vsubsh ,xmm0 ,xmm1 ,xmm2) + "62F576085CC2 VSUBSH XMM0, XMM1, XMM2") + (test-assemble `(vmulsh ,xmm0 ,xmm1 ,xmm2) + "62F5760859C2 VMULSH XMM0, XMM1, XMM2") + (test-assemble `(vdivsh ,xmm0 ,xmm1 ,xmm2) + "62F576085EC2 VDIVSH XMM0, XMM1, XMM2") + (test-assemble `(vsqrtsh ,xmm0 ,xmm1 ,xmm2) + "62F5760851C2 VSQRTSH XMM0, XMM1, XMM2") + + ;; Map 6 instructions (vrcpph, vrsqrtph, vscalefph, FMA) + (test-assemble `(vrcpph ,zmm0 ,zmm1) + "62F67D484CC1 VRCPPH ZMM0, ZMM1") + (test-assemble `(vrsqrtph ,zmm0 ,zmm1) + "62F67D484EC1 VRSQRTPH ZMM0, ZMM1") + (test-assemble `(vscalefph ,zmm0 ,zmm1 ,zmm2) + "62F675482CC2 VSCALEFPH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfmadd132ph ,zmm0 ,zmm1 ,zmm2) + "62F6754898C2 VFMADD132PH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfmadd213ph ,zmm0 ,zmm1 ,zmm2) + "62F67548A8C2 VFMADD213PH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfmadd231ph ,zmm0 ,zmm1 ,zmm2) + "62F67548B8C2 VFMADD231PH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfmsub132ph ,zmm0 ,zmm1 ,zmm2) + "62F675489AC2 VFMSUB132PH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfnmadd132ph ,zmm0 ,zmm1 ,zmm2) + "62F675489CC2 VFNMADD132PH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfnmsub132ph ,zmm0 ,zmm1 ,zmm2) + "62F675489EC2 VFNMSUB132PH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfmaddsub132ph ,zmm0 ,zmm1 ,zmm2) + "62F6754896C2 VFMADDSUB132PH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfmsubadd132ph ,zmm0 ,zmm1 ,zmm2) + "62F6754897C2 VFMSUBADD132PH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfcmaddcph ,zmm0 ,zmm1 ,zmm2) + "62F6774856C2 VFCMADDCPH ZMM0, ZMM1, ZMM2") + (test-assemble `(vfmaddcph ,zmm0 ,zmm1 ,zmm2) + "62F6764856C2 VFMADDCPH ZMM0, ZMM1, ZMM2") + + ;; Comparisons (vcmpph, vcmpsh) + (try `(vcmpph :eq ,k1 ,zmm1 ,zmm2)) + (try `(vcmpph :eq ,k1 ,zmm1 ,zmm2 ,k2)) + (try `(vcmpph ,k1 ,zmm1 ,zmm2 :eq)) + (try `(vcmpsh :eq ,k1 ,xmm1 ,xmm2)) + (try `(vcomish ,xmm0 ,xmm1)) + (try `(vucomish ,xmm0 ,xmm1)) + (try `(vfpclassph ,k1 ,zmm1 0)) + (try `(vfpclasssh ,k1 ,xmm1 0)) + (try `(vrndscaleph ,zmm0 ,zmm1 0)) + (try `(vrndscalesh ,xmm0 ,xmm1 ,xmm2 0)) + + ;; Conversions + (test-assemble `(vcvtph2psx ,zmm0 ,ymm1) + "62F67D4813C1 VCVTPH2PSX ZMM0, ZMM1") + (test-assemble `(vcvtps2phx ,ymm0 ,zmm1) + "62F57D481DC1 VCVTPS2PHX ZMM0, ZMM1") + (test-assemble `(vcvtdq2ph ,ymm0 ,zmm1) + "62F57C485BC1 VCVTDQ2PH ZMM0, ZMM1") + (test-assemble `(vcvtph2dq ,zmm0 ,ymm1) + "62F57D485BC1 VCVTPH2DQ ZMM0, ZMM1") + (test-assemble `(vcvtuw2ph ,zmm0 ,zmm1) + "62F57F487DC1 VCVTUW2PH ZMM0, ZMM1") + (test-assemble `(vcvtw2ph ,zmm0 ,zmm1) + "62F57E487DC1 VCVTW2PH ZMM0, ZMM1") + (test-assemble `(vcvtph2w ,zmm0 ,zmm1) + "62F57D487DC1 VCVTPH2W ZMM0, ZMM1") + (test-assemble `(vcvtph2uw ,zmm0 ,zmm1) + "62F57C487DC1 VCVTPH2UW ZMM0, ZMM1") + + ;; Moves + (test-assemble `(vmovw ,xmm0 ,eax) + "62F57D086EC0 VMOVW XMM0, EAX") + (test-assemble `(vmovw ,eax ,xmm0) + "62F57D087EC0 VMOVW EAX, XMM0") + (test-assemble `(vmovsh ,xmm0 ,xmm1 ,xmm2) + "62F5760810C2 VMOVSH XMM0, XMM1, XMM2"))) + +