From 0546c9258889e542cfdaccdfc1af9d2a5bdb3464 Mon Sep 17 00:00:00 2001 From: Aaron Estrada Date: Fri, 4 Sep 2026 16:57:06 -0600 Subject: [PATCH] x86-64: add AVX-512CD, AVX-512VNNI, and AVX-512BF16 instruction definitions --- src/compiler/x86-64/avx512-insts.lisp | 227 +++++++++++++++++++++++++- tests/assembler.pure.lisp | 44 ++++- 2 files changed, 266 insertions(+), 5 deletions(-) diff --git a/src/compiler/x86-64/avx512-insts.lisp b/src/compiler/x86-64/avx512-insts.lisp index 9e5f37c3b..123156537 100644 --- a/src/compiler/x86-64/avx512-insts.lisp +++ b/src/compiler/x86-64/avx512-insts.lisp @@ -3,8 +3,9 @@ ;;;; AVX-512 Instruction Support ;;;; ;;;; Implemented subsets: -;;;; AVX-512F, AVX-512BW, AVX-512DQ, AVX-512IFMA, +;;;; AVX-512F, AVX-512BW, AVX-512DQ, AVX-512CD, AVX-512IFMA, ;;;; AVX-512VBMI, AVX-512VBMI2, AVX-512VPOPCNTDQ, AVX-512BITALG, +;;;; AVX-512VNNI, AVX-512BF16, ;;;; GFNI (in avx2-insts.lisp), ;;;; VPCLMULQDQ-256/512 (via VEX auto-promotion to EVEX), ;;;; EVEX Compare-to-Opmask (vcmpps, vcmppd, vcmpss, vcmpsd), @@ -14,12 +15,9 @@ ;;;; EVEX Gather & Scatter (vpgather*, vgather*, vpscatter*, vscatter*). ;;;; ;;;; Not yet implemented / Future extensions: -;;;; AVX-512CD - vpconflictd/q, vplzcntd/q ;;;; AVX-512VL - Explicit EVEX 128/256-bit forms with masking/broadcast ;;;; (auto-promotion handles basic ZMM; full VL needs ;;;; explicit EVEX for XMM/YMM with masking) -;;;; AVX-512VNNI - vpdpbusd/s, vpdpwssd/s (Neural network integer dot products) -;;;; AVX-512BF16 - vcvtne2ps2bf16, vcvtneps2bf16, vdpbf16ps (Bfloat16) ;;;; AVX-512FP16 - Half-precision FP16 arithmetic (~100 instructions) ;;;; VAES-256/512 - Wide forms of vaesenc/vaesdec ;;;; VP2INTERSECT - vp2intersectd/q @@ -1325,3 +1323,224 @@ (def vscatterdps-z #xa2 0) (def vpscatterdq-z #xa0 1) (def vscatterdpd-z #xa2 1)) + +;;;; ---- AVX-512CD instructions ---- + +;;; Conflict detection (2-operand) +(macrolet ((def (name opcode w) + `(define-instruction ,name (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 opcode + :opcode-prefix #x0f38 :w w) + (:emitter + (emit-avx512-inst segment src dst #x66 ,opcode + :opcode-prefix #x0f38 + :w ,w + :disp-n (full-vector-disp-n dst)))))) + (def vpconflictd #xc4 0) + (def vpconflictq #xc4 1) + (def vplzcntd #x44 0) + (def vplzcntq #x44 1)) + +;;; Conflict detection with opmask +(macrolet ((def (name opcode w) + `(define-instruction ,name (segment dst src mask &optional (zeroing 0)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list + 'ymm-ymm/mem #x66 opcode + :opcode-prefix #x0f38 + :w w + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " reg/mem " {" aaa "}")) + append + (avx512-inst-printer-list + 'ymm-ymm/mem #x66 opcode + :opcode-prefix #x0f38 + :w w + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " reg/mem " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src dst #x66 ,opcode + :opcode-prefix #x0f38 + :w ,w + :aaa mask-num + :z z-num + :disp-n (full-vector-disp-n dst))))))) + (def vpconflictd-masked #xc4 0) + (def vpconflictq-masked #xc4 1) + (def vplzcntd-masked #x44 0) + (def vplzcntq-masked #x44 1)) + +;;; Broadcast mask bit to vector register +(macrolet ((def (name opcode w) + `(define-instruction ,name (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #xf3 opcode + :opcode-prefix #x0f38 :w w) + (:emitter + (emit-avx512-inst segment src dst #xf3 ,opcode + :opcode-prefix #x0f38 + :w ,w))))) + (def vpbroadcastmb2q #x2a 1) + (def vpbroadcastmw2d #x3a 0)) + +;;;; ---- AVX-512VNNI instructions ---- + +;;; Vector Neural Network Instructions (3-operand NDS) +(macrolet ((def (name opcode) + `(define-instruction ,name (segment dst src1 src2) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #x66 opcode + :opcode-prefix #x0f38 :w 0 :nds t) + (:emitter + (emit-avx512-inst segment src2 dst #x66 ,opcode + :opcode-prefix #x0f38 + :vvvv src1 + :w 0 + :disp-n (full-vector-disp-n dst)))))) + (def vpdpbusd #x50) + (def vpdpbusds #x51) + (def vpdpwssd #x52) + (def vpdpwssds #x53)) + +;;; Vector Neural Network Instructions with opmask +(macrolet ((def (name opcode) + `(define-instruction ,name (segment dst src1 src2 mask &optional (zeroing 0)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list + 'ymm-ymm/mem #x66 opcode + :opcode-prefix #x0f38 + :w 0 + :nds t + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "}")) + append + (avx512-inst-printer-list + 'ymm-ymm/mem #x66 opcode + :opcode-prefix #x0f38 + :w 0 + :nds t + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src2 dst #x66 ,opcode + :opcode-prefix #x0f38 + :vvvv src1 + :w 0 + :aaa mask-num + :z z-num + :disp-n (full-vector-disp-n dst))))))) + (def vpdpbusd-masked #x50) + (def vpdpbusds-masked #x51) + (def vpdpwssd-masked #x52) + (def vpdpwssds-masked #x53)) + +;;;; ---- AVX-512BF16 instructions ---- + +;;; Convert two single-precision vectors to bfloat16 (3-operand NDS) +(macrolet ((def (name opcode prefix) + `(define-instruction ,name (segment dst src1 src2) + ,@(avx512-inst-printer-list 'ymm-ymm/mem prefix opcode + :opcode-prefix #x0f38 :w 0 :nds t) + (:emitter + (emit-avx512-inst segment src2 dst ,prefix ,opcode + :opcode-prefix #x0f38 + :vvvv src1 + :w 0 + :disp-n (full-vector-disp-n dst)))))) + (def vcvtne2ps2bf16 #x72 #xf2) + (def vdpbf16ps #x52 #xf3)) + +;;; 3-operand BF16 with opmask +(macrolet ((def (name opcode prefix) + `(define-instruction ,name (segment dst src1 src2 mask &optional (zeroing 0)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list + 'ymm-ymm/mem prefix opcode + :opcode-prefix #x0f38 + :w 0 + :nds t + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "}")) + append + (avx512-inst-printer-list + 'ymm-ymm/mem prefix opcode + :opcode-prefix #x0f38 + :w 0 + :nds t + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " vvvv ", " reg/mem " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0))) + (emit-avx512-inst segment src2 dst ,prefix ,opcode + :opcode-prefix #x0f38 + :vvvv src1 + :w 0 + :aaa mask-num + :z z-num + :disp-n (full-vector-disp-n dst))))))) + (def vcvtne2ps2bf16-masked #x72 #xf2) + (def vdpbf16ps-masked #x52 #xf3)) + +;;; Convert single-precision vector to bfloat16 (2-operand) +(macrolet ((def () + `(define-instruction vcvtneps2bf16 (segment dst src) + ,@(avx512-inst-printer-list 'ymm-ymm/mem #xf3 #x72 + :opcode-prefix #x0f38 :w 0) + (:emitter + (let ((ll (cond ((or (zmm-register-p src) (ymm-register-p dst)) #b10) + ((ymm-register-p src) #b01) + (t #b00))) + (disp-n (if (or (zmm-register-p src) (ymm-register-p dst)) 64 (full-vector-disp-n dst)))) + (emit-avx512-inst segment src dst #xf3 #x72 + :opcode-prefix #x0f38 + :w 0 + :ll ll + :disp-n disp-n)))))) + (def)) + +(macrolet ((def () + `(define-instruction vcvtneps2bf16-masked (segment dst src mask &optional (zeroing 0)) + ,@(loop for k from 1 to 7 + append + (avx512-inst-printer-list + 'ymm-ymm/mem #xf3 #x72 + :opcode-prefix #x0f38 + :w 0 + :more-fields `((aaa ,k) (z-bit 0)) + :printer '(:name :tab reg ", " reg/mem " {" aaa "}")) + append + (avx512-inst-printer-list + 'ymm-ymm/mem #xf3 #x72 + :opcode-prefix #x0f38 + :w 0 + :more-fields `((aaa ,k) (z-bit 1)) + :printer '(:name :tab reg ", " reg/mem " {" aaa "} {z}"))) + (:emitter + (let ((mask-num (cond ((integerp mask) mask) + ((k-register-p mask) (reg-id-num (reg-id mask))) + (t (error "Invalid mask ~S" mask)))) + (z-num (if (or (eq zeroing :z) (eql zeroing 1)) 1 0)) + (ll (cond ((or (zmm-register-p src) (ymm-register-p dst)) #b10) + ((ymm-register-p src) #b01) + (t #b00))) + (disp-n (if (or (zmm-register-p src) (ymm-register-p dst)) 64 (full-vector-disp-n dst)))) + (emit-avx512-inst segment src dst #xf3 #x72 + :opcode-prefix #x0f38 + :w 0 + :ll ll + :aaa mask-num + :z z-num + :disp-n disp-n)))))) + (def)) diff --git a/tests/assembler.pure.lisp b/tests/assembler.pure.lisp index c636a863d..6d9d68215 100644 --- a/tests/assembler.pure.lisp +++ b/tests/assembler.pure.lisp @@ -374,5 +374,47 @@ (test-assemble `(vaddps-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z) "62F174C958C2 VADDPS-MASKED ZMM0, ZMM1, ZMM2 {K1} {z}") (try `(vdivps-masked ,zmm0 ,zmm1 ,zmm2 ,k1)) - (try `(vsqrtps-masked ,zmm0 ,zmm1 ,k1)))) + (try `(vsqrtps-masked ,zmm0 ,zmm1 ,k1)) + + ;; AVX-512CD (Conflict Detection) + (test-assemble `(vpconflictd ,zmm0 ,zmm1) + "62F27D48C4C1 VPCONFLICTD ZMM0, ZMM1") + (test-assemble `(vpconflictq ,zmm0 ,zmm1) + "62F2FD48C4C1 VPCONFLICTQ ZMM0, ZMM1") + (test-assemble `(vplzcntd ,zmm0 ,zmm1) + "62F27D4844C1 VPLZCNTD ZMM0, ZMM1") + (test-assemble `(vplzcntq ,zmm0 ,zmm1) + "62F2FD4844C1 VPLZCNTQ ZMM0, ZMM1") + (test-assemble `(vpconflictd-masked ,zmm0 ,zmm1 ,k1 :z) + "62F27DC9C4C1 VPCONFLICTD-MASKED ZMM0, ZMM1 {K1} {z}") + (test-assemble `(vplzcntd-masked ,zmm0 ,zmm1 ,k1) + "62F27D4944C1 VPLZCNTD-MASKED ZMM0, ZMM1 {K1}") + (try `(vpbroadcastmb2q ,zmm0 ,k1)) + (try `(vpbroadcastmw2d ,zmm0 ,k1)) + + ;; AVX-512VNNI (Vector Neural Network Instructions) + (test-assemble `(vpdpbusd ,zmm0 ,zmm1 ,zmm2) + "62F2754850C2 VPDPBUSD ZMM0, ZMM1, ZMM2") + (test-assemble `(vpdpbusds ,zmm0 ,zmm1 ,zmm2) + "62F2754851C2 VPDPBUSDS ZMM0, ZMM1, ZMM2") + (test-assemble `(vpdpwssd ,zmm0 ,zmm1 ,zmm2) + "62F2754852C2 VPDPWSSD ZMM0, ZMM1, ZMM2") + (test-assemble `(vpdpwssds ,zmm0 ,zmm1 ,zmm2) + "62F2754853C2 VPDPWSSDS ZMM0, ZMM1, ZMM2") + (test-assemble `(vpdpbusd-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z) + "62F275C950C2 VPDPBUSD-MASKED ZMM0, ZMM1, ZMM2 {K1} {z}") + + ;; AVX-512BF16 (Bfloat16 Operations) + (test-assemble `(vcvtne2ps2bf16 ,zmm0 ,zmm1 ,zmm2) + "62F2774872C2 VCVTNE2PS2BF16 ZMM0, ZMM1, ZMM2") + (test-assemble `(vcvtne2ps2bf16-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z) + "62F277C972C2 VCVTNE2PS2BF16-MASKED ZMM0, ZMM1, ZMM2 {K1} {z}") + (test-assemble `(vcvtneps2bf16 ,ymm0 ,zmm1) + "62F27E4872C1 VCVTNEPS2BF16 ZMM0, ZMM1") + (test-assemble `(vcvtneps2bf16-masked ,ymm0 ,zmm1 ,k1 :z) + "62F27EC972C1 VCVTNEPS2BF16-MASKED ZMM0, ZMM1 {K1} {z}") + (test-assemble `(vdpbf16ps ,zmm0 ,zmm1 ,zmm2) + "62F2764852C2 VDPBF16PS ZMM0, ZMM1, ZMM2") + (test-assemble `(vdpbf16ps-masked ,zmm0 ,zmm1 ,zmm2 ,k1 :z) + "62F276C952C2 VDPBF16PS-MASKED ZMM0, ZMM1, ZMM2 {K1} {z}")))