x64: add APX NDD lowering for unary and 32-bit integer ops

cranelift/assembler-x64/meta/src/instructions/add.rs

@@ -104,5 +104,6 @@ pub fn list() -> Vec<Inst> {
         // the two sources occupy ModRM.reg and ModRM.rm. `addq` is 64-bit so
         // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `addl` form.
         inst("addq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x01).r(), _64b & apx),
+        inst("addl", fmt("RVM", [r(r32b), w(r32a), r(rm32)]), evex(L128, Full).map4().w0().nd().op(0x01).r(), _64b & apx),
     ]
 }

cranelift/assembler-x64/meta/src/instructions/and.rs

@@ -31,6 +31,7 @@ pub fn list() -> Vec<Inst> {
         inst("andq", fmt("RM", [rw(r64), r(rm64)]), rex(0x23).w().r(), _64b),
         // APX: NDD form of `AND`, promoted into EVEX map 4.
         inst("andq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x21).r(), _64b & apx),
+        inst("andl", fmt("RVM", [r(r32b), w(r32a), r(rm32)]), evex(L128, Full).map4().w0().nd().op(0x21).r(), _64b & apx),
         // BMI1 andn
         inst("andnl", fmt("RVM", [w(r32a), r(r32b), r(rm32)]), vex(LZ)._0f38().w0().op(0xF2), (_64b | compat) & bmi1),
         inst("andnq", fmt("RVM", [w(r64a), r(r64b), r(rm64)]), vex(LZ)._0f38().w1().op(0xF2), _64b & bmi1),

cranelift/assembler-x64/meta/src/instructions/neg.rs

@@ -1,5 +1,5 @@
-use crate::dsl::{Feature::*, Inst, Location::*};
-use crate::dsl::{fmt, inst, rex, rw};
+use crate::dsl::{Feature::*, Inst, Length::*, Location::*, TupleType::*};
+use crate::dsl::{evex, fmt, inst, r, rex, rw, w};
 
 #[rustfmt::skip] // Keeps instructions on a single line.
 pub fn list() -> Vec<Inst> {
@@ -9,10 +9,20 @@ pub fn list() -> Vec<Inst> {
         inst("negw", fmt("M", [rw(rm16)]), rex([0x66, 0xF7]).digit(3), _64b | compat),
         inst("negl", fmt("M", [rw(rm32)]), rex(0xF7).digit(3), _64b | compat),
         inst("negq", fmt("M", [rw(rm64)]), rex(0xF7).w().digit(3), _64b),
+        // APX: NDD (new data destination) form of `NEG`, promoted into EVEX
+        // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural
+        // destination is the `vvvv`-encoded register, so the written operand is
+        // placed in the `V` slot while the single source stays in ModRM.rm; the
+        // ModRM.reg field continues to hold the `/3` opcode extension.
+        inst("negq", fmt("VM", [w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0xF7).digit(3), _64b & apx),
+        inst("negl", fmt("VM", [w(r32a), r(rm32)]), evex(L128, Full).map4().w0().nd().op(0xF7).digit(3), _64b & apx),
         // One's complement negation.
         inst("notb", fmt("M", [rw(rm8)]), rex(0xF6).digit(2), _64b | compat),
         inst("notw", fmt("M", [rw(rm16)]), rex([0x66, 0xF7]).digit(2), _64b | compat),
         inst("notl", fmt("M", [rw(rm32)]), rex(0xF7).digit(2), _64b | compat),
         inst("notq", fmt("M", [rw(rm64)]), rex(0xF7).w().digit(2), _64b),
+        // APX: NDD form of `NOT`, promoted into EVEX "map 4"; see `negq` above.
+        inst("notq", fmt("VM", [w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0xF7).digit(2), _64b & apx),
+        inst("notl", fmt("VM", [w(r32a), r(rm32)]), evex(L128, Full).map4().w0().nd().op(0xF7).digit(2), _64b & apx),
     ]
 }

cranelift/assembler-x64/meta/src/instructions/or.rs

@@ -24,6 +24,7 @@ pub fn list() -> Vec<Inst> {
         inst("orq", fmt("RM", [rw(r64), r(rm64)]), rex(0x0B).w().r(), _64b),
         // APX: NDD form of `OR`, promoted into EVEX map 4.
         inst("orq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x09).r(), _64b & apx),
+        inst("orl", fmt("RVM", [r(r32b), w(r32a), r(rm32)]), evex(L128, Full).map4().w0().nd().op(0x09).r(), _64b & apx),
         // `LOCK`-prefixed memory-writing instructions.
         inst("lock_orb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(1).ib(), _64b | compat).custom(Mnemonic),
         inst("lock_orw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(1).iw(), _64b | compat).custom(Mnemonic),

cranelift/assembler-x64/meta/src/instructions/sub.rs

@@ -29,6 +29,7 @@ pub fn list() -> Vec<Inst> {
         // the two sources occupy ModRM.reg and ModRM.rm. `subq` is 64-bit so
         // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `subl` form.
         inst("subq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x29).r(), _64b & apx),
+        inst("subl", fmt("RVM", [r(r32b), w(r32a), r(rm32)]), evex(L128, Full).map4().w0().nd().op(0x29).r(), _64b & apx),
         // Subtract with borrow.
         inst("sbbb", fmt("I", [rw(al), r(imm8)]), rex(0x1C).ib(), _64b | compat),
         inst("sbbw", fmt("I", [rw(ax), r(imm16)]), rex([0x66, 0x1D]).iw(), _64b | compat),

cranelift/assembler-x64/meta/src/instructions/xor.rs

@@ -24,6 +24,7 @@ pub fn list() -> Vec<Inst> {
         inst("xorq", fmt("RM", [rw(r64), r(rm64)]), rex(0x33).w().r(), _64b),
         // APX: NDD form of `XOR`, promoted into EVEX map 4.
         inst("xorq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x31).r(), _64b & apx),
+        inst("xorl", fmt("RVM", [r(r32b), w(r32a), r(rm32)]), evex(L128, Full).map4().w0().nd().op(0x31).r(), _64b & apx),
         // `LOCK`-prefixed memory-writing instructions.
         inst("lock_xorb", fmt("MI", [rw(m8), r(imm8)]), rex([0xf0, 0x80]).digit(6).ib(), _64b | compat).custom(Mnemonic),
         inst("lock_xorw", fmt("MI", [rw(m16), r(imm16)]), rex([0xf0, 0x66, 0x81]).digit(6).iw(), _64b | compat).custom(Mnemonic),

cranelift/assembler-x64/src/fuzz.rs

@@ -772,4 +772,147 @@ mod test {
         let assembled = assemble(&inst.into());
         assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418317F50");
     }
+
+    /// Byte-level encoding check for the APX NDD form of `NEG`.
+    ///
+    /// `NEG` is a one-source instruction encoded with a `/3` opcode extension,
+    /// so ModRM.reg holds the digit rather than a register. In `VM` format the
+    /// constructor arguments are `[vvvv destination, ModRM.rm source]`. For a
+    /// destination of `%rax` (0, in `vvvv`) and a source of `%rdx` (2, in
+    /// ModRM.rm) with `W = 1` and `ND = 1`, the expected sequence is:
+    ///
+    /// ```text
+    ///   62 F4 FC 18 F7 DA
+    ///   ^^ ^^ ^^ ^^ ^^ ^^
+    ///   |  |  |  |  |  `- ModRM: mod=11, reg=/3, r/m=rdx
+    ///   |  |  |  |  `- opcode 0xF7 (group 3)
+    ///   |  |  |  `- P2: ND=1, V4=1 (inverted), NF=0
+    ///   |  |  `- P1: W=1, vvvv=~rax=1111, pp=00
+    ///   |  `- P0: inverted R/X/B bits, map=100 (map 4)
+    ///   `- EVEX identifier
+    /// ```
+    #[test]
+    fn apx_negq_vm_ndd_encoding() {
+        use crate::inst::negq_vm;
+
+        let inst = negq_vm::<FuzzRegs>::new(FuzzReg::new(0), FuzzReg::new(2));
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC18F7DA");
+    }
+
+    /// Companion to [`apx_negq_vm_ndd_encoding`] covering a memory operand,
+    /// confirming that map-4 `disp8` stays unscaled (`0x50` is not compressed).
+    #[test]
+    fn apx_negq_vm_ndd_disp8_is_unscaled() {
+        use crate::inst::negq_vm;
+        use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+        let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+            base: FuzzReg::new(7),
+            simm32: AmodeOffsetPlusKnownOffset {
+                simm32: AmodeOffset::new(0x50),
+                offset: None,
+            },
+            trap: None,
+        });
+        let inst = negq_vm::<FuzzRegs>::new(FuzzReg::new(7), mem);
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418F75F50");
+    }
+
+    /// Byte-level encoding check for the APX NDD form of `NOT`, which shares the
+    /// group-3 opcode with `NEG` but uses the `/2` extension.
+    #[test]
+    fn apx_notq_vm_ndd_encoding() {
+        use crate::inst::notq_vm;
+
+        let inst = notq_vm::<FuzzRegs>::new(FuzzReg::new(0), FuzzReg::new(2));
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC18F7D2");
+    }
+
+    /// Companion to [`apx_notq_vm_ndd_encoding`] covering a memory operand.
+    #[test]
+    fn apx_notq_vm_ndd_disp8_is_unscaled() {
+        use crate::inst::notq_vm;
+        use crate::mem::{Amode, AmodeOffset, AmodeOffsetPlusKnownOffset, GprMem};
+
+        let mem: GprMem<FuzzReg, FuzzReg> = GprMem::Mem(Amode::ImmReg {
+            base: FuzzReg::new(7),
+            simm32: AmodeOffsetPlusKnownOffset {
+                simm32: AmodeOffset::new(0x50),
+                offset: None,
+            },
+            trap: None,
+        });
+        let inst = notq_vm::<FuzzRegs>::new(FuzzReg::new(7), mem);
+        let assembled = assemble(&inst.into());
+        assert_eq!(pretty_print_hexadecimal(&assembled), "62F4C418F75750");
+    }
+
+    /// Byte-level encoding checks for the 32-bit APX NDD forms.
+    ///
+    /// These differ from their 64-bit counterparts only in the `W` bit, which
+    /// lives in the second payload byte (`P1 = W vvvv U pp`). With `vvvv`
+    /// holding the inverted destination `%rax` (`1111`), `U = 1` and `pp = 00`,
+    /// `W = 1` gives `0xFC` for the `*q` forms and `W = 0` gives `0x7C` here.
+    #[test]
+    fn apx_32bit_rvm_ndd_encodings() {
+        use crate::inst::{addl_rvm, andl_rvm, orl_rvm, subl_rvm, xorl_rvm};
+
+        // `<op>l %ecx, %eax, %edx`: ModRM.reg = %ecx, vvvv = %eax, ModRM.rm = %edx.
+        let (reg, vvvv, rm) = (FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2));
+
+        let inst = addl_rvm::<FuzzRegs>::new(reg, vvvv, rm);
+        assert_eq!(
+            pretty_print_hexadecimal(&assemble(&inst.into())),
+            "62F47C1801CA"
+        );
+
+        let inst = subl_rvm::<FuzzRegs>::new(reg, vvvv, rm);
+        assert_eq!(
+            pretty_print_hexadecimal(&assemble(&inst.into())),
+            "62F47C1829CA"
+        );
+
+        let inst = andl_rvm::<FuzzRegs>::new(reg, vvvv, rm);
+        assert_eq!(
+            pretty_print_hexadecimal(&assemble(&inst.into())),
+            "62F47C1821CA"
+        );
+
+        let inst = orl_rvm::<FuzzRegs>::new(reg, vvvv, rm);
+        assert_eq!(
+            pretty_print_hexadecimal(&assemble(&inst.into())),
+            "62F47C1809CA"
+        );
+
+        let inst = xorl_rvm::<FuzzRegs>::new(reg, vvvv, rm);
+        assert_eq!(
+            pretty_print_hexadecimal(&assemble(&inst.into())),
+            "62F47C1831CA"
+        );
+    }
+
+    /// Byte-level encoding checks for the 32-bit APX NDD unary forms, which
+    /// keep the `/3` and `/2` opcode extensions in ModRM.reg.
+    #[test]
+    fn apx_32bit_vm_ndd_encodings() {
+        use crate::inst::{negl_vm, notl_vm};
+
+        // `<op>l %edx, %eax`: vvvv = %eax, ModRM.rm = %edx.
+        let (vvvv, rm) = (FuzzReg::new(0), FuzzReg::new(2));
+
+        let inst = negl_vm::<FuzzRegs>::new(vvvv, rm);
+        assert_eq!(
+            pretty_print_hexadecimal(&assemble(&inst.into())),
+            "62F47C18F7DA"
+        );
+
+        let inst = notl_vm::<FuzzRegs>::new(vvvv, rm);
+        assert_eq!(
+            pretty_print_hexadecimal(&assemble(&inst.into())),
+            "62F47C18F7D2"
+        );
+    }
 }

cranelift/codegen/src/isa/x64/lower.isle

@@ -103,6 +103,21 @@
       (if-let true (use_apx))
       (x64_addq_rvm x y))
 
+;; APX: the same for `i32` and narrower. Note that `i8`/`i16` deliberately use
+;; the 32-bit NDD form rather than a byte/word-sized one: writing a 32-bit
+;; register breaks the false dependency on the destination's previous value,
+;; which is the same reason `x64_add_break_deps` widens these types for the
+;; legacy encoding. The upper bits of a narrow result are don't-care.
+(rule iadd_apx_ndd_32 1 (lower (iadd $I32 x y))
+      (if-let true (use_apx))
+      (x64_addl_rvm x y))
+(rule iadd_apx_ndd_16 1 (lower (iadd $I16 x y))
+      (if-let true (use_apx))
+      (x64_addl_rvm (put_in_gpr x) (gpr_to_gpr_mem_32 (put_in_gpr y))))
+(rule iadd_apx_ndd_8 1 (lower (iadd $I8 x y))
+      (if-let true (use_apx))
+      (x64_addl_rvm (put_in_gpr x) (gpr_to_gpr_mem_32 (put_in_gpr y))))
+
 ;; Higher-priority cases than the previous two where a load can be sunk into
 ;; the add instruction itself. Note that both operands are tested for
 ;; sink-ability since addition is commutative
@@ -356,6 +371,18 @@
       (if-let true (use_apx))
       (x64_subq_rvm (put_in_gpr y) x))
 
+;; APX: the same for `i32` and narrower; see `iadd_apx_ndd_32` for why `i8` and
+;; `i16` use the 32-bit form.
+(rule isub_apx_ndd_32 1 (lower (isub $I32 x y))
+      (if-let true (use_apx))
+      (x64_subl_rvm (put_in_gpr y) x))
+(rule isub_apx_ndd_16 1 (lower (isub $I16 x y))
+      (if-let true (use_apx))
+      (x64_subl_rvm (put_in_gpr y) (gpr_to_gpr_mem_32 (put_in_gpr x))))
+(rule isub_apx_ndd_8 1 (lower (isub $I8 x y))
+      (if-let true (use_apx))
+      (x64_subl_rvm (put_in_gpr y) (gpr_to_gpr_mem_32 (put_in_gpr x))))
+
 ;; Sub two registers.
 (rule -3 (lower (isub (fits_in_64 ty) x y))
       (x64_sub ty x y))
@@ -452,6 +479,18 @@
       (if-let true (use_apx))
       (x64_andq_rvm x y))
 
+;; APX: the same for `i32` and narrower; see `iadd_apx_ndd_32` for why `i8` and
+;; `i16` use the 32-bit form.
+(rule band_apx_ndd_32 3 (lower (band $I32 x y))
+      (if-let true (use_apx))
+      (x64_andl_rvm x y))
+(rule band_apx_ndd_16 3 (lower (band $I16 x y))
+      (if-let true (use_apx))
+      (x64_andl_rvm (put_in_gpr x) (gpr_to_gpr_mem_32 (put_in_gpr y))))
+(rule band_apx_ndd_8 3 (lower (band $I8 x y))
+      (if-let true (use_apx))
+      (x64_andl_rvm (put_in_gpr x) (gpr_to_gpr_mem_32 (put_in_gpr y))))
+
 ;; And two registers.
 (rule 0 (lower (band ty x y))
       (if (ty_int_ref_scalar_64 ty))
@@ -587,6 +626,18 @@
       (if-let true (use_apx))
       (x64_orq_rvm x y))
 
+;; APX: the same for `i32` and narrower; see `iadd_apx_ndd_32` for why `i8` and
+;; `i16` use the 32-bit form.
+(rule bor_apx_ndd_32 3 (lower (bor $I32 x y))
+      (if-let true (use_apx))
+      (x64_orl_rvm x y))
+(rule bor_apx_ndd_16 3 (lower (bor $I16 x y))
+      (if-let true (use_apx))
+      (x64_orl_rvm (put_in_gpr x) (gpr_to_gpr_mem_32 (put_in_gpr y))))
+(rule bor_apx_ndd_8 3 (lower (bor $I8 x y))
+      (if-let true (use_apx))
+      (x64_orl_rvm (put_in_gpr x) (gpr_to_gpr_mem_32 (put_in_gpr y))))
+
 ;; Or two registers.
 (rule 0 (lower (bor ty x y))
       (if (ty_int_ref_scalar_64 ty))
@@ -666,6 +717,18 @@
       (if-let true (use_apx))
       (x64_xorq_rvm x y))
 
+;; APX: the same for `i32` and narrower; see `iadd_apx_ndd_32` for why `i8` and
+;; `i16` use the 32-bit form.
+(rule bxor_apx_ndd_32 3 (lower (bxor $I32 x y))
+      (if-let true (use_apx))
+      (x64_xorl_rvm x y))
+(rule bxor_apx_ndd_16 3 (lower (bxor $I16 x y))
+      (if-let true (use_apx))
+      (x64_xorl_rvm (put_in_gpr x) (gpr_to_gpr_mem_32 (put_in_gpr y))))
+(rule bxor_apx_ndd_8 3 (lower (bxor $I8 x y))
+      (if-let true (use_apx))
+      (x64_xorl_rvm (put_in_gpr x) (gpr_to_gpr_mem_32 (put_in_gpr y))))
+
 ;; Xor two registers.
 (rule 0 (lower (bxor ty x y))
       (if (ty_int_ref_scalar_64 ty))
@@ -1152,6 +1215,26 @@
 
 ;; `i64` and smaller.
 
+;; APX: when the `has_apx` feature is enabled, use the NDD (new-data-
+;; destination) form of `neg` for 64-bit values. The NDD form writes its result
+;; to a register encoded in the EVEX `vvvv` field, so the register allocator
+;; does not need to preserve the input with a separate move.
+(rule ineg_apx_ndd 1 (lower (ineg $I64 x))
+      (if-let true (use_apx))
+      (x64_negq_vm x))
+
+;; APX: the same for `i32` and narrower; see `iadd_apx_ndd_32` for why `i8` and
+;; `i16` use the 32-bit form.
+(rule ineg_apx_ndd_32 1 (lower (ineg $I32 x))
+      (if-let true (use_apx))
+      (x64_negl_vm x))
+(rule ineg_apx_ndd_16 1 (lower (ineg $I16 x))
+      (if-let true (use_apx))
+      (x64_negl_vm (gpr_to_gpr_mem_32 (put_in_gpr x))))
+(rule ineg_apx_ndd_8 1 (lower (ineg $I8 x))
+      (if-let true (use_apx))
+      (x64_negl_vm (gpr_to_gpr_mem_32 (put_in_gpr x))))
+
 (rule -1 (lower (ineg (fits_in_64 ty) x))
       (x64_neg ty x))
 
@@ -1616,6 +1699,24 @@
 
 ;; `i64` and smaller.
 
+;; APX: as with `ineg` above, prefer the NDD form of `not` for 64-bit values
+;; when APX is enabled so that the input register is not clobbered.
+(rule bnot_apx_ndd 1 (lower (bnot $I64 x))
+      (if-let true (use_apx))
+      (x64_notq_vm x))
+
+;; APX: the same for `i32` and narrower; see `iadd_apx_ndd_32` for why `i8` and
+;; `i16` use the 32-bit form.
+(rule bnot_apx_ndd_32 1 (lower (bnot $I32 x))
+      (if-let true (use_apx))
+      (x64_notl_vm x))
+(rule bnot_apx_ndd_16 1 (lower (bnot $I16 x))
+      (if-let true (use_apx))
+      (x64_notl_vm (gpr_to_gpr_mem_32 (put_in_gpr x))))
+(rule bnot_apx_ndd_8 1 (lower (bnot $I8 x))
+      (if-let true (use_apx))
+      (x64_notl_vm (gpr_to_gpr_mem_32 (put_in_gpr x))))
+
 (rule -2 (lower (bnot ty x))
       (if (ty_int_ref_scalar_64 ty))
       (x64_not ty x))