GCM: Add bulk processing for ARMv8/AArch64 implementation
authorJussi Kivilinna <jussi.kivilinna@iki.fi>
Sun, 9 Oct 2016 09:53:48 +0000 (12:53 +0300)
committerJussi Kivilinna <jussi.kivilinna@iki.fi>
Sun, 9 Oct 2016 14:57:03 +0000 (17:57 +0300)
* cipher/cipher-gcm-armv8-aarch64-ce.S: Add 6 blocks bulk processing.
--

Benchmark on Cortex-A53 (1152 Mhz):

Before:
                    |  nanosecs/byte   mebibytes/sec   cycles/byte
 GMAC_AES           |      1.30 ns/B     731.6 MiB/s      1.50 c/B

After (1.49x faster):
                    |  nanosecs/byte   mebibytes/sec   cycles/byte
 GMAC_AES           |     0.873 ns/B    1092.1 MiB/s      1.01 c/B

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>
cipher/cipher-gcm-armv8-aarch64-ce.S

index 51d67b7..4830b61 100644 (file)
@@ -43,14 +43,37 @@ gcry_gcm_reduction_constant:
 /* Register macros */
 
 #define rhash   v0
-#define rbuf    v1
-#define rh0     v2
-#define rr0     v3
-#define rr1     v4
-#define rrconst v5
-#define vT0     v16
-#define vT1     v17
-#define vZZ     v18
+#define rr0     v1
+#define rr1     v2
+#define rbuf    v3
+#define rbuf1   v4
+#define rbuf2   v5
+#define rbuf3   v6
+#define rbuf4   v7
+#define rbuf5   v8
+#define rr2     v9
+#define rr3     v10
+#define rr4     v11
+#define rr5     v12
+#define rr6     v13
+#define rr7     v14
+#define rr8     v15
+#define rr9     v16
+
+#define rrconst v18
+#define rh1     v19
+#define rh2     v20
+#define rh3     v21
+#define rh4     v22
+#define rh5     v23
+#define rh6     v24
+#define t0      v25
+#define t1      v26
+#define t2      v27
+#define t3      v28
+#define t4      v29
+#define t5      v30
+#define vZZ     v31
 
 /* GHASH macros */
 
@@ -59,38 +82,90 @@ gcry_gcm_reduction_constant:
  */
 
 /* Input: 'a' and 'b', Output: 'r0:r1' (low 128-bits in r0, high in r1) */
-#define PMUL_128x128(r0, r1, a, b, interleave_op) \
-       ext vT0.16b, b.16b, b.16b, #8; \
+#define PMUL_128x128(r0, r1, a, b, T0, T1, interleave_op) \
+       ext T0.16b, b.16b, b.16b, #8; \
        pmull r0.1q, a.1d, b.1d; \
        pmull2 r1.1q, a.2d, b.2d; \
-       pmull vT1.1q, a.1d, vT0.1d; \
-       pmull2 vT0.1q, a.2d, vT0.2d; \
-       interleave_op(); \
-       eor vT0.16b, vT0.16b, vT1.16b; \
-       ext vT1.16b, vZZ.16b, vT0.16b, #8; \
-       ext vT0.16b, vT0.16b, vZZ.16b, #8; \
-       eor r0.16b, r0.16b, vT1.16b; \
-       eor r1.16b, r1.16b, vT0.16b;
+       pmull T1.1q, a.1d, T0.1d; \
+       pmull2 T0.1q, a.2d, T0.2d; \
+       interleave_op; \
+       eor T0.16b, T0.16b, T1.16b; \
+       ext T1.16b, vZZ.16b, T0.16b, #8; \
+       ext T0.16b, T0.16b, vZZ.16b, #8; \
+       eor r0.16b, r0.16b, T1.16b; \
+       eor r1.16b, r1.16b, T0.16b;
+
+/* Input: 'aA' and 'bA', Output: 'r0A:r1A' (low 128-bits in r0A, high in r1A)
+ * Input: 'aB' and 'bB', Output: 'r0B:r1B' (low 128-bits in r0B, high in r1B)
+ * Input: 'aC' and 'bC', Output: 'r0C:r1C' (low 128-bits in r0C, high in r1C)
+ */
+#define PMUL_128x128_3(r0A, r1A, aA, bA, t0A, t1A, \
+                       r0B, r1B, aB, bB, t0B, t1B, \
+                       r0C, r1C, aC, bC, t0C, t1C,  interleave_op) \
+        ext t0A.16b, bA.16b, bA.16b, #8; \
+        pmull r0A.1q, aA.1d, bA.1d; \
+        pmull2 r1A.1q, aA.2d, bA.2d; \
+          ext t0B.16b, bB.16b, bB.16b, #8; \
+          pmull r0B.1q, aB.1d, bB.1d; \
+          pmull2 r1B.1q, aB.2d, bB.2d; \
+            ext t0C.16b, bC.16b, bC.16b, #8; \
+            pmull r0C.1q, aC.1d, bC.1d; \
+            pmull2 r1C.1q, aC.2d, bC.2d; \
+        pmull t1A.1q, aA.1d, t0A.1d; \
+        pmull2 t0A.1q, aA.2d, t0A.2d; \
+          pmull t1B.1q, aB.1d, t0B.1d; \
+          pmull2 t0B.1q, aB.2d, t0B.2d; \
+            pmull t1C.1q, aC.1d, t0C.1d; \
+            pmull2 t0C.1q, aC.2d, t0C.2d; \
+        eor t0A.16b, t0A.16b, t1A.16b; \
+          eor t0B.16b, t0B.16b, t1B.16b; \
+            eor t0C.16b, t0C.16b, t1C.16b; \
+              interleave_op; \
+        ext t1A.16b, vZZ.16b, t0A.16b, #8; \
+        ext t0A.16b, t0A.16b, vZZ.16b, #8; \
+          ext t1B.16b, vZZ.16b, t0B.16b, #8; \
+          ext t0B.16b, t0B.16b, vZZ.16b, #8; \
+            ext t1C.16b, vZZ.16b, t0C.16b, #8; \
+            ext t0C.16b, t0C.16b, vZZ.16b, #8; \
+        eor r0A.16b, r0A.16b, t1A.16b; \
+        eor r1A.16b, r1A.16b, t0A.16b; \
+          eor r0B.16b, r0B.16b, t1B.16b; \
+          eor r1B.16b, r1B.16b, t0B.16b; \
+            eor r0C.16b, r0C.16b, t1C.16b; \
+            eor r1C.16b, r1C.16b, t0C.16b; \
 
 /* Input: 'r0:r1', Output: 'a' */
-#define REDUCTION(a, r0, r1, rconst, interleave_op) \
-        pmull2 vT0.1q, r1.2d, rconst.2d; \
-        interleave_op(); \
-        ext vT1.16b, vT0.16b, vZZ.16b, #8; \
-        ext vT0.16b, vZZ.16b, vT0.16b, #8; \
-        eor r1.16b, r1.16b, vT1.16b; \
-        eor r0.16b, r0.16b, vT0.16b; \
-        pmull vT0.1q, r1.1d, rconst.1d; \
-        eor a.16b, r0.16b, vT0.16b;
-
-#define _(...) /*_*/
-#define ld1_rbuf()  ld1 {rbuf.16b}, [x2], #16;
-#define rbit_rbuf() rbit rbuf.16b, rbuf.16b;
+#define REDUCTION(a, r0, r1, rconst, T0, T1, interleave_op1, interleave_op2, \
+                  interleave_op3) \
+        pmull2 T0.1q, r1.2d, rconst.2d; \
+        interleave_op1; \
+        ext T1.16b, T0.16b, vZZ.16b, #8; \
+        ext T0.16b, vZZ.16b, T0.16b, #8; \
+        interleave_op2; \
+        eor r1.16b, r1.16b, T1.16b; \
+        eor r0.16b, r0.16b, T0.16b; \
+        pmull T0.1q, r1.1d, rconst.1d; \
+        interleave_op3; \
+        eor a.16b, r0.16b, T0.16b;
 
 /* Other functional macros */
 
+#define _(...) __VA_ARGS__
+#define __ _()
+
 #define CLEAR_REG(reg) eor reg.16b, reg.16b, reg.16b;
 
+#define VPUSH_ABI \
+        stp d8, d9, [sp, #-16]!; \
+        stp d10, d11, [sp, #-16]!; \
+        stp d12, d13, [sp, #-16]!; \
+        stp d14, d15, [sp, #-16]!;
+
+#define VPOP_ABI \
+        ldp d14, d15, [sp], #16; \
+        ldp d12, d13, [sp], #16; \
+        ldp d10, d11, [sp], #16; \
+        ldp d8, d9, [sp], #16;
 
 /*
  * unsigned int _gcry_ghash_armv8_ce_pmull (void *gcm_key, byte *result,
@@ -112,16 +187,145 @@ _gcry_ghash_armv8_ce_pmull:
 
   GET_DATA_POINTER(x5, .Lrconst)
 
-  sub x3, x3, #1
-
   eor vZZ.16b, vZZ.16b, vZZ.16b
   ld1 {rhash.16b}, [x1]
-  ld1 {rh0.16b}, [x0]
+  ld1 {rh1.16b}, [x0]
 
   rbit rhash.16b, rhash.16b /* bit-swap */
   ld1r {rrconst.2d}, [x5]
 
+  cmp x3, #6
+  b.lo .Less_than_6
+
+  add x6, x4, #64
+  VPUSH_ABI
+
+  ld1 {rh2.16b-rh5.16b}, [x4]
+  ld1 {rh6.16b}, [x6]
+
+  sub x3, x3, #6
+
+  ld1 {rbuf.16b-rbuf2.16b}, [x2], #(3*16)
+  ld1 {rbuf3.16b-rbuf5.16b}, [x2], #(3*16)
+
+  rbit rbuf.16b, rbuf.16b /* bit-swap */
+  rbit rbuf1.16b, rbuf1.16b /* bit-swap */
+  rbit rbuf2.16b, rbuf2.16b /* bit-swap */
+  rbit rbuf3.16b, rbuf3.16b /* bit-swap */
+  rbit rbuf4.16b, rbuf4.16b /* bit-swap */
+  rbit rbuf5.16b, rbuf5.16b /* bit-swap */
+  eor rhash.16b, rhash.16b, rbuf.16b
+
+  cmp x3, #6
+  b.lo .Lend_6
+
+.Loop_6:
+
+  /* (in1) * H⁵ => rr0:rr1 */
+  /* (in2) * H⁴ => rr2:rr3 */
+  /* (in0 ^ hash) * H⁶ => rr4:rr5 */
+  PMUL_128x128_3(rr0, rr1, rbuf1, rh5, t0, t1,
+                 rr2, rr3, rbuf2, rh4, t2, t3,
+                 rr4, rr5, rhash, rh6, t4, t5,
+                 _(sub x3, x3, #6))
+
+  ld1 {rbuf.16b-rbuf2.16b}, [x2], #(3*16)
+  cmp x3, #6
+
+  eor rr0.16b, rr0.16b, rr2.16b
+  eor rr1.16b, rr1.16b, rr3.16b
+
+  /* (in3) * H³ => rr2:rr3 */
+  /* (in4) * H² => rr6:rr7 */
+  /* (in5) * H¹ => rr8:rr9 */
+  PMUL_128x128_3(rr2, rr3, rbuf3, rh3, t0, t1,
+                 rr6, rr7, rbuf4, rh2, t2, t3,
+                 rr8, rr9, rbuf5, rh1, t4, t5,
+                 _(eor rr0.16b, rr0.16b, rr4.16b;
+                   eor rr1.16b, rr1.16b, rr5.16b))
+
+  eor rr0.16b, rr0.16b, rr2.16b
+  eor rr1.16b, rr1.16b, rr3.16b
+  rbit rbuf.16b, rbuf.16b
+  eor rr0.16b, rr0.16b, rr6.16b
+  eor rr1.16b, rr1.16b, rr7.16b
+  rbit rbuf1.16b, rbuf1.16b
+  eor rr0.16b, rr0.16b, rr8.16b
+  eor rr1.16b, rr1.16b, rr9.16b
+  ld1 {rbuf3.16b-rbuf5.16b}, [x2], #(3*16)
+
+  REDUCTION(rhash, rr0, rr1, rrconst, t0, t1,
+            _(rbit rbuf2.16b, rbuf2.16b),
+            _(rbit rbuf3.16b, rbuf3.16b),
+            _(rbit rbuf4.16b, rbuf4.16b))
+
+  rbit rbuf5.16b, rbuf5.16b
+  eor rhash.16b, rhash.16b, rbuf.16b
+
+  b.hs .Loop_6
+
+.Lend_6:
+
+  /* (in1) * H⁵ => rr0:rr1 */
+  /* (in0 ^ hash) * H⁶ => rr2:rr3 */
+  /* (in2) * H⁴ => rr4:rr5 */
+  PMUL_128x128_3(rr0, rr1, rbuf1, rh5, t0, t1,
+                 rr2, rr3, rhash, rh6, t2, t3,
+                 rr4, rr5, rbuf2, rh4, t4, t5,
+                 __)
+  eor rr0.16b, rr0.16b, rr2.16b
+  eor rr1.16b, rr1.16b, rr3.16b
+  eor rr0.16b, rr0.16b, rr4.16b
+  eor rr1.16b, rr1.16b, rr5.16b
+
+  /* (in3) * H³ => rhash:rbuf */
+  /* (in4) * H² => rr6:rr7 */
+  /* (in5) * H¹ => rr8:rr9 */
+  PMUL_128x128_3(rhash, rbuf, rbuf3, rh3, t0, t1,
+                 rr6, rr7, rbuf4, rh2, t2, t3,
+                 rr8, rr9, rbuf5, rh1, t4, t5,
+                 _(CLEAR_REG(rh4);
+                   CLEAR_REG(rh5);
+                   CLEAR_REG(rh6)))
+  eor rr0.16b, rr0.16b, rhash.16b
+  eor rr1.16b, rr1.16b, rbuf.16b
+  eor rr0.16b, rr0.16b, rr6.16b
+  eor rr1.16b, rr1.16b, rr7.16b
+  eor rr0.16b, rr0.16b, rr8.16b
+  eor rr1.16b, rr1.16b, rr9.16b
+
+  REDUCTION(rhash, rr0, rr1, rrconst, t0, t1,
+            _(CLEAR_REG(rh2);
+              CLEAR_REG(rh3);
+              CLEAR_REG(rr2);
+              CLEAR_REG(rbuf2);
+              CLEAR_REG(rbuf3)),
+            _(CLEAR_REG(rr3);
+              CLEAR_REG(rr4);
+              CLEAR_REG(rr5);
+              CLEAR_REG(rr6);
+              CLEAR_REG(rr7)),
+            _(CLEAR_REG(rr8);
+              CLEAR_REG(rr9);
+              CLEAR_REG(rbuf1);
+              CLEAR_REG(rbuf2)))
+
+  CLEAR_REG(rbuf4)
+  CLEAR_REG(rbuf5)
+  CLEAR_REG(t2)
+  CLEAR_REG(t3)
+  CLEAR_REG(t4)
+  CLEAR_REG(t5)
+
+  VPOP_ABI
+
+  cbz x3, .Ldone
+
+.Less_than_6:
+  /* Handle remaining blocks. */
+
   ld1 {rbuf.16b}, [x2], #16
+  sub x3, x3, #1
 
   rbit rbuf.16b, rbuf.16b /* bit-swap */
 
@@ -130,24 +334,25 @@ _gcry_ghash_armv8_ce_pmull:
   cbz x3, .Lend
 
 .Loop:
-  PMUL_128x128(rr0, rr1, rh0, rhash, ld1_rbuf)
-  sub x3, x3, #1
-  REDUCTION(rhash, rr0, rr1, rrconst, rbit_rbuf)
+  PMUL_128x128(rr0, rr1, rh1, rhash, t0, t1, _(ld1 {rbuf.16b}, [x2], #16))
+  REDUCTION(rhash, rr0, rr1, rrconst, t0, t1,
+            _(sub x3, x3, #1),
+            _(rbit rbuf.16b, rbuf.16b),
+            __)
   eor rhash.16b, rhash.16b, rbuf.16b
 
   cbnz x3, .Loop
 
 .Lend:
-  PMUL_128x128(rr0, rr1, rh0, rhash, _)
-  REDUCTION(rhash, rr0, rr1, rrconst, _)
+  PMUL_128x128(rr0, rr1, rh1, rhash, t0, t1, _(CLEAR_REG(rbuf)))
+  REDUCTION(rhash, rr0, rr1, rrconst, t0, t1, __, _(CLEAR_REG(rh1)), __)
 
+.Ldone:
   CLEAR_REG(rr1)
   CLEAR_REG(rr0)
   rbit rhash.16b, rhash.16b /* bit-swap */
-  CLEAR_REG(rbuf)
-  CLEAR_REG(vT0)
-  CLEAR_REG(vT1)
-  CLEAR_REG(rh0)
+  CLEAR_REG(t0)
+  CLEAR_REG(t1)
 
   st1 {rhash.2d}, [x1]
   CLEAR_REG(rhash)
@@ -170,9 +375,37 @@ _gcry_ghash_setup_armv8_ce_pmull:
    *   x1: gcm_table
    */
 
-  ld1 {vT0.16b}, [x0]
-  rbit vT0.16b, vT0.16b
-  st1 {vT0.16b}, [x0]
+  GET_DATA_POINTER(x2, .Lrconst)
+
+  /* H¹ */
+  ld1 {rh1.16b}, [x0]
+  rbit rh1.16b, rh1.16b
+  st1 {rh1.16b}, [x0]
+
+  ld1r {rrconst.2d}, [x2]
+
+  /* H² */
+  PMUL_128x128(rr0, rr1, rh1, rh1, t0, t1, __)
+  REDUCTION(rh2, rr0, rr1, rrconst, t0, t1, __, __, __)
+
+  /* H³ */
+  PMUL_128x128(rr0, rr1, rh2, rh1, t0, t1, __)
+  REDUCTION(rh3, rr0, rr1, rrconst, t0, t1, __, __, __)
+
+  /* H⁴ */
+  PMUL_128x128(rr0, rr1, rh2, rh2, t0, t1, __)
+  REDUCTION(rh4, rr0, rr1, rrconst, t0, t1, __, __, __)
+
+  /* H⁵ */
+  PMUL_128x128(rr0, rr1, rh2, rh3, t0, t1, __)
+  REDUCTION(rh5, rr0, rr1, rrconst, t0, t1, __, __, __)
+
+  /* H⁶ */
+  PMUL_128x128(rr0, rr1, rh3, rh3, t0, t1, __)
+  REDUCTION(rh6, rr0, rr1, rrconst, t0, t1, __, __, __)
+
+  st1 {rh2.16b-rh4.16b}, [x1], #(3*16)
+  st1 {rh5.16b-rh6.16b}, [x1]
 
   ret
 .size _gcry_ghash_setup_armv8_ce_pmull,.-_gcry_ghash_setup_armv8_ce_pmull;