Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | /* SPDX-License-Identifier: GPL-2.0 */ /* * NH - ε-almost-universal hash function, ARM64 NEON accelerated version * * Copyright 2018 Google LLC * * Author: Eric Biggers <ebiggers@google.com> */ #include <linux/linkage.h> #include <linux/cfi_types.h> KEY .req x0 MESSAGE .req x1 MESSAGE_LEN .req x2 HASH .req x3 PASS0_SUMS .req v0 PASS1_SUMS .req v1 PASS2_SUMS .req v2 PASS3_SUMS .req v3 K0 .req v4 K1 .req v5 K2 .req v6 K3 .req v7 T0 .req v8 T1 .req v9 T2 .req v10 T3 .req v11 T4 .req v12 T5 .req v13 T6 .req v14 T7 .req v15 .macro _nh_stride k0, k1, k2, k3 // Load next message stride ld1 {T3.16b}, [MESSAGE], #16 // Load next key stride ld1 {\k3\().4s}, [KEY], #16 // Add message words to key words add T0.4s, T3.4s, \k0\().4s add T1.4s, T3.4s, \k1\().4s add T2.4s, T3.4s, \k2\().4s add T3.4s, T3.4s, \k3\().4s // Multiply 32x32 => 64 and accumulate mov T4.d[0], T0.d[1] mov T5.d[0], T1.d[1] mov T6.d[0], T2.d[1] mov T7.d[0], T3.d[1] umlal PASS0_SUMS.2d, T0.2s, T4.2s umlal PASS1_SUMS.2d, T1.2s, T5.2s umlal PASS2_SUMS.2d, T2.2s, T6.2s umlal PASS3_SUMS.2d, T3.2s, T7.2s .endm /* * void nh_neon(const u32 *key, const u8 *message, size_t message_len, * __le64 hash[NH_NUM_PASSES]) * * It's guaranteed that message_len % 16 == 0. */ SYM_TYPED_FUNC_START(nh_neon) ld1 {K0.4s,K1.4s}, [KEY], #32 movi PASS0_SUMS.2d, #0 movi PASS1_SUMS.2d, #0 ld1 {K2.4s}, [KEY], #16 movi PASS2_SUMS.2d, #0 movi PASS3_SUMS.2d, #0 subs MESSAGE_LEN, MESSAGE_LEN, #64 blt .Lloop4_done .Lloop4: _nh_stride K0, K1, K2, K3 _nh_stride K1, K2, K3, K0 _nh_stride K2, K3, K0, K1 _nh_stride K3, K0, K1, K2 subs MESSAGE_LEN, MESSAGE_LEN, #64 bge .Lloop4 .Lloop4_done: ands MESSAGE_LEN, MESSAGE_LEN, #63 beq .Ldone _nh_stride K0, K1, K2, K3 subs MESSAGE_LEN, MESSAGE_LEN, #16 beq .Ldone _nh_stride K1, K2, K3, K0 subs MESSAGE_LEN, MESSAGE_LEN, #16 beq .Ldone _nh_stride K2, K3, K0, K1 .Ldone: // Sum the accumulators for each pass, then store the sums to 'hash' addp T0.2d, PASS0_SUMS.2d, PASS1_SUMS.2d addp T1.2d, PASS2_SUMS.2d, PASS3_SUMS.2d st1 {T0.16b,T1.16b}, [HASH] ret SYM_FUNC_END(nh_neon) |