Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_ARM_DIV64 #define __ASM_ARM_DIV64 #include <linux/types.h> #include <asm/compiler.h> /* * The semantics of __div64_32() are: * * uint32_t __div64_32(uint64_t *n, uint32_t base) * { * uint32_t remainder = *n % base; * *n = *n / base; * return remainder; * } * * In other words, a 64-bit dividend with a 32-bit divisor producing * a 64-bit result and a 32-bit remainder. To accomplish this optimally * we override the generic version in lib/div64.c to call our __do_div64 * assembly implementation with completely non standard calling convention * for arguments and results (beware). */ static inline uint32_t __div64_32(uint64_t *n, uint32_t base) { register unsigned int __base asm("r4") = base; register unsigned long long __n asm("r0") = *n; register unsigned long long __res asm("r2"); unsigned int __rem; asm( __asmeq("%0", "r0") __asmeq("%1", "r2") __asmeq("%2", "r4") "bl __do_div64" : "+r" (__n), "=r" (__res) : "r" (__base) : "ip", "lr", "cc"); __rem = __n >> 32; *n = __res; return __rem; } #define __div64_32 __div64_32 #if !defined(CONFIG_AEABI) /* * In OABI configurations, some uses of the do_div function * cause gcc to run out of registers. To work around that, * we can force the use of the out-of-line version for * configurations that build a OABI kernel. */ #define do_div(n, base) __div64_32(&(n), base) #else static inline uint64_t __arch_xprod_64(uint64_t m, uint64_t n, bool bias) { unsigned long long res; register unsigned int tmp asm("ip") = 0; if (!bias) { asm ( "umull %Q0, %R0, %Q1, %Q2\n\t" "mov %Q0, #0" : "=&r" (res) : "r" (m), "r" (n) : "cc"); } else if (!(m & ((1ULL << 63) | (1ULL << 31)))) { res = m; asm ( "umlal %Q0, %R0, %Q1, %Q2\n\t" "mov %Q0, #0" : "+&r" (res) : "r" (m), "r" (n) : "cc"); } else { asm ( "umull %Q0, %R0, %Q2, %Q3\n\t" "cmn %Q0, %Q2\n\t" "adcs %R0, %R0, %R2\n\t" "adc %Q0, %1, #0" : "=&r" (res), "+&r" (tmp) : "r" (m), "r" (n) : "cc"); } if (!(m & ((1ULL << 63) | (1ULL << 31)))) { asm ( "umlal %R0, %Q0, %R1, %Q2\n\t" "umlal %R0, %Q0, %Q1, %R2\n\t" "mov %R0, #0\n\t" "umlal %Q0, %R0, %R1, %R2" : "+&r" (res) : "r" (m), "r" (n) : "cc"); } else { asm ( "umlal %R0, %Q0, %R2, %Q3\n\t" "umlal %R0, %1, %Q2, %R3\n\t" "mov %R0, #0\n\t" "adds %Q0, %1, %Q0\n\t" "adc %R0, %R0, #0\n\t" "umlal %Q0, %R0, %R2, %R3" : "+&r" (res), "+&r" (tmp) : "r" (m), "r" (n) : "cc"); } return res; } #define __arch_xprod_64 __arch_xprod_64 #include <asm-generic/div64.h> #endif #endif |