Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 | /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2013 ARM Ltd. * Copyright (C) 2013 Linaro. * * This code is based on glibc cortex strings work originally authored by Linaro * be found @ * * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ * files/head:/src/aarch64/ */ /* * Copy a buffer from src to dest (alignment handled by the hardware) * * Parameters: * x0 - dest * x1 - src * x2 - n * Returns: * x0 - dest */ dstin .req x0 src .req x1 count .req x2 tmp1 .req x3 tmp1w .req w3 tmp2 .req x4 tmp2w .req w4 dst .req x6 A_l .req x7 A_h .req x8 B_l .req x9 B_h .req x10 C_l .req x11 C_h .req x12 D_l .req x13 D_h .req x14 mov dst, dstin cmp count, #16 /*When memory length is less than 16, the accessed are not aligned.*/ b.lo .Ltiny15 neg tmp2, src ands tmp2, tmp2, #15/* Bytes to reach alignment. */ b.eq .LSrcAligned sub count, count, tmp2 /* * Copy the leading memory data from src to dst in an increasing * address order.By this way,the risk of overwriting the source * memory data is eliminated when the distance between src and * dst is less than 16. The memory accesses here are alignment. */ tbz tmp2, #0, 1f ldrb1 tmp1w, src, #1 strb1 tmp1w, dst, #1 1: tbz tmp2, #1, 2f ldrh1 tmp1w, src, #2 strh1 tmp1w, dst, #2 2: tbz tmp2, #2, 3f ldr1 tmp1w, src, #4 str1 tmp1w, dst, #4 3: tbz tmp2, #3, .LSrcAligned ldr1 tmp1, src, #8 str1 tmp1, dst, #8 .LSrcAligned: cmp count, #64 b.ge .Lcpy_over64 /* * Deal with small copies quickly by dropping straight into the * exit block. */ .Ltail63: /* * Copy up to 48 bytes of data. At this point we only need the * bottom 6 bits of count to be accurate. */ ands tmp1, count, #0x30 b.eq .Ltiny15 cmp tmp1w, #0x20 b.eq 1f b.lt 2f ldp1 A_l, A_h, src, #16 stp1 A_l, A_h, dst, #16 1: ldp1 A_l, A_h, src, #16 stp1 A_l, A_h, dst, #16 2: ldp1 A_l, A_h, src, #16 stp1 A_l, A_h, dst, #16 .Ltiny15: /* * Prefer to break one ldp/stp into several load/store to access * memory in an increasing address order,rather than to load/store 16 * bytes from (src-16) to (dst-16) and to backward the src to aligned * address,which way is used in original cortex memcpy. If keeping * the original memcpy process here, memmove need to satisfy the * precondition that src address is at least 16 bytes bigger than dst * address,otherwise some source data will be overwritten when memove * call memcpy directly. To make memmove simpler and decouple the * memcpy's dependency on memmove, withdrew the original process. */ tbz count, #3, 1f ldr1 tmp1, src, #8 str1 tmp1, dst, #8 1: tbz count, #2, 2f ldr1 tmp1w, src, #4 str1 tmp1w, dst, #4 2: tbz count, #1, 3f ldrh1 tmp1w, src, #2 strh1 tmp1w, dst, #2 3: tbz count, #0, .Lexitfunc ldrb1 tmp1w, src, #1 strb1 tmp1w, dst, #1 b .Lexitfunc .Lcpy_over64: subs count, count, #128 b.ge .Lcpy_body_large /* * Less than 128 bytes to copy, so handle 64 here and then jump * to the tail. */ ldp1 A_l, A_h, src, #16 stp1 A_l, A_h, dst, #16 ldp1 B_l, B_h, src, #16 ldp1 C_l, C_h, src, #16 stp1 B_l, B_h, dst, #16 stp1 C_l, C_h, dst, #16 ldp1 D_l, D_h, src, #16 stp1 D_l, D_h, dst, #16 tst count, #0x3f b.ne .Ltail63 b .Lexitfunc /* * Critical loop. Start at a new cache line boundary. Assuming * 64 bytes per line this ensures the entire loop is in one line. */ .p2align L1_CACHE_SHIFT .Lcpy_body_large: /* pre-get 64 bytes data. */ ldp1 A_l, A_h, src, #16 ldp1 B_l, B_h, src, #16 ldp1 C_l, C_h, src, #16 ldp1 D_l, D_h, src, #16 1: /* * interlace the load of next 64 bytes data block with store of the last * loaded 64 bytes data. */ stp1 A_l, A_h, dst, #16 ldp1 A_l, A_h, src, #16 stp1 B_l, B_h, dst, #16 ldp1 B_l, B_h, src, #16 stp1 C_l, C_h, dst, #16 ldp1 C_l, C_h, src, #16 stp1 D_l, D_h, dst, #16 ldp1 D_l, D_h, src, #16 subs count, count, #64 b.ge 1b stp1 A_l, A_h, dst, #16 stp1 B_l, B_h, dst, #16 stp1 C_l, C_h, dst, #16 stp1 D_l, D_h, dst, #16 tst count, #0x3f b.ne .Ltail63 .Lexitfunc: |