! Fast memmove/memcpy/bcopy
! Copyright Australian National University, 1995
! This file may be used under the terms of the GNU Public License
! Author: Paul Mackerras, September 95
! Minor beautifications David S. Miller
#include <asm/cprefix.h>
.globl C_LABEL(bcopy)
C_LABEL(bcopy):
mov %o0,%o3
mov %o1,%o0
mov %o3,%o1
.globl C_LABEL(amemmove)
C_LABEL(amemmove):
.globl C_LABEL(memmove)
.globl C_LABEL(memcpy)
C_LABEL(memmove):
C_LABEL(memcpy):
save %sp,-96,%sp
mov %i0,%l7
cmp %i0,%i1 ! check for dest within source area
bleu,a 1f
andcc %i0,3,%l1
add %i1,%i2,%l0
cmp %i0,%l0
blu,a Lback
mov %l0,%i1
! copying forwards
! first get dest to be word-aligned
andcc %i0,3,%l1
1:
be,a Lwalign ! if dest already word-aligned
cmp %i2,4
mov 4,%l2
sub %l2,%l1,%l2 ! #bytes until word-aligned
subcc %i2,%l2,%i2
ble,a Lend ! not copying enough to get past word bdry
addcc %i2,%l2,%i2
1:
ldub [%i1],%o0 ! copy single bytes until word-aligned
add %i1,1,%i1
subcc %l2,1,%l2
stb %o0,[%i0]
bgt 1b
add %i0,1,%i0
cmp %i2,4
Lwalign: ! dest now word aligned
blt,a Lend
orcc %i2,%g0,%g0
andcc %i1,3,%l0
be,a Ldoword ! if dest word aligned wrt src
andcc %i0,4,%g0
! yucky cases where we have to shift
mov 4,%l2
sub %l2,%l0,%l2 ! address adjustment, used at Lendn
sll %l0,3,%l0 ! bit offset = shift left count
sll %l2,3,%l1 ! shift right count
add %i1,%l2,%i1 ! round up to next word
ld [%i1-4],%o0 ! get first word
andcc %i0,4,%g0 ! get destination double-word aligned
be,a 1f
andcc %i1,4,%g0
ld [%i1],%o1 ! by constructing and storing one word
add %i0,4,%i0
add %i1,4,%i1
sub %i2,4,%i2
sll %o0,%l0,%o0
srl %o1,%l1,%l6
or %o0,%l6,%o0
st %o0,[%i0-4]
mov %o1,%o0
andcc %i1,4,%g0 ! now construct & store pairs of double-words
1:
bne,a 3f ! if source now not double-word aligned
subcc %i2,4,%i2
subcc %i2,16,%i2
blt 2f
mov %o0,%o1
4:
ldd [%i1],%o2
sll %o1,%l0,%o4
ldd [%i1+8],%o0
add %i0,16,%i0
add %i1,16,%i1
subcc %i2,16,%i2
srl %o2,%l1,%l6
or %l6,%o4,%o4
sll %o2,%l0,%o5
srl %o3,%l1,%l6
or %l6,%o5,%o5
std %o4,[%i0-16]
sll %o3,%l0,%o4
srl %o0,%l1,%l6
or %l6,%o4,%o4
sll %o0,%l0,%o5
srl %o1,%l1,%l6
or %l6,%o5,%o5
bge 4b
std %o4,[%i0-8]
2:
addcc %i2,12,%i2
blt,a Lendn
addcc %i2,4,%i2
5:
ld [%i1],%o2
add %i0,4,%i0
add %i1,4,%i1
subcc %i2,4,%i2
sll %o1,%l0,%o0
srl %o2,%l1,%o1
or %o1,%o0,%o0
st %o0,[%i0-4]
bge 5b
mov %o2,%o1
ba Lendn
addcc %i2,4,%i2
3:
blt,a Lendn
addcc %i2,4,%i2
ld [%i1],%o1
add %i1,4,%i1
subcc %i2,16,%i2
blt,a 8f
addcc %i2,16,%i2
7:
ldd [%i1],%o2
sll %o0,%l0,%o4
srl %o1,%l1,%l6
or %l6,%o4,%o4
sll %o1,%l0,%o5
ldd [%i1+8],%o0
add %i0,16,%i0
add %i1,16,%i1
subcc %i2,16,%i2
srl %o2,%l1,%l6
or %l6,%o5,%o5
std %o4,[%i0-16]
sll %o2,%l0,%o4
srl %o3,%l1,%l6
or %l6,%o4,%o4
sll %o3,%l0,%o5
srl %o0,%l1,%l6
or %l6,%o5,%o5
bge 7b
std %o4,[%i0-8]
addcc %i2,16,%i2
8:
sll %o0,%l0,%o4
srl %o1,%l1,%l6
or %l6,%o4,%o4
st %o4,[%i0]
add %i0,4,%i0
subcc %i2,4,%i2
blt,a Lendn
addcc %i2,4,%i2
mov %o1,%o0
ld [%i1],%o1
ba 8b
add %i1,4,%i1
Ldoword:
! here both dest and src are word-aligned
! make dest double-word aligned
be,a 1f
andcc %i1,4,%g0
ld [%i1],%o0
add %i0,4,%i0
add %i1,4,%i1
sub %i2,4,%i2
st %o0,[%i0-4]
cmp %i2,4
blt,a Lend
orcc %i2,%g0,%g0
andcc %i1,4,%g0
1:
be,a Ldodble ! if source double-word aligned now
subcc %i2,32,%i2
ld [%i1],%o5
add %i1,4,%i1
subcc %i2,36,%i2
blt,a 3f
add %i2,32,%i2
2:
ldd [%i1],%o2
add %i1,32,%i1
subcc %i2,32,%i2
mov %o5,%o0
ldd [%i1-24],%o4
mov %o2,%o1
std %o0,[%i0]
mov %o3,%o2
ldd [%i1-16],%o0
mov %o4,%o3
std %o2,[%i0+8]
mov %o5,%o2
ldd [%i1-8],%o4
mov %o0,%o3
std %o2,[%i0+16]
mov %o1,%o0
mov %o4,%o1
std %o0,[%i0+24]
bge 2b
add %i0,32,%i0
add %i2,32,%i2
3:
st %o5,[%i0]
add %i0,4,%i0
subcc %i2,4,%i2
blt,a Lend
addcc %i2,4,%i2
ld [%i1],%o5
ba 3b
add %i1,4,%i1
Ldodble:
! dest and source are both double-word aligned
blt,a 2f
addcc %i2,28,%i2
1:
ldd [%i1],%o0 ! copy sets of 4 double-words
subcc %i2,32,%i2
ldd [%i1+8],%o2
add %i1,32,%i1
ldd [%i1-16],%o4
add %i0,32,%i0
std %o0,[%i0-32]
ldd [%i1-8],%o0
std %o2,[%i0-24]
std %o4,[%i0-16]
bge 1b
std %o0,[%i0-8]
addcc %i2,28,%i2
2:
blt,a Lend
addcc %i2,4,%i2
3:
ld [%i1],%o0 ! copy words
add %i1,4,%i1
add %i0,4,%i0
subcc %i2,4,%i2
bge 3b
st %o0,[%i0-4]
ba Lend
addcc %i2,4,%i2
Lendn:
sub %i1,%l2,%i1
Lend:
ble Lout
nop
1:
ldub [%i1],%o0
add %i1,1,%i1
subcc %i2,1,%i2
stb %o0,[%i0]
bgt 1b
add %i0,1,%i0
ba Lout
nop
Lback: ! Here we have to copy backwards
add %i0,%i2,%i0
! first get dest to be word-aligned
andcc %i0,3,%l2 ! #bytes until word-aligned
be,a Lbwal ! if dest already word-aligned
cmp %i2,4
subcc %i2,%l2,%i2
ble,a Lbend ! not copying enough to get past word bdry
addcc %i2,%l2,%i2
1:
ldub [%i1-1],%o0 ! copy single bytes until word-aligned
sub %i1,1,%i1
subcc %l2,1,%l2
stb %o0,[%i0-1]
bgt 1b
sub %i0,1,%i0
cmp %i2,4
Lbwal: ! dest now word aligned
blt,a Lbend
orcc %i2,%g0,%g0
andcc %i1,3,%l2
be,a Lbword ! if dest word aligned wrt src
andcc %i0,4,%g0
! yucky cases where we have to shift
! note %l2 used below at Lbendn
mov 4,%l0
sub %l0,%l2,%l0 ! # bytes to right of src in word
sll %l0,3,%l0 ! bit offset = shift right count
sll %l2,3,%l1 ! shift left count
sub %i1,%l2,%i1 ! round down to word boundary
ld [%i1],%o1 ! get first word
andcc %i0,4,%g0 ! get destination double-word aligned
be,a 1f
andcc %i1,4,%g0
ld [%i1-4],%o0 ! by constructing and storing one word
sub %i0,4,%i0
sub %i1,4,%i1
sub %i2,4,%i2
srl %o1,%l0,%o1
sll %o0,%l1,%l6
or %o1,%l6,%o1
st %o1,[%i0]
mov %o0,%o1
andcc %i1,4,%g0 ! now construct & store pairs of double-words
1:
bne,a 3f ! if source now not double-word aligned
subcc %i2,4,%i2
subcc %i2,16,%i2
blt 2f
mov %o1,%o0
4:
ldd [%i1-8],%o2
srl %o0,%l0,%o5
ldd [%i1-16],%o0
sub %i0,16,%i0
sub %i1,16,%i1
subcc %i2,16,%i2
sll %o3,%l1,%l6
or %l6,%o5,%o5
srl %o3,%l0,%o4
sll %o2,%l1,%l6
or %l6,%o4,%o4
std %o4,[%i0+8]
srl %o2,%l0,%o5
sll %o1,%l1,%l6
or %l6,%o5,%o5
srl %o1,%l0,%o4
sll %o0,%l1,%l6
or %l6,%o4,%o4
bge 4b
std %o4,[%i0]
2:
addcc %i2,12,%i2
blt,a Lbendn
addcc %i2,4,%i2
5:
ld [%i1-4],%o2
sub %i0,4,%i0
sub %i1,4,%i1
subcc %i2,4,%i2
srl %o0,%l0,%o0
sll %o2,%l1,%o1
or %o1,%o0,%o0
st %o0,[%i0]
bge 5b
mov %o2,%o0
ba Lbendn
addcc %i2,4,%i2
3:
blt,a Lbendn
addcc %i2,4,%i2
ld [%i1-4],%o0
sub %i1,4,%i1
subcc %i2,16,%i2
blt,a 8f
addcc %i2,16,%i2
7:
ldd [%i1-8],%o2
srl %o1,%l0,%o5
sll %o0,%l1,%l6
or %l6,%o5,%o5
srl %o0,%l0,%o4
ldd [%i1-16],%o0
sub %i0,16,%i0
sub %i1,16,%i1
subcc %i2,16,%i2
sll %o3,%l1,%l6
or %l6,%o4,%o4
std %o4,[%i0+8]
srl %o3,%l0,%o5
sll %o2,%l1,%l6
or %l6,%o5,%o5
srl %o2,%l0,%o4
sll %o1,%l1,%l6
or %l6,%o4,%o4
bge 7b
std %o4,[%i0]
addcc %i2,16,%i2
8:
srl %o1,%l0,%o5
sll %o0,%l1,%l6
or %l6,%o5,%o5
st %o5,[%i0-4]
sub %i0,4,%i0
subcc %i2,4,%i2
blt,a Lbendn
addcc %i2,4,%i2
mov %o0,%o1
ld [%i1-4],%o0
ba 8b
sub %i1,4,%i1
Lbword:
! here both dest and src are word-aligned
! make dest double-word aligned
be,a 1f
andcc %i1,4,%g0
ld [%i1-4],%o0
sub %i0,4,%i0
sub %i1,4,%i1
sub %i2,4,%i2
st %o0,[%i0]
cmp %i2,4
blt,a Lbend
orcc %i2,%g0,%g0
andcc %i1,4,%g0
1:
be,a Lbdble ! if source double-word aligned now
subcc %i2,32,%i2
ld [%i1-4],%o4
sub %i1,4,%i1
subcc %i2,36,%i2
blt,a 3f
add %i2,32,%i2
2:
ldd [%i1-8],%o2
sub %i1,32,%i1
subcc %i2,32,%i2
mov %o4,%o1
ldd [%i1+16],%o4
mov %o3,%o0
std %o0,[%i0-8]
mov %o2,%o3
ldd [%i1+8],%o0
mov %o5,%o2
std %o2,[%i0-16]
mov %o4,%o3
ldd [%i1],%o4
mov %o1,%o2
std %o2,[%i0-24]
mov %o0,%o1
mov %o5,%o0
std %o0,[%i0-32]
bge 2b
sub %i0,32,%i0
add %i2,32,%i2
3:
st %o4,[%i0-4]
sub %i0,4,%i0
subcc %i2,4,%i2
blt,a Lbend
addcc %i2,4,%i2
ld [%i1-4],%o4
ba 3b
sub %i1,4,%i1
Lbdble:
! dest and source are both double-word aligned
blt,a 2f
addcc %i2,28,%i2
1:
ldd [%i1-8],%o0 ! copy sets of 4 double-words
subcc %i2,32,%i2
ldd [%i1-16],%o2
sub %i1,32,%i1
ldd [%i1+8],%o4
sub %i0,32,%i0
std %o0,[%i0+24]
ldd [%i1],%o0
std %o2,[%i0+16]
std %o4,[%i0+8]
bge 1b
std %o0,[%i0]
addcc %i2,28,%i2
2:
blt,a Lbend
addcc %i2,4,%i2
3:
ld [%i1-4],%o0 ! copy words
sub %i1,4,%i1
sub %i0,4,%i0
subcc %i2,4,%i2
bge 3b
st %o0,[%i0]
ba Lbend
addcc %i2,4,%i2
Lbendn:
add %i1,%l2,%i1
Lbend:
ble Lout
nop
1:
ldub [%i1-1],%o0
sub %i1,1,%i1
subcc %i2,1,%i2
stb %o0,[%i0-1]
bgt 1b
sub %i0,1,%i0
Lout:
ret
restore %l7,0,%o0
