Linux Audio

Check our new training course

Embedded Linux Audio

Check our new training course
with Creative Commons CC-BY-SA
lecture materials

Bootlin logo

Elixir Cross Referencer

Loading...
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
/*
 * linux/arch/arm/lib/uaccess.S
 *
 * Copyright (C) 1995, 1996,1997,1998 Russell King
 *
 * Routines to block copy data to/from user memory
 *  These are highly optimised both for the 4k page size
 *  and for various alignments.
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/errno.h>

		.text

#define USER(x...)					\
9999:		x;					\
		.section __ex_table,"a";		\
		.align	3;				\
		.long	9999b,9001f;			\
		.previous

#define PAGE_SHIFT 12

/* Prototype: int __arch_copy_to_user(void *to, const char *from, size_t n)
 * Purpose  : copy a block to user memory from kernel memory
 * Params   : to   - user memory
 *          : from - kernel memory
 *          : n    - number of bytes to copy
 * Returns  : Number of bytes NOT copied.
 */

.c2u_dest_not_aligned:
		rsb	ip, ip, #4
		cmp	ip, #2
		ldrb	r3, [r1], #1
USER(		strbt	r3, [r0], #1)			@ May fault
		ldrgeb	r3, [r1], #1
USER(		strgebt	r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #1
USER(		strgtbt	r3, [r0], #1)			@ May fault
		sub	r2, r2, ip
		b	.c2u_dest_aligned

ENTRY(__arch_copy_to_user)
		stmfd	sp!, {r2, r4 - r7, lr}
		cmp	r2, #4
		blt	.c2u_not_enough
		ands	ip, r0, #3
		bne	.c2u_dest_not_aligned
.c2u_dest_aligned:

		ands	ip, r1, #3
		bne	.c2u_src_not_aligned
/*
 * Seeing as there has to be at least 8 bytes to copy, we can
 * copy one word, and force a user-mode page fault...
 */

.c2u_0fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.c2u_0nowords
		ldr	r3, [r1], #4
USER(		strt	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT	@ On each page, use a ld/st??t instruction
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.c2u_0fupi
/*
 * ip = max no. of bytes to copy before needing another "strt" insn
 */
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #32
		blt	.c2u_0rem8lp

.c2u_0cpy8lp:	ldmia	r1!, {r3 - r6}
		stmia	r0!, {r3 - r6}			@ Shouldn't fault
		ldmia	r1!, {r3 - r6}
		stmia	r0!, {r3 - r6}			@ Shouldn't fault
		subs	ip, ip, #32
		bpl	.c2u_0cpy8lp
.c2u_0rem8lp:	cmn	ip, #16
		ldmgeia	r1!, {r3 - r6}
		stmgeia	r0!, {r3 - r6}			@ Shouldn't fault
		tst	ip, #8
		ldmneia	r1!, {r3 - r4}
		stmneia	r0!, {r3 - r4}			@ Shouldn't fault
		tst	ip, #4
		ldrne	r3, [r1], #4
		strnet	r3, [r0], #4			@ Shouldn't fault
		ands	ip, ip, #3
		beq	.c2u_0fupi
.c2u_0nowords:	teq	ip, #0
		beq	.c2u_finished
.c2u_nowords:	cmp	ip, #2
		ldrb	r3, [r1], #1
USER(		strbt	r3, [r0], #1)			@ May fault
		ldrgeb	r3, [r1], #1
USER(		strgebt	r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #1
USER(		strgtbt	r3, [r0], #1)			@ May fault
		b	.c2u_finished

.c2u_not_enough:
		movs	ip, r2
		bne	.c2u_nowords
.c2u_finished:	mov	r0, #0
		LOADREGS(fd,sp!,{r2, r4 - r7, pc})

.c2u_src_not_aligned:
		bic	r1, r1, #3
		ldr	r7, [r1], #4
		cmp	ip, #2
		bgt	.c2u_3fupi
		beq	.c2u_2fupi
.c2u_1fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.c2u_1nowords
		mov	r3, r7, lsr #8
		ldr	r7, [r1], #4
		orr	r3, r3, r7, lsl #24
USER(		strt	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.c2u_1fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.c2u_1rem8lp

.c2u_1cpy8lp:	mov	r3, r7, lsr #8
		ldmia	r1!, {r4 - r7}
		orr	r3, r3, r4, lsl #24
		mov	r4, r4, lsr #8
		orr	r4, r4, r5, lsl #24
		mov	r5, r5, lsr #8
		orr	r5, r5, r6, lsl #24
		mov	r6, r6, lsr #8
		orr	r6, r6, r7, lsl #24
		stmia	r0!, {r3 - r6}			@ Shouldn't fault
		subs	ip, ip, #16
		bpl	.c2u_1cpy8lp
.c2u_1rem8lp:	tst	ip, #8
		movne	r3, r7, lsr #8
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, lsl #24
		movne	r4, r4, lsr #8
		orrne	r4, r4, r7, lsl #24
		stmneia	r0!, {r3 - r4}			@ Shouldn't fault
		tst	ip, #4
		movne	r3, r7, lsr #8
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, lsl #24
		strnet	r3, [r0], #4			@ Shouldn't fault
		ands	ip, ip, #3
		beq	.c2u_1fupi
.c2u_1nowords:	mov	r3, r7, lsr #8
		teq	ip, #0
		beq	.c2u_finished
		cmp	ip, #2
USER(		strbt	r3, [r0], #1)			@ May fault
		movge	r3, r3, lsr #8
USER(		strgebt	r3, [r0], #1)			@ May fault
		movgt	r3, r3, lsr #8
USER(		strgtbt	r3, [r0], #1)			@ May fault
		b	.c2u_finished

.c2u_2fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.c2u_2nowords
		mov	r3, r7, lsr #16
		ldr	r7, [r1], #4
		orr	r3, r3, r7, lsl #16
USER(		strt	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.c2u_2fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.c2u_2rem8lp

.c2u_2cpy8lp:	mov	r3, r7, lsr #16
		ldmia	r1!, {r4 - r7}
		orr	r3, r3, r4, lsl #16
		mov	r4, r4, lsr #16
		orr	r4, r4, r5, lsl #16
		mov	r5, r5, lsr #16
		orr	r5, r5, r6, lsl #16
		mov	r6, r6, lsr #16
		orr	r6, r6, r7, lsl #16
		stmia	r0!, {r3 - r6}			@ Shouldn't fault
		subs	ip, ip, #16
		bpl	.c2u_2cpy8lp
.c2u_2rem8lp:	tst	ip, #8
		movne	r3, r7, lsr #16
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, lsl #16
		movne	r4, r4, lsr #16
		orrne	r4, r4, r7, lsl #16
		stmneia	r0!, {r3 - r4}			@ Shouldn't fault
		tst	ip, #4
		movne	r3, r7, lsr #16
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, lsl #16
		strnet	r3, [r0], #4			@ Shouldn't fault
		ands	ip, ip, #3
		beq	.c2u_2fupi
.c2u_2nowords:	mov	r3, r7, lsr #16
		teq	ip, #0
		beq	.c2u_finished
		cmp	ip, #2
USER(		strbt	r3, [r0], #1)			@ May fault
		movge	r3, r3, lsr #8
USER(		strgebt	r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #0
USER(		strgtbt	r3, [r0], #1)			@ May fault
		b	.c2u_finished

.c2u_3fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.c2u_3nowords
		mov	r3, r7, lsr #24
		ldr	r7, [r1], #4
		orr	r3, r3, r7, lsl #8
USER(		strt	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.c2u_3fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.c2u_3rem8lp

.c2u_3cpy8lp:	mov	r3, r7, lsr #24
		ldmia	r1!, {r4 - r7}
		orr	r3, r3, r4, lsl #8
		mov	r4, r4, lsr #24
		orr	r4, r4, r5, lsl #8
		mov	r5, r5, lsr #24
		orr	r5, r5, r6, lsl #8
		mov	r6, r6, lsr #24
		orr	r6, r6, r7, lsl #8
		stmia	r0!, {r3 - r6}			@ Shouldn't fault
		subs	ip, ip, #16
		bpl	.c2u_3cpy8lp
.c2u_3rem8lp:	tst	ip, #8
		movne	r3, r7, lsr #24
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, lsl #8
		movne	r4, r4, lsr #24
		orrne	r4, r4, r7, lsl #8
		stmneia	r0!, {r3 - r4}			@ Shouldn't fault
		tst	ip, #4
		movne	r3, r7, lsr #24
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, lsl #8
		strnet	r3, [r0], #4			@ Shouldn't fault
		ands	ip, ip, #3
		beq	.c2u_3fupi
.c2u_3nowords:	mov	r3, r7, lsr #24
		teq	ip, #0
		beq	.c2u_finished
		cmp	ip, #2
USER(		strbt	r3, [r0], #1)			@ May fault
		ldrge	r3, [r1], #0
USER(		strgebt	r3, [r0], #1)			@ May fault
		movgt	r3, r3, lsr #8
USER(		strgtbt	r3, [r0], #1)			@ May fault
		b	.c2u_finished

		.section .fixup,"ax"
		.align	0
9001:		LOADREGS(fd,sp!, {r0, r4 - r7, pc})
		.previous

/* Prototype: unsigned long __arch_copy_from_user(void *to,const void *from,unsigned long n);
 * Purpose  : copy a block from user memory to kernel memory
 * Params   : to   - kernel memory
 *          : from - user memory
 *          : n    - number of bytes to copy
 * Returns  : Number of bytes NOT copied.
 */
.cfu_dest_not_aligned:
		rsb	ip, ip, #4
		cmp	ip, #2
USER(		ldrbt	r3, [r1], #1)			@ May fault
		strb	r3, [r0], #1
USER(		ldrgebt	r3, [r1], #1)			@ May fault
		strgeb	r3, [r0], #1
USER(		ldrgtbt	r3, [r1], #1)			@ May fault
		strgtb	r3, [r0], #1
		sub	r2, r2, ip
		b	.cfu_dest_aligned

ENTRY(__arch_copy_from_user)
		stmfd	sp!, {r0, r2, r4 - r7, lr}
		cmp	r2, #4
		blt	.cfu_not_enough
		ands	ip, r0, #3
		bne	.cfu_dest_not_aligned
.cfu_dest_aligned:
		ands	ip, r1, #3
		bne	.cfu_src_not_aligned
/*
 * Seeing as there has to be at least 8 bytes to copy, we can
 * copy one word, and force a user-mode page fault...
 */

.cfu_0fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.cfu_0nowords
USER(		ldrt	r3, [r1], #4)
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT	@ On each page, use a ld/st??t instruction
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.cfu_0fupi
/*
 * ip = max no. of bytes to copy before needing another "strt" insn
 */
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #32
		blt	.cfu_0rem8lp

.cfu_0cpy8lp:	ldmia	r1!, {r3 - r6}			@ Shouldn't fault
		stmia	r0!, {r3 - r6}
		ldmia	r1!, {r3 - r6}			@ Shouldn't fault
		stmia	r0!, {r3 - r6}
		subs	ip, ip, #32
		bpl	.cfu_0cpy8lp
.cfu_0rem8lp:	cmn	ip, #16
		ldmgeia	r1!, {r3 - r6}			@ Shouldn't fault
		stmgeia	r0!, {r3 - r6}
		tst	ip, #8
		ldmneia	r1!, {r3 - r4}			@ Shouldn't fault
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		ldrnet	r3, [r1], #4			@ Shouldn't fault
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.cfu_0fupi
.cfu_0nowords:	teq	ip, #0
		beq	.cfu_finished
.cfu_nowords:	cmp	ip, #2
USER(		ldrbt	r3, [r1], #1)			@ May fault
		strb	r3, [r0], #1
USER(		ldrgebt	r3, [r1], #1)			@ May fault
		strgeb	r3, [r0], #1
USER(		ldrgtbt	r3, [r1], #1)			@ May fault
		strgtb	r3, [r0], #1
		b	.cfu_finished

.cfu_not_enough:
		movs	ip, r2
		bne	.cfu_nowords
.cfu_finished:	mov	r0, #0
		add	sp, sp, #8
		LOADREGS(fd,sp!,{r4 - r7, pc})

.cfu_src_not_aligned:
		bic	r1, r1, #3
USER(		ldrt	r7, [r1], #4)			@ May fault
		cmp	ip, #2
		bgt	.cfu_3fupi
		beq	.cfu_2fupi
.cfu_1fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.cfu_1nowords
		mov	r3, r7, lsr #8
USER(		ldrt	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, lsl #24
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.cfu_1fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.cfu_1rem8lp

.cfu_1cpy8lp:	mov	r3, r7, lsr #8
		ldmia	r1!, {r4 - r7}			@ Shouldn't fault
		orr	r3, r3, r4, lsl #24
		mov	r4, r4, lsr #8
		orr	r4, r4, r5, lsl #24
		mov	r5, r5, lsr #8
		orr	r5, r5, r6, lsl #24
		mov	r6, r6, lsr #8
		orr	r6, r6, r7, lsl #24
		stmia	r0!, {r3 - r6}
		subs	ip, ip, #16
		bpl	.cfu_1cpy8lp
.cfu_1rem8lp:	tst	ip, #8
		movne	r3, r7, lsr #8
		ldmneia	r1!, {r4, r7}			@ Shouldn't fault
		orrne	r3, r3, r4, lsl #24
		movne	r4, r4, lsr #8
		orrne	r4, r4, r7, lsl #24
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, lsr #8
USER(		ldrnet	r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, lsl #24
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.cfu_1fupi
.cfu_1nowords:	mov	r3, r7, lsr #8
		teq	ip, #0
		beq	.cfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
		movge	r3, r3, lsr #8
		strgeb	r3, [r0], #1
		movgt	r3, r3, lsr #8
		strgtb	r3, [r0], #1
		b	.cfu_finished

.cfu_2fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.cfu_2nowords
		mov	r3, r7, lsr #16
USER(		ldrt	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, lsl #16
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.cfu_2fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.cfu_2rem8lp

.cfu_2cpy8lp:	mov	r3, r7, lsr #16
		ldmia	r1!, {r4 - r7}			@ Shouldn't fault
		orr	r3, r3, r4, lsl #16
		mov	r4, r4, lsr #16
		orr	r4, r4, r5, lsl #16
		mov	r5, r5, lsr #16
		orr	r5, r5, r6, lsl #16
		mov	r6, r6, lsr #16
		orr	r6, r6, r7, lsl #16
		stmia	r0!, {r3 - r6}
		subs	ip, ip, #16
		bpl	.cfu_2cpy8lp
.cfu_2rem8lp:	tst	ip, #8
		movne	r3, r7, lsr #16
		ldmneia	r1!, {r4, r7}			@ Shouldn't fault
		orrne	r3, r3, r4, lsl #16
		movne	r4, r4, lsr #16
		orrne	r4, r4, r7, lsl #16
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, lsr #16
USER(		ldrnet	r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, lsl #16
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.cfu_2fupi
.cfu_2nowords:	mov	r3, r7, lsr #16
		teq	ip, #0
		beq	.cfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
		movge	r3, r3, lsr #8
		strgeb	r3, [r0], #1
USER(		ldrgtbt	r3, [r1], #0)			@ May fault
		strgtb	r3, [r0], #1
		b	.cfu_finished

.cfu_3fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.cfu_3nowords
		mov	r3, r7, lsr #24
USER(		ldrt	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, lsl #8
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.cfu_3fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.cfu_3rem8lp

.cfu_3cpy8lp:	mov	r3, r7, lsr #24
		ldmia	r1!, {r4 - r7}			@ Shouldn't fault
		orr	r3, r3, r4, lsl #8
		mov	r4, r4, lsr #24
		orr	r4, r4, r5, lsl #8
		mov	r5, r5, lsr #24
		orr	r5, r5, r6, lsl #8
		mov	r6, r6, lsr #24
		orr	r6, r6, r7, lsl #8
		stmia	r0!, {r3 - r6}
		subs	ip, ip, #16
		bpl	.cfu_3cpy8lp
.cfu_3rem8lp:	tst	ip, #8
		movne	r3, r7, lsr #24
		ldmneia	r1!, {r4, r7}			@ Shouldn't fault
		orrne	r3, r3, r4, lsl #8
		movne	r4, r4, lsr #24
		orrne	r4, r4, r7, lsl #8
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, lsr #24
USER(		ldrnet	r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, lsl #8
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.cfu_3fupi
.cfu_3nowords:	mov	r3, r7, lsr #24
		teq	ip, #0
		beq	.cfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
USER(		ldrget	r3, [r1], #0)			@ May fault
		strgeb	r3, [r0], #1
		movgt	r3, r3, lsr #8
		strgtb	r3, [r0], #1
		b	.cfu_finished

		.section .fixup,"ax"
		.align	0
		/* We took an exception.  Zero out the buffer and pretend no
		   data was copied.  */
9001:		ldr	r0, [sp], #4
		ldr	r1, [sp]
		teq	r1, #0
		blne	SYMBOL_NAME(__memzero)
		LOADREGS(fd,sp!, {r0, r4 - r7, pc})
		.previous

/* Prototype: int __arch_clear_user(void *addr, size_t sz)
 * Purpose  : clear some user memory
 * Params   : addr - user memory address to clear
 *          : sz   - number of bytes to clear
 * Returns  : number of bytes NOT cleared
 */
ENTRY(__arch_clear_user)
		stmfd	sp!, {r1, lr}
		mov	r2, #0
		cmp	r1, #4
		blt	2f
		ands	ip, r0, #3
		beq	1f
		cmp	ip, #2
USER(		strbt	r2, [r0], #1)
USER(		strlebt	r2, [r0], #1)
USER(		strltbt	r2, [r0], #1)
		rsb	ip, ip, #4
		sub	r1, r1, ip		@  7  6  5  4  3  2  1
1:		subs	r1, r1, #8		@ -1 -2 -3 -4 -5 -6 -7
USER(		strplt	r2, [r0], #4)
USER(		strplt	r2, [r0], #4)
		bpl	1b
		adds	r1, r1, #4		@  3  2  1  0 -1 -2 -3
USER(		strplt	r2, [r0], #4)
2:		tst	r1, #2			@ 1x 1x 0x 0x 1x 1x 0x
USER(		strnebt	r2, [r0], #1)
USER(		strnebt	r2, [r0], #1)
		tst	r1, #1			@ x1 x0 x1 x0 x1 x0 x1
USER(		strnebt	r2, [r0], #1)
		mov	r0, #0
		LOADREGS(fd,sp!, {r1, pc})

		.section .fixup,"ax"
		.align	0
9001:		LOADREGS(fd,sp!, {r0, pc})
		.previous

/* Prototype: unsigned long __arch_strnlen_user(const char *str, long n)
 * Purpose  : get length of a string in user memory
 * Params   : str - address of string in user memory
 * Returns  : length of string *including terminator*
 *	      or zero on exception, or n + 1 if too long
 */
ENTRY(__arch_strnlen_user)
		stmfd	sp!, {lr}
		mov	r2, r0
1:
USER(		ldrbt	r3, [r0], #1)
		teq	r3, #0
		beq	2f
		subs	r1, r1, #1
		bne	1b
		add	r0, r0, #1
2:		sub	r0, r0, r2
		LOADREGS(fd,sp!, {pc})

		.section .fixup,"ax"
		.align	0
9001:		mov	r0, #0
		LOADREGS(fd,sp!,{pc})
		.previous

/* Prototype: size_t __arch_strncpy_from_user(char *dst, char *src, size_t len)
 * Purpose  : copy a string from user memory to kernel memory
 * Params   : dst - kernel memory destination
 *          : src - user memory source
 *          : len - maximum length of string
 * Returns  : number of characters copied
 */
ENTRY(__arch_strncpy_from_user)
		stmfd	sp!, {lr}
		add	ip, r1, #1
1:		subs	r2, r2, #1
		bmi	2f
USER(		ldrbt	r3, [r1], #1)
		strb	r3, [r0], #1
		teq	r3, #0
		bne	1b
		sub	r0, r1, ip
		LOADREGS(fd, sp!, {pc})
2:		sub	ip, ip, #1
		sub	r0, r1, ip
		LOADREGS(fd, sp!, {pc})

		.section .fixup,"ax"
		.align	0
9001:		mov	ip, #0
1:		strb	ip, [r0], #1
		subs	r2, r2, #1
		bpl	1b
		mov	r0, #-EFAULT
		LOADREGS(fd, sp!, {pc})
		.previous

		.align