1
2
3
4
5
6
7
8
9
10
11 #include <linux/linkage.h>
12
13
14
15
16
17
18
19
20 !
21 ! GHIJ KLMN OPQR --> ...G HIJK LMNO PQR.
22 !
23
24 ! Size is 16 or greater, and may have trailing bytes
25
26 .balign 32
27 .Lcase1:
28 ! Read a long word and write a long word at once
29 ! At the start of each iteration, r7 contains last long load
30 add #-1,r5 ! 79 EX
31 mov r4,r2 ! 5 MT (0 cycles latency)
32
33 mov.l @(r0,r5),r7 ! 21 LS (2 cycles latency)
34 add #-4,r5 ! 50 EX
35
36 add #7,r2 ! 79 EX
37 !
38 #ifdef CONFIG_CPU_LITTLE_ENDIAN
39 ! 6 cycles, 4 bytes per iteration
40 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! NMLK
41 mov r7, r3 ! 5 MT (latency=0) ! RQPO
42
43 cmp/hi r2,r0 ! 57 MT
44 shll16 r3 ! 103 EX
45
46 mov r1,r6 ! 5 MT (latency=0)
47 shll8 r3 ! 102 EX ! Oxxx
48
49 shlr8 r6 ! 106 EX ! xNML
50 mov r1, r7 ! 5 MT (latency=0)
51
52 or r6,r3 ! 82 EX ! ONML
53 bt/s 3b ! 109 BR
54
55 mov.l r3,@-r0 ! 30 LS
56 #else
57 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! KLMN
58 mov r7,r3 ! 5 MT (latency=0) ! OPQR
59
60 cmp/hi r2,r0 ! 57 MT
61 shlr16 r3 ! 107 EX
62
63 shlr8 r3 ! 106 EX ! xxxO
64 mov r1,r6 ! 5 MT (latency=0)
65
66 shll8 r6 ! 102 EX ! LMNx
67 mov r1,r7 ! 5 MT (latency=0)
68
69 or r6,r3 ! 82 EX ! LMNO
70 bt/s 3b ! 109 BR
71
72 mov.l r3,@-r0 ! 30 LS
73 #endif
74 ! Finally, copy a byte at once, if necessary
75
76 add #4,r5 ! 50 EX
77 cmp/eq r4,r0 ! 54 MT
78
79 add #-6,r2 ! 50 EX
80 bt 9f ! 109 BR
81
82 8: cmp/hi r2,r0 ! 57 MT
83 mov.b @(r0,r5),r1 ! 20 LS (latency=2)
84
85 bt/s 8b ! 109 BR
86
87 mov.b r1,@-r0 ! 29 LS
88
89 9: rts
90 nop
91
92
93 !
94 ! GHIJ KLMN OPQR --> .GHI JKLM NOPQ R...
95 !
96
97 ! Size is 16 or greater, and may have trailing bytes
98
99 .balign 32
100 .Lcase3:
101 ! Read a long word and write a long word at once
102 ! At the start of each iteration, r7 contains last long load
103 add #-3,r5 ! 79 EX
104 mov r4,r2 ! 5 MT (0 cycles latency)
105
106 mov.l @(r0,r5),r7 ! 21 LS (2 cycles latency)
107 add #-4,r5 ! 50 EX
108
109 add #7,r2 ! 79 EX
110 !
111 #ifdef CONFIG_CPU_LITTLE_ENDIAN
112 ! 6 cycles, 4 bytes per iteration
113 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! NMLK
114 mov r7, r3 ! 5 MT (latency=0) ! RQPO
115
116 cmp/hi r2,r0 ! 57 MT
117 shll8 r3 ! 102 EX ! QPOx
118
119 mov r1,r6 ! 5 MT (latency=0)
120 shlr16 r6 ! 107 EX
121
122 shlr8 r6 ! 106 EX ! xxxN
123 mov r1, r7 ! 5 MT (latency=0)
124
125 or r6,r3 ! 82 EX ! QPON
126 bt/s 3b ! 109 BR
127
128 mov.l r3,@-r0 ! 30 LS
129 #else
130 3: mov r7,r3 ! OPQR
131 shlr8 r3 ! xOPQ
132 mov.l @(r0,r5),r7 ! KLMN
133 mov r7,r6
134 shll16 r6
135 shll8 r6 ! Nxxx
136 or r6,r3 ! NOPQ
137 cmp/hi r2,r0
138 bt/s 3b
139 mov.l r3,@-r0
140 #endif
141
142 ! Finally, copy a byte at once, if necessary
143
144 add #6,r5 ! 50 EX
145 cmp/eq r4,r0 ! 54 MT
146
147 add #-6,r2 ! 50 EX
148 bt 9f ! 109 BR
149
150 8: cmp/hi r2,r0 ! 57 MT
151 mov.b @(r0,r5),r1 ! 20 LS (latency=2)
152
153 bt/s 8b ! 109 BR
154
155 mov.b r1,@-r0 ! 29 LS
156
157 9: rts
158 nop
159
160 ENTRY(memcpy)
161
162 ! Calculate the invariants which will be used in the remainder
163 ! of the code:
164 !
165 ! r4 --> [ ... ] DST [ ... ] SRC
166 ! [ ... ] [ ... ]
167 ! : :
168 ! r0 --> [ ... ] r0+r5 --> [ ... ]
169 !
170 !
171
172 ! Short circuit the common case of src, dst and len being 32 bit aligned
173 ! and test for zero length move
174
175 mov r6, r0 ! 5 MT (0 cycle latency)
176 or r4, r0 ! 82 EX
177
178 or r5, r0 ! 82 EX
179 tst r6, r6 ! 86 MT
180
181 bt/s 99f ! 111 BR (zero len)
182 tst #3, r0 ! 87 MT
183
184 mov r4, r0 ! 5 MT (0 cycle latency)
185 add r6, r0 ! 49 EX
186
187 mov #16, r1 ! 6 EX
188 bt/s .Lcase00 ! 111 BR (aligned)
189
190 sub r4, r5 ! 75 EX
191
192 ! Arguments are not nicely long word aligned or zero len.
193 ! Check for small copies, and if so do a simple byte at a time copy.
194 !
195 ! Deciding on an exact value of 'small' is not easy, as the point at which
196 ! using the optimised routines become worthwhile varies (these are the
197 ! cycle counts for differnet sizes using byte-at-a-time vs. optimised):
198 ! size byte-at-time long word byte
199 ! 16 42 39-40 46-50 50-55
200 ! 24 58 43-44 54-58 62-67
201 ! 36 82 49-50 66-70 80-85
202 ! However the penalty for getting it 'wrong' is much higher for long word
203 ! aligned data (and this is more common), so use a value of 16.
204
205 cmp/gt r6,r1 ! 56 MT
206
207 add #-1,r5 ! 50 EX
208 bf/s 6f ! 108 BR (not small)
209
210 mov r5, r3 ! 5 MT (latency=0)
211 shlr r6 ! 104 EX
212
213 mov.b @(r0,r5),r1 ! 20 LS (latency=2)
214 bf/s 4f ! 111 BR
215
216 add #-1,r3 ! 50 EX
217 tst r6, r6 ! 86 MT
218
219 bt/s 98f ! 110 BR
220 mov.b r1,@-r0 ! 29 LS
221
222 ! 4 cycles, 2 bytes per iteration
223 3: mov.b @(r0,r5),r1 ! 20 LS (latency=2)
224
225 4: mov.b @(r0,r3),r2 ! 20 LS (latency=2)
226 dt r6 ! 67 EX
227
228 mov.b r1,@-r0 ! 29 LS
229 bf/s 3b ! 111 BR
230
231 mov.b r2,@-r0 ! 29 LS
232 98:
233 rts
234 nop
235
236 99: rts
237 mov r4, r0
238
239 ! Size is not small, so its worthwhile looking for optimisations.
240 ! First align destination to a long word boundary.
241 !
242 ! r5 = normal value -1
243
244 6: tst #3, r0 ! 87 MT
245 mov #3, r3 ! 6 EX
246
247 bt/s 2f ! 111 BR
248 and r0,r3 ! 78 EX
249
250 ! 3 cycles, 1 byte per iteration
251 1: dt r3 ! 67 EX
252 mov.b @(r0,r5),r1 ! 19 LS (latency=2)
253
254 add #-1, r6 ! 79 EX
255 bf/s 1b ! 109 BR
256
257 mov.b r1,@-r0 ! 28 LS
258
259 2: add #1, r5 ! 79 EX
260
261 ! Now select the appropriate bulk transfer code based on relative
262 ! alignment of src and dst.
263
264 mov r0, r3 ! 5 MT (latency=0)
265
266 mov r5, r0 ! 5 MT (latency=0)
267 tst #1, r0 ! 87 MT
268
269 bf/s 1f ! 111 BR
270 mov #64, r7 ! 6 EX
271
272 ! bit 0 clear
273
274 cmp/ge r7, r6 ! 55 MT
275
276 bt/s 2f ! 111 BR
277 tst #2, r0 ! 87 MT
278
279 ! small
280 bt/s .Lcase0
281 mov r3, r0
282
283 bra .Lcase2
284 nop
285
286 ! big
287 2: bt/s .Lcase0b
288 mov r3, r0
289
290 bra .Lcase2b
291 nop
292
293 ! bit 0 set
294 1: tst #2, r0 ! 87 MT
295
296 bt/s .Lcase1
297 mov r3, r0
298
299 bra .Lcase3
300 nop
301
302
303 !
304 ! GHIJ KLMN OPQR --> GHIJ KLMN OPQR
305 !
306
307 ! src, dst and size are all long word aligned
308 ! size is non-zero
309
310 .balign 32
311 .Lcase00:
312 mov #64, r1 ! 6 EX
313 mov r5, r3 ! 5 MT (latency=0)
314
315 cmp/gt r6, r1 ! 56 MT
316 add #-4, r5 ! 50 EX
317
318 bf .Lcase00b ! 108 BR (big loop)
319 shlr2 r6 ! 105 EX
320
321 shlr r6 ! 104 EX
322 mov.l @(r0, r5), r1 ! 21 LS (latency=2)
323
324 bf/s 4f ! 111 BR
325 add #-8, r3 ! 50 EX
326
327 tst r6, r6 ! 86 MT
328 bt/s 5f ! 110 BR
329
330 mov.l r1,@-r0 ! 30 LS
331
332 ! 4 cycles, 2 long words per iteration
333 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
334
335 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
336 dt r6 ! 67 EX
337
338 mov.l r1, @-r0 ! 30 LS
339 bf/s 3b ! 109 BR
340
341 mov.l r2, @-r0 ! 30 LS
342
343 5: rts
344 nop
345
346
347 ! Size is 16 or greater and less than 64, but may have trailing bytes
348
349 .balign 32
350 .Lcase0:
351 add #-4, r5 ! 50 EX
352 mov r4, r7 ! 5 MT (latency=0)
353
354 mov.l @(r0, r5), r1 ! 21 LS (latency=2)
355 mov #4, r2 ! 6 EX
356
357 add #11, r7 ! 50 EX
358 tst r2, r6 ! 86 MT
359
360 mov r5, r3 ! 5 MT (latency=0)
361 bt/s 4f ! 111 BR
362
363 add #-4, r3 ! 50 EX
364 mov.l r1,@-r0 ! 30 LS
365
366 ! 4 cycles, 2 long words per iteration
367 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
368
369 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
370 cmp/hi r7, r0
371
372 mov.l r1, @-r0 ! 30 LS
373 bt/s 3b ! 109 BR
374
375 mov.l r2, @-r0 ! 30 LS
376
377 ! Copy the final 0-3 bytes
378
379 add #3,r5 ! 50 EX
380
381 cmp/eq r0, r4 ! 54 MT
382 add #-10, r7 ! 50 EX
383
384 bt 9f ! 110 BR
385
386 ! 3 cycles, 1 byte per iteration
387 1: mov.b @(r0,r5),r1 ! 19 LS
388 cmp/hi r7,r0 ! 57 MT
389
390 bt/s 1b ! 111 BR
391 mov.b r1,@-r0 ! 28 LS
392
393 9: rts
394 nop
395
396 ! Size is at least 64 bytes, so will be going round the big loop at least once.
397 !
398 ! r2 = rounded up r4
399 ! r3 = rounded down r0
400
401 .balign 32
402 .Lcase0b:
403 add #-4, r5 ! 50 EX
404
405 .Lcase00b:
406 mov r0, r3 ! 5 MT (latency=0)
407 mov #(~0x1f), r1 ! 6 EX
408
409 and r1, r3 ! 78 EX
410 mov r4, r2 ! 5 MT (latency=0)
411
412 cmp/eq r3, r0 ! 54 MT
413 add #0x1f, r2 ! 50 EX
414
415 bt/s 1f ! 110 BR
416 and r1, r2 ! 78 EX
417
418 ! copy initial words until cache line aligned
419
420 mov.l @(r0, r5), r1 ! 21 LS (latency=2)
421 tst #4, r0 ! 87 MT
422
423 mov r5, r6 ! 5 MT (latency=0)
424 add #-4, r6 ! 50 EX
425
426 bt/s 4f ! 111 BR
427 add #8, r3 ! 50 EX
428
429 tst #0x18, r0 ! 87 MT
430
431 bt/s 1f ! 109 BR
432 mov.l r1,@-r0 ! 30 LS
433
434 ! 4 cycles, 2 long words per iteration
435 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
436
437 4: mov.l @(r0, r6), r7 ! 21 LS (latency=2)
438 cmp/eq r3, r0 ! 54 MT
439
440 mov.l r1, @-r0 ! 30 LS
441 bf/s 3b ! 109 BR
442
443 mov.l r7, @-r0 ! 30 LS
444
445 ! Copy the cache line aligned blocks
446 !
447 ! In use: r0, r2, r4, r5
448 ! Scratch: r1, r3, r6, r7
449 !
450 ! We could do this with the four scratch registers, but if src
451 ! and dest hit the same cache line, this will thrash, so make
452 ! use of additional registers.
453 !
454 ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
455 ! r5: src (was r0+r5)
456 ! r1: dest (was r0)
457 ! this can be reversed at the end, so we don't need to save any extra
458 ! state.
459 !
460 1: mov.l r8, @-r15 ! 30 LS
461 add r0, r5 ! 49 EX
462
463 mov.l r9, @-r15 ! 30 LS
464 mov r0, r1 ! 5 MT (latency=0)
465
466 mov.l r10, @-r15 ! 30 LS
467 add #-0x1c, r5 ! 50 EX
468
469 mov.l r11, @-r15 ! 30 LS
470
471 ! 16 cycles, 32 bytes per iteration
472 2: mov.l @(0x00,r5),r0 ! 18 LS (latency=2)
473 add #-0x20, r1 ! 50 EX
474 mov.l @(0x04,r5),r3 ! 18 LS (latency=2)
475 mov.l @(0x08,r5),r6 ! 18 LS (latency=2)
476 mov.l @(0x0c,r5),r7 ! 18 LS (latency=2)
477 mov.l @(0x10,r5),r8 ! 18 LS (latency=2)
478 mov.l @(0x14,r5),r9 ! 18 LS (latency=2)
479 mov.l @(0x18,r5),r10 ! 18 LS (latency=2)
480 mov.l @(0x1c,r5),r11 ! 18 LS (latency=2)
481 movca.l r0,@r1 ! 40 LS (latency=3-7)
482 mov.l r3,@(0x04,r1) ! 33 LS
483 mov.l r6,@(0x08,r1) ! 33 LS
484 mov.l r7,@(0x0c,r1) ! 33 LS
485
486 mov.l r8,@(0x10,r1) ! 33 LS
487 add #-0x20, r5 ! 50 EX
488
489 mov.l r9,@(0x14,r1) ! 33 LS
490 cmp/eq r2,r1 ! 54 MT
491
492 mov.l r10,@(0x18,r1) ! 33 LS
493 bf/s 2b ! 109 BR
494
495 mov.l r11,@(0x1c,r1) ! 33 LS
496
497 mov r1, r0 ! 5 MT (latency=0)
498
499 mov.l @r15+, r11 ! 15 LS
500 sub r1, r5 ! 75 EX
501
502 mov.l @r15+, r10 ! 15 LS
503 cmp/eq r4, r0 ! 54 MT
504
505 bf/s 1f ! 109 BR
506 mov.l @r15+, r9 ! 15 LS
507
508 rts
509 1: mov.l @r15+, r8 ! 15 LS
510 sub r4, r1 ! 75 EX (len remaining)
511
512 ! number of trailing bytes is non-zero
513 !
514 ! invariants restored (r5 already decremented by 4)
515 ! also r1=num bytes remaining
516
517 mov #4, r2 ! 6 EX
518 mov r4, r7 ! 5 MT (latency=0)
519
520 add #0x1c, r5 ! 50 EX (back to -4)
521 cmp/hs r2, r1 ! 58 MT
522
523 bf/s 5f ! 108 BR
524 add #11, r7 ! 50 EX
525
526 mov.l @(r0, r5), r6 ! 21 LS (latency=2)
527 tst r2, r1 ! 86 MT
528
529 mov r5, r3 ! 5 MT (latency=0)
530 bt/s 4f ! 111 BR
531
532 add #-4, r3 ! 50 EX
533 cmp/hs r2, r1 ! 58 MT
534
535 bt/s 5f ! 111 BR
536 mov.l r6,@-r0 ! 30 LS
537
538 ! 4 cycles, 2 long words per iteration
539 3: mov.l @(r0, r5), r6 ! 21 LS (latency=2)
540
541 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
542 cmp/hi r7, r0
543
544 mov.l r6, @-r0 ! 30 LS
545 bt/s 3b ! 109 BR
546
547 mov.l r2, @-r0 ! 30 LS
548
549 ! Copy the final 0-3 bytes
550
551 5: cmp/eq r0, r4 ! 54 MT
552 add #-10, r7 ! 50 EX
553
554 bt 9f ! 110 BR
555 add #3,r5 ! 50 EX
556
557 ! 3 cycles, 1 byte per iteration
558 1: mov.b @(r0,r5),r1 ! 19 LS
559 cmp/hi r7,r0 ! 57 MT
560
561 bt/s 1b ! 111 BR
562 mov.b r1,@-r0 ! 28 LS
563
564 9: rts
565 nop
566
567 !
568 ! GHIJ KLMN OPQR --> ..GH IJKL MNOP QR..
569 !
570
571 .balign 32
572 .Lcase2:
573 ! Size is 16 or greater and less then 64, but may have trailing bytes
574
575 2: mov r5, r6 ! 5 MT (latency=0)
576 add #-2,r5 ! 50 EX
577
578 mov r4,r2 ! 5 MT (latency=0)
579 add #-4,r6 ! 50 EX
580
581 add #7,r2 ! 50 EX
582 3: mov.w @(r0,r5),r1 ! 20 LS (latency=2)
583
584 mov.w @(r0,r6),r3 ! 20 LS (latency=2)
585 cmp/hi r2,r0 ! 57 MT
586
587 mov.w r1,@-r0 ! 29 LS
588 bt/s 3b ! 111 BR
589
590 mov.w r3,@-r0 ! 29 LS
591
592 bra 10f
593 nop
594
595
596 .balign 32
597 .Lcase2b:
598 ! Size is at least 64 bytes, so will be going round the big loop at least once.
599 !
600 ! r2 = rounded up r4
601 ! r3 = rounded down r0
602
603 mov r0, r3 ! 5 MT (latency=0)
604 mov #(~0x1f), r1 ! 6 EX
605
606 and r1, r3 ! 78 EX
607 mov r4, r2 ! 5 MT (latency=0)
608
609 cmp/eq r3, r0 ! 54 MT
610 add #0x1f, r2 ! 50 EX
611
612 add #-2, r5 ! 50 EX
613 bt/s 1f ! 110 BR
614 and r1, r2 ! 78 EX
615
616 ! Copy a short word one at a time until we are cache line aligned
617 ! Normal values: r0, r2, r3, r4
618 ! Unused: r1, r6, r7
619 ! Mod: r5 (=r5-2)
620 !
621 add #2, r3 ! 50 EX
622
623 2: mov.w @(r0,r5),r1 ! 20 LS (latency=2)
624 cmp/eq r3,r0 ! 54 MT
625
626 bf/s 2b ! 111 BR
627
628 mov.w r1,@-r0 ! 29 LS
629
630 ! Copy the cache line aligned blocks
631 !
632 ! In use: r0, r2, r4, r5 (=r5-2)
633 ! Scratch: r1, r3, r6, r7
634 !
635 ! We could do this with the four scratch registers, but if src
636 ! and dest hit the same cache line, this will thrash, so make
637 ! use of additional registers.
638 !
639 ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
640 ! r5: src (was r0+r5)
641 ! r1: dest (was r0)
642 ! this can be reversed at the end, so we don't need to save any extra
643 ! state.
644 !
645 1: mov.l r8, @-r15 ! 30 LS
646 add r0, r5 ! 49 EX
647
648 mov.l r9, @-r15 ! 30 LS
649 mov r0, r1 ! 5 MT (latency=0)
650
651 mov.l r10, @-r15 ! 30 LS
652 add #-0x1e, r5 ! 50 EX
653
654 mov.l r11, @-r15 ! 30 LS
655
656 mov.l r12, @-r15 ! 30 LS
657
658 ! 17 cycles, 32 bytes per iteration
659 #ifdef CONFIG_CPU_LITTLE_ENDIAN
660 2: mov.w @r5+, r0 ! 14 LS (latency=2) ..JI
661 add #-0x20, r1 ! 50 EX
662
663 mov.l @r5+, r3 ! 15 LS (latency=2) NMLK
664
665 mov.l @r5+, r6 ! 15 LS (latency=2) RQPO
666 shll16 r0 ! 103 EX JI..
667
668 mov.l @r5+, r7 ! 15 LS (latency=2)
669 xtrct r3, r0 ! 48 EX LKJI
670
671 mov.l @r5+, r8 ! 15 LS (latency=2)
672 xtrct r6, r3 ! 48 EX PONM
673
674 mov.l @r5+, r9 ! 15 LS (latency=2)
675 xtrct r7, r6 ! 48 EX
676
677 mov.l @r5+, r10 ! 15 LS (latency=2)
678 xtrct r8, r7 ! 48 EX
679
680 mov.l @r5+, r11 ! 15 LS (latency=2)
681 xtrct r9, r8 ! 48 EX
682
683 mov.w @r5+, r12 ! 15 LS (latency=2)
684 xtrct r10, r9 ! 48 EX
685
686 movca.l r0,@r1 ! 40 LS (latency=3-7)
687 xtrct r11, r10 ! 48 EX
688
689 mov.l r3, @(0x04,r1) ! 33 LS
690 xtrct r12, r11 ! 48 EX
691
692 mov.l r6, @(0x08,r1) ! 33 LS
693
694 mov.l r7, @(0x0c,r1) ! 33 LS
695
696 mov.l r8, @(0x10,r1) ! 33 LS
697 add #-0x40, r5 ! 50 EX
698
699 mov.l r9, @(0x14,r1) ! 33 LS
700 cmp/eq r2,r1 ! 54 MT
701
702 mov.l r10, @(0x18,r1) ! 33 LS
703 bf/s 2b ! 109 BR
704
705 mov.l r11, @(0x1c,r1) ! 33 LS
706 #else
707 2: mov.w @(0x1e,r5), r0 ! 17 LS (latency=2)
708 add #-2, r5 ! 50 EX
709
710 mov.l @(0x1c,r5), r3 ! 18 LS (latency=2)
711 add #-4, r1 ! 50 EX
712
713 mov.l @(0x18,r5), r6 ! 18 LS (latency=2)
714 shll16 r0 ! 103 EX
715
716 mov.l @(0x14,r5), r7 ! 18 LS (latency=2)
717 xtrct r3, r0 ! 48 EX
718
719 mov.l @(0x10,r5), r8 ! 18 LS (latency=2)
720 xtrct r6, r3 ! 48 EX
721
722 mov.l @(0x0c,r5), r9 ! 18 LS (latency=2)
723 xtrct r7, r6 ! 48 EX
724
725 mov.l @(0x08,r5), r10 ! 18 LS (latency=2)
726 xtrct r8, r7 ! 48 EX
727
728 mov.l @(0x04,r5), r11 ! 18 LS (latency=2)
729 xtrct r9, r8 ! 48 EX
730
731 mov.l @(0x00,r5), r12 ! 18 LS (latency=2)
732 xtrct r10, r9 ! 48 EX
733
734 movca.l r0,@r1 ! 40 LS (latency=3-7)
735 add #-0x1c, r1 ! 50 EX
736
737 mov.l r3, @(0x18,r1) ! 33 LS
738 xtrct r11, r10 ! 48 EX
739
740 mov.l r6, @(0x14,r1) ! 33 LS
741 xtrct r12, r11 ! 48 EX
742
743 mov.l r7, @(0x10,r1) ! 33 LS
744
745 mov.l r8, @(0x0c,r1) ! 33 LS
746 add #-0x1e, r5 ! 50 EX
747
748 mov.l r9, @(0x08,r1) ! 33 LS
749 cmp/eq r2,r1 ! 54 MT
750
751 mov.l r10, @(0x04,r1) ! 33 LS
752 bf/s 2b ! 109 BR
753
754 mov.l r11, @(0x00,r1) ! 33 LS
755 #endif
756
757 mov.l @r15+, r12
758 mov r1, r0 ! 5 MT (latency=0)
759
760 mov.l @r15+, r11 ! 15 LS
761 sub r1, r5 ! 75 EX
762
763 mov.l @r15+, r10 ! 15 LS
764 cmp/eq r4, r0 ! 54 MT
765
766 bf/s 1f ! 109 BR
767 mov.l @r15+, r9 ! 15 LS
768
769 rts
770 1: mov.l @r15+, r8 ! 15 LS
771
772 add #0x1e, r5 ! 50 EX
773
774 ! Finish off a short word at a time
775 ! r5 must be invariant - 2
776 10: mov r4,r2 ! 5 MT (latency=0)
777 add #1,r2 ! 50 EX
778
779 cmp/hi r2, r0 ! 57 MT
780 bf/s 1f ! 109 BR
781
782 add #2, r2 ! 50 EX
783
784 3: mov.w @(r0,r5),r1 ! 20 LS
785 cmp/hi r2,r0 ! 57 MT
786
787 bt/s 3b ! 109 BR
788
789 mov.w r1,@-r0 ! 29 LS
790 1:
791
792 !
793 ! Finally, copy the last byte if necessary
794 cmp/eq r4,r0 ! 54 MT
795 bt/s 9b
796 add #1,r5
797 mov.b @(r0,r5),r1
798 rts
799 mov.b r1,@-r0
800