1
2
3
4
5
6 #include <linux/linkage.h>
7 #include <linux/const.h>
8 #include <asm/assembler.h>
9 #include <asm/page.h>
10 #include <asm/cpufeature.h>
11 #include <asm/alternative.h>
12
13
14
15
16
17
18
19
20 ENTRY(copy_page)
21 alternative_if ARM64_HAS_NO_HW_PREFETCH
22
23 prfm pldl1strm, [x1, #128]
24 prfm pldl1strm, [x1, #256]
25 prfm pldl1strm, [x1, #384]
26 alternative_else_nop_endif
27
28 ldp x2, x3, [x1]
29 ldp x4, x5, [x1, #16]
30 ldp x6, x7, [x1, #32]
31 ldp x8, x9, [x1, #48]
32 ldp x10, x11, [x1, #64]
33 ldp x12, x13, [x1, #80]
34 ldp x14, x15, [x1, #96]
35 ldp x16, x17, [x1, #112]
36
37 mov x18, #(PAGE_SIZE - 128)
38 add x1, x1, #128
39 1:
40 subs x18, x18, #128
41
42 alternative_if ARM64_HAS_NO_HW_PREFETCH
43 prfm pldl1strm, [x1, #384]
44 alternative_else_nop_endif
45
46 stnp x2, x3, [x0]
47 ldp x2, x3, [x1]
48 stnp x4, x5, [x0, #16]
49 ldp x4, x5, [x1, #16]
50 stnp x6, x7, [x0, #32]
51 ldp x6, x7, [x1, #32]
52 stnp x8, x9, [x0, #48]
53 ldp x8, x9, [x1, #48]
54 stnp x10, x11, [x0, #64]
55 ldp x10, x11, [x1, #64]
56 stnp x12, x13, [x0, #80]
57 ldp x12, x13, [x1, #80]
58 stnp x14, x15, [x0, #96]
59 ldp x14, x15, [x1, #96]
60 stnp x16, x17, [x0, #112]
61 ldp x16, x17, [x1, #112]
62
63 add x0, x0, #128
64 add x1, x1, #128
65
66 b.gt 1b
67
68 stnp x2, x3, [x0]
69 stnp x4, x5, [x0, #16]
70 stnp x6, x7, [x0, #32]
71 stnp x8, x9, [x0, #48]
72 stnp x10, x11, [x0, #64]
73 stnp x12, x13, [x0, #80]
74 stnp x14, x15, [x0, #96]
75 stnp x16, x17, [x0, #112]
76
77 ret
78 ENDPROC(copy_page)
79 EXPORT_SYMBOL(copy_page)