1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/linkage.h>
13 #include <asm/mcpm.h>
14 #include <asm/assembler.h>
15
16 #include "vlock.h"
17
18 .if MCPM_SYNC_CLUSTER_CPUS
19 .error "cpus must be the first member of struct mcpm_sync_struct"
20 .endif
21
22 .macro pr_dbg string
23 #if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
24 b 1901f
25 1902: .asciz "CPU"
26 1903: .asciz " cluster"
27 1904: .asciz ": \string"
28 .align
29 1901: adr r0, 1902b
30 bl printascii
31 mov r0, r9
32 bl printhex2
33 adr r0, 1903b
34 bl printascii
35 mov r0, r10
36 bl printhex2
37 adr r0, 1904b
38 bl printascii
39 #endif
40 .endm
41
42 .arm
43 .align
44
45 ENTRY(mcpm_entry_point)
46
47 ARM_BE8(setend be)
48 THUMB( badr r12, 1f )
49 THUMB( bx r12 )
50 THUMB( .thumb )
51 1:
52 mrc p15, 0, r0, c0, c0, 5 @ MPIDR
53 ubfx r9, r0, #0, #8 @ r9 = cpu
54 ubfx r10, r0, #8, #8 @ r10 = cluster
55 mov r3, #MAX_CPUS_PER_CLUSTER
56 mla r4, r3, r10, r9 @ r4 = canonical CPU index
57 cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
58 blo 2f
59
60
61 1: wfi
62 wfe
63 b 1b
64
65 2: pr_dbg "kernel mcpm_entry_point\n"
66
67
68
69
70
71 adr r5, 3f
72 ldmia r5, {r0, r6, r7, r8, r11}
73 add r0, r5, r0 @ r0 = mcpm_entry_early_pokes
74 add r6, r5, r6 @ r6 = mcpm_entry_vectors
75 ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys
76 add r8, r5, r8 @ r8 = mcpm_sync
77 add r11, r5, r11 @ r11 = first_man_locks
78
79 @ Perform an early poke, if any
80 add r0, r0, r4, lsl #3
81 ldmia r0, {r0, r1}
82 teq r0, #0
83 strne r1, [r0]
84
85 mov r0, #MCPM_SYNC_CLUSTER_SIZE
86 mla r8, r0, r10, r8 @ r8 = sync cluster base
87
88 @ Signal that this CPU is coming UP:
89 mov r0, #CPU_COMING_UP
90 mov r5, #MCPM_SYNC_CPU_SIZE
91 mla r5, r9, r5, r8 @ r5 = sync cpu address
92 strb r0, [r5]
93
94 @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
95 @ state, because there is at least one active CPU (this CPU).
96
97 mov r0, #VLOCK_SIZE
98 mla r11, r0, r10, r11 @ r11 = cluster first man lock
99 mov r0, r11
100 mov r1, r9 @ cpu
101 bl vlock_trylock @ implies DMB
102
103 cmp r0, #0 @ failed to get the lock?
104 bne mcpm_setup_wait @ wait for cluster setup if so
105
106 ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
107 cmp r0, #CLUSTER_UP @ cluster already up?
108 bne mcpm_setup @ if not, set up the cluster
109
110 @ Otherwise, release the first man lock and skip setup:
111 mov r0, r11
112 bl vlock_unlock
113 b mcpm_setup_complete
114
115 mcpm_setup:
116 @ Control dependency implies strb not observable before previous ldrb.
117
118 @ Signal that the cluster is being brought up:
119 mov r0, #INBOUND_COMING_UP
120 strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
121 dmb
122
123 @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
124 @ point onwards will observe INBOUND_COMING_UP and abort.
125
126 @ Wait for any previously-pending cluster teardown operations to abort
127 @ or complete:
128 mcpm_teardown_wait:
129 ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
130 cmp r0, #CLUSTER_GOING_DOWN
131 bne first_man_setup
132 wfe
133 b mcpm_teardown_wait
134
135 first_man_setup:
136 dmb
137
138 @ If the outbound gave up before teardown started, skip cluster setup:
139
140 cmp r0, #CLUSTER_UP
141 beq mcpm_setup_leave
142
143 @ power_up_setup is now responsible for setting up the cluster:
144
145 cmp r7, #0
146 mov r0, #1 @ second (cluster) affinity level
147 blxne r7 @ Call power_up_setup if defined
148 dmb
149
150 mov r0, #CLUSTER_UP
151 strb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
152 dmb
153
154 mcpm_setup_leave:
155 @ Leave the cluster setup critical section:
156
157 mov r0, #INBOUND_NOT_COMING_UP
158 strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
159 dsb st
160 sev
161
162 mov r0, r11
163 bl vlock_unlock @ implies DMB
164 b mcpm_setup_complete
165
166 @ In the contended case, non-first men wait here for cluster setup
167 @ to complete:
168 mcpm_setup_wait:
169 ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
170 cmp r0, #CLUSTER_UP
171 wfene
172 bne mcpm_setup_wait
173 dmb
174
175 mcpm_setup_complete:
176 @ If a platform-specific CPU setup hook is needed, it is
177 @ called from here.
178
179 cmp r7, #0
180 mov r0, #0 @ first (CPU) affinity level
181 blxne r7 @ Call power_up_setup if defined
182 dmb
183
184 @ Mark the CPU as up:
185
186 mov r0, #CPU_UP
187 strb r0, [r5]
188
189 @ Observability order of CPU_UP and opening of the gate does not matter.
190
191 mcpm_entry_gated:
192 ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector
193 cmp r5, #0
194 wfeeq
195 beq mcpm_entry_gated
196 dmb
197
198 pr_dbg "released\n"
199 bx r5
200
201 .align 2
202
203 3: .word mcpm_entry_early_pokes - .
204 .word mcpm_entry_vectors - 3b
205 .word mcpm_power_up_setup_phys - 3b
206 .word mcpm_sync - 3b
207 .word first_man_locks - 3b
208
209 ENDPROC(mcpm_entry_point)
210
211 .bss
212
213 .align CACHE_WRITEBACK_ORDER
214 .type first_man_locks, #object
215 first_man_locks:
216 .space VLOCK_SIZE * MAX_NR_CLUSTERS
217 .align CACHE_WRITEBACK_ORDER
218
219 .type mcpm_entry_vectors, #object
220 ENTRY(mcpm_entry_vectors)
221 .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
222
223 .type mcpm_entry_early_pokes, #object
224 ENTRY(mcpm_entry_early_pokes)
225 .space 8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
226
227 .type mcpm_power_up_setup_phys, #object
228 ENTRY(mcpm_power_up_setup_phys)
229 .space 4 @ set by mcpm_sync_init()