1/*
2 *  linux/boot/head.S
3 *
4 *  Copyright (C) 1991, 1992, 1993  Linus Torvalds
5 */
6
7/*
8 *  head.S contains the 32-bit startup code.
9 *
10 * NOTE!!! Startup happens at absolute address 0x00001000, which is also where
11 * the page directory will exist. The startup code will be overwritten by
12 * the page directory. [According to comments etc elsewhere on a compressed
13 * kernel it will end up at 0x1000 + 1Mb I hope so as I assume this. - AC]
14 *
15 * Page 0 is deliberately kept safe, since System Management Mode code in
16 * laptops may need to access the BIOS data stored there.  This is also
17 * useful for future device drivers that either access the BIOS via VM86
18 * mode.
19 */
20
21/*
22 * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
23 */
24	.code32
25	.text
26
27#include <linux/init.h>
28#include <linux/linkage.h>
29#include <asm/segment.h>
30#include <asm/boot.h>
31#include <asm/msr.h>
32#include <asm/processor-flags.h>
33#include <asm/asm-offsets.h>
34#include <asm/bootparam.h>
35
36	__HEAD
37	.code32
38ENTRY(startup_32)
39	/*
40	 * 32bit entry is 0 and it is ABI so immutable!
41	 * If we come here directly from a bootloader,
42	 * kernel(text+data+bss+brk) ramdisk, zero_page, command line
43	 * all need to be under the 4G limit.
44	 */
45	cld
46	/*
47	 * Test KEEP_SEGMENTS flag to see if the bootloader is asking
48	 * us to not reload segments
49	 */
50	testb $KEEP_SEGMENTS, BP_loadflags(%esi)
51	jnz 1f
52
53	cli
54	movl	$(__BOOT_DS), %eax
55	movl	%eax, %ds
56	movl	%eax, %es
57	movl	%eax, %ss
581:
59
60/*
61 * Calculate the delta between where we were compiled to run
62 * at and where we were actually loaded at.  This can only be done
63 * with a short local call on x86.  Nothing  else will tell us what
64 * address we are running at.  The reserved chunk of the real-mode
65 * data at 0x1e4 (defined as a scratch field) are used as the stack
66 * for this calculation. Only 4 bytes are needed.
67 */
68	leal	(BP_scratch+4)(%esi), %esp
69	call	1f
701:	popl	%ebp
71	subl	$1b, %ebp
72
73/* setup a stack and make sure cpu supports long mode. */
74	movl	$boot_stack_end, %eax
75	addl	%ebp, %eax
76	movl	%eax, %esp
77
78	call	verify_cpu
79	testl	%eax, %eax
80	jnz	no_longmode
81
82/*
83 * Compute the delta between where we were compiled to run at
84 * and where the code will actually run at.
85 *
86 * %ebp contains the address we are loaded at by the boot loader and %ebx
87 * contains the address where we should move the kernel image temporarily
88 * for safe in-place decompression.
89 */
90
91#ifdef CONFIG_RELOCATABLE
92	movl	%ebp, %ebx
93	movl	BP_kernel_alignment(%esi), %eax
94	decl	%eax
95	addl	%eax, %ebx
96	notl	%eax
97	andl	%eax, %ebx
98	cmpl	$LOAD_PHYSICAL_ADDR, %ebx
99	jge	1f
100#endif
101	movl	$LOAD_PHYSICAL_ADDR, %ebx
1021:
103
104	/* Target address to relocate to for decompression */
105	addl	$z_extract_offset, %ebx
106
107/*
108 * Prepare for entering 64 bit mode
109 */
110
111	/* Load new GDT with the 64bit segments using 32bit descriptor */
112	leal	gdt(%ebp), %eax
113	movl	%eax, gdt+2(%ebp)
114	lgdt	gdt(%ebp)
115
116	/* Enable PAE mode */
117	movl	%cr4, %eax
118	orl	$X86_CR4_PAE, %eax
119	movl	%eax, %cr4
120
121 /*
122  * Build early 4G boot pagetable
123  */
124	/* Initialize Page tables to 0 */
125	leal	pgtable(%ebx), %edi
126	xorl	%eax, %eax
127	movl	$((4096*6)/4), %ecx
128	rep	stosl
129
130	/* Build Level 4 */
131	leal	pgtable + 0(%ebx), %edi
132	leal	0x1007 (%edi), %eax
133	movl	%eax, 0(%edi)
134
135	/* Build Level 3 */
136	leal	pgtable + 0x1000(%ebx), %edi
137	leal	0x1007(%edi), %eax
138	movl	$4, %ecx
1391:	movl	%eax, 0x00(%edi)
140	addl	$0x00001000, %eax
141	addl	$8, %edi
142	decl	%ecx
143	jnz	1b
144
145	/* Build Level 2 */
146	leal	pgtable + 0x2000(%ebx), %edi
147	movl	$0x00000183, %eax
148	movl	$2048, %ecx
1491:	movl	%eax, 0(%edi)
150	addl	$0x00200000, %eax
151	addl	$8, %edi
152	decl	%ecx
153	jnz	1b
154
155	/* Enable the boot page tables */
156	leal	pgtable(%ebx), %eax
157	movl	%eax, %cr3
158
159	/* Enable Long mode in EFER (Extended Feature Enable Register) */
160	movl	$MSR_EFER, %ecx
161	rdmsr
162	btsl	$_EFER_LME, %eax
163	wrmsr
164
165	/* After gdt is loaded */
166	xorl	%eax, %eax
167	lldt	%ax
168	movl    $__BOOT_TSS, %eax
169	ltr	%ax
170
171	/*
172	 * Setup for the jump to 64bit mode
173	 *
174	 * When the jump is performend we will be in long mode but
175	 * in 32bit compatibility mode with EFER.LME = 1, CS.L = 0, CS.D = 1
176	 * (and in turn EFER.LMA = 1).	To jump into 64bit mode we use
177	 * the new gdt/idt that has __KERNEL_CS with CS.L = 1.
178	 * We place all of the values on our mini stack so lret can
179	 * used to perform that far jump.
180	 */
181	pushl	$__KERNEL_CS
182	leal	startup_64(%ebp), %eax
183#ifdef CONFIG_EFI_MIXED
184	movl	efi32_config(%ebp), %ebx
185	cmp	$0, %ebx
186	jz	1f
187	leal	handover_entry(%ebp), %eax
1881:
189#endif
190	pushl	%eax
191
192	/* Enter paged protected Mode, activating Long Mode */
193	movl	$(X86_CR0_PG | X86_CR0_PE), %eax /* Enable Paging and Protected mode */
194	movl	%eax, %cr0
195
196	/* Jump from 32bit compatibility mode into 64bit mode. */
197	lret
198ENDPROC(startup_32)
199
200#ifdef CONFIG_EFI_MIXED
201	.org 0x190
202ENTRY(efi32_stub_entry)
203	add	$0x4, %esp		/* Discard return address */
204	popl	%ecx
205	popl	%edx
206	popl	%esi
207
208	leal	(BP_scratch+4)(%esi), %esp
209	call	1f
2101:	pop	%ebp
211	subl	$1b, %ebp
212
213	movl	%ecx, efi32_config(%ebp)
214	movl	%edx, efi32_config+8(%ebp)
215	sgdtl	efi32_boot_gdt(%ebp)
216
217	leal	efi32_config(%ebp), %eax
218	movl	%eax, efi_config(%ebp)
219
220	jmp	startup_32
221ENDPROC(efi32_stub_entry)
222#endif
223
224	.code64
225	.org 0x200
226ENTRY(startup_64)
227	/*
228	 * 64bit entry is 0x200 and it is ABI so immutable!
229	 * We come here either from startup_32 or directly from a
230	 * 64bit bootloader.
231	 * If we come here from a bootloader, kernel(text+data+bss+brk),
232	 * ramdisk, zero_page, command line could be above 4G.
233	 * We depend on an identity mapped page table being provided
234	 * that maps our entire kernel(text+data+bss+brk), zero page
235	 * and command line.
236	 */
237#ifdef CONFIG_EFI_STUB
238	/*
239	 * The entry point for the PE/COFF executable is efi_pe_entry, so
240	 * only legacy boot loaders will execute this jmp.
241	 */
242	jmp	preferred_addr
243
244ENTRY(efi_pe_entry)
245	movq	%rcx, efi64_config(%rip)	/* Handle */
246	movq	%rdx, efi64_config+8(%rip) /* EFI System table pointer */
247
248	leaq	efi64_config(%rip), %rax
249	movq	%rax, efi_config(%rip)
250
251	call	1f
2521:	popq	%rbp
253	subq	$1b, %rbp
254
255	/*
256	 * Relocate efi_config->call().
257	 */
258	addq	%rbp, efi64_config+88(%rip)
259
260	movq	%rax, %rdi
261	call	make_boot_params
262	cmpq	$0,%rax
263	je	fail
264	mov	%rax, %rsi
265	leaq	startup_32(%rip), %rax
266	movl	%eax, BP_code32_start(%rsi)
267	jmp	2f		/* Skip the relocation */
268
269handover_entry:
270	call	1f
2711:	popq	%rbp
272	subq	$1b, %rbp
273
274	/*
275	 * Relocate efi_config->call().
276	 */
277	movq	efi_config(%rip), %rax
278	addq	%rbp, 88(%rax)
2792:
280	movq	efi_config(%rip), %rdi
281	call	efi_main
282	movq	%rax,%rsi
283	cmpq	$0,%rax
284	jne	2f
285fail:
286	/* EFI init failed, so hang. */
287	hlt
288	jmp	fail
2892:
290	movl	BP_code32_start(%esi), %eax
291	leaq	preferred_addr(%rax), %rax
292	jmp	*%rax
293
294preferred_addr:
295#endif
296
297	/* Setup data segments. */
298	xorl	%eax, %eax
299	movl	%eax, %ds
300	movl	%eax, %es
301	movl	%eax, %ss
302	movl	%eax, %fs
303	movl	%eax, %gs
304
305	/*
306	 * Compute the decompressed kernel start address.  It is where
307	 * we were loaded at aligned to a 2M boundary. %rbp contains the
308	 * decompressed kernel start address.
309	 *
310	 * If it is a relocatable kernel then decompress and run the kernel
311	 * from load address aligned to 2MB addr, otherwise decompress and
312	 * run the kernel from LOAD_PHYSICAL_ADDR
313	 *
314	 * We cannot rely on the calculation done in 32-bit mode, since we
315	 * may have been invoked via the 64-bit entry point.
316	 */
317
318	/* Start with the delta to where the kernel will run at. */
319#ifdef CONFIG_RELOCATABLE
320	leaq	startup_32(%rip) /* - $startup_32 */, %rbp
321	movl	BP_kernel_alignment(%rsi), %eax
322	decl	%eax
323	addq	%rax, %rbp
324	notq	%rax
325	andq	%rax, %rbp
326	cmpq	$LOAD_PHYSICAL_ADDR, %rbp
327	jge	1f
328#endif
329	movq	$LOAD_PHYSICAL_ADDR, %rbp
3301:
331
332	/* Target address to relocate to for decompression */
333	leaq	z_extract_offset(%rbp), %rbx
334
335	/* Set up the stack */
336	leaq	boot_stack_end(%rbx), %rsp
337
338	/* Zero EFLAGS */
339	pushq	$0
340	popfq
341
342/*
343 * Copy the compressed kernel to the end of our buffer
344 * where decompression in place becomes safe.
345 */
346	pushq	%rsi
347	leaq	(_bss-8)(%rip), %rsi
348	leaq	(_bss-8)(%rbx), %rdi
349	movq	$_bss /* - $startup_32 */, %rcx
350	shrq	$3, %rcx
351	std
352	rep	movsq
353	cld
354	popq	%rsi
355
356/*
357 * Jump to the relocated address.
358 */
359	leaq	relocated(%rbx), %rax
360	jmp	*%rax
361
362#ifdef CONFIG_EFI_STUB
363	.org 0x390
364ENTRY(efi64_stub_entry)
365	movq	%rdi, efi64_config(%rip)	/* Handle */
366	movq	%rsi, efi64_config+8(%rip) /* EFI System table pointer */
367
368	leaq	efi64_config(%rip), %rax
369	movq	%rax, efi_config(%rip)
370
371	movq	%rdx, %rsi
372	jmp	handover_entry
373ENDPROC(efi64_stub_entry)
374#endif
375
376	.text
377relocated:
378
379/*
380 * Clear BSS (stack is currently empty)
381 */
382	xorl	%eax, %eax
383	leaq    _bss(%rip), %rdi
384	leaq    _ebss(%rip), %rcx
385	subq	%rdi, %rcx
386	shrq	$3, %rcx
387	rep	stosq
388
389/*
390 * Adjust our own GOT
391 */
392	leaq	_got(%rip), %rdx
393	leaq	_egot(%rip), %rcx
3941:
395	cmpq	%rcx, %rdx
396	jae	2f
397	addq	%rbx, (%rdx)
398	addq	$8, %rdx
399	jmp	1b
4002:
401
402/*
403 * Do the decompression, and jump to the new kernel..
404 */
405	pushq	%rsi			/* Save the real mode argument */
406	movq	$z_run_size, %r9	/* size of kernel with .bss and .brk */
407	pushq	%r9
408	movq	%rsi, %rdi		/* real mode address */
409	leaq	boot_heap(%rip), %rsi	/* malloc area for uncompression */
410	leaq	input_data(%rip), %rdx  /* input_data */
411	movl	$z_input_len, %ecx	/* input_len */
412	movq	%rbp, %r8		/* output target address */
413	movq	$z_output_len, %r9	/* decompressed length, end of relocs */
414	call	decompress_kernel	/* returns kernel location in %rax */
415	popq	%r9
416	popq	%rsi
417
418/*
419 * Jump to the decompressed kernel.
420 */
421	jmp	*%rax
422
423	.code32
424no_longmode:
425	/* This isn't an x86-64 CPU so hang */
4261:
427	hlt
428	jmp     1b
429
430#include "../../kernel/verify_cpu.S"
431
432	.data
433gdt:
434	.word	gdt_end - gdt
435	.long	gdt
436	.word	0
437	.quad	0x0000000000000000	/* NULL descriptor */
438	.quad	0x00af9a000000ffff	/* __KERNEL_CS */
439	.quad	0x00cf92000000ffff	/* __KERNEL_DS */
440	.quad	0x0080890000000000	/* TS descriptor */
441	.quad   0x0000000000000000	/* TS continued */
442gdt_end:
443
444#ifdef CONFIG_EFI_STUB
445efi_config:
446	.quad	0
447
448#ifdef CONFIG_EFI_MIXED
449	.global efi32_config
450efi32_config:
451	.fill	11,8,0
452	.quad	efi64_thunk
453	.byte	0
454#endif
455
456	.global efi64_config
457efi64_config:
458	.fill	11,8,0
459	.quad	efi_call
460	.byte	1
461#endif /* CONFIG_EFI_STUB */
462
463/*
464 * Stack and heap for uncompression
465 */
466	.bss
467	.balign 4
468boot_heap:
469	.fill BOOT_HEAP_SIZE, 1, 0
470boot_stack:
471	.fill BOOT_STACK_SIZE, 1, 0
472boot_stack_end:
473
474/*
475 * Space for page tables (not in .bss so not zeroed)
476 */
477	.section ".pgtable","a",@nobits
478	.balign 4096
479pgtable:
480	.fill 6*4096, 1, 0
481