1
2
3
4
5
6
7
8
9
10 .text
11 #include <linux/threads.h>
12 #include <linux/init.h>
13 #include <linux/linkage.h>
14 #include <asm/segment.h>
15 #include <asm/page_types.h>
16 #include <asm/pgtable_types.h>
17 #include <asm/cache.h>
18 #include <asm/thread_info.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/setup.h>
21 #include <asm/processor-flags.h>
22 #include <asm/msr-index.h>
23 #include <asm/cpufeatures.h>
24 #include <asm/percpu.h>
25 #include <asm/nops.h>
26 #include <asm/bootparam.h>
27 #include <asm/export.h>
28 #include <asm/pgtable_32.h>
29
30
31 #define pa(X) ((X) - __PAGE_OFFSET)
32
33
34
35
36
37 #define X86 new_cpu_data+CPUINFO_x86
38 #define X86_VENDOR new_cpu_data+CPUINFO_x86_vendor
39 #define X86_MODEL new_cpu_data+CPUINFO_x86_model
40 #define X86_STEPPING new_cpu_data+CPUINFO_x86_stepping
41 #define X86_HARD_MATH new_cpu_data+CPUINFO_hard_math
42 #define X86_CPUID new_cpu_data+CPUINFO_cpuid_level
43 #define X86_CAPABILITY new_cpu_data+CPUINFO_x86_capability
44 #define X86_VENDOR_ID new_cpu_data+CPUINFO_x86_vendor_id
45
46
47 #define SIZEOF_PTREGS 17*4
48
49
50
51
52
53
54 KERNEL_PAGES = LOWMEM_PAGES
55
56 INIT_MAP_SIZE = PAGE_TABLE_SIZE(KERNEL_PAGES) * PAGE_SIZE
57 RESERVE_BRK(pagetables, INIT_MAP_SIZE)
58
59
60
61
62
63
64
65
66 __HEAD
67 ENTRY(startup_32)
68 movl pa(initial_stack),%ecx
69
70
71
72 testb $KEEP_SEGMENTS, BP_loadflags(%esi)
73 jnz 2f
74
75
76
77
78 lgdt pa(boot_gdt_descr)
79 movl $(__BOOT_DS),%eax
80 movl %eax,%ds
81 movl %eax,%es
82 movl %eax,%fs
83 movl %eax,%gs
84 movl %eax,%ss
85 2:
86 leal -__PAGE_OFFSET(%ecx),%esp
87
88
89
90
91 cld
92 xorl %eax,%eax
93 movl $pa(__bss_start),%edi
94 movl $pa(__bss_stop),%ecx
95 subl %edi,%ecx
96 shrl $2,%ecx
97 rep ; stosl
98
99
100
101
102
103
104
105
106 movl $pa(boot_params),%edi
107 movl $(PARAM_SIZE/4),%ecx
108 cld
109 rep
110 movsl
111 movl pa(boot_params) + NEW_CL_POINTER,%esi
112 andl %esi,%esi
113 jz 1f # No command line
114 movl $pa(boot_command_line),%edi
115 movl $(COMMAND_LINE_SIZE/4),%ecx
116 rep
117 movsl
118 1:
119
120 #ifdef CONFIG_OLPC
121
122 movl %cr3, %eax
123 movl %eax, pa(olpc_ofw_pgd)
124 #endif
125
126 #ifdef CONFIG_MICROCODE
127
128 call load_ucode_bsp
129 #endif
130
131
132 call mk_early_pgtbl_32
133
134
135 movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax
136 #ifdef CONFIG_X86_PAE
137 #define KPMDS (((-__PAGE_OFFSET) >> 30) & 3)
138 movl %eax,pa(initial_pg_pmd+0x1000*KPMDS-8)
139 #else
140 movl %eax,pa(initial_page_table+0xffc)
141 #endif
142
143 #ifdef CONFIG_PARAVIRT
144
145 cmpw $0x207, pa(boot_params + BP_version)
146 jb .Ldefault_entry
147
148
149
150 movl pa(boot_params + BP_hardware_subarch), %eax
151 cmpl $num_subarch_entries, %eax
152 jae .Lbad_subarch
153
154 movl pa(subarch_entries)(,%eax,4), %eax
155 subl $__PAGE_OFFSET, %eax
156 jmp *%eax
157
158 .Lbad_subarch:
159 WEAK(xen_entry)
160
161
162 ud2a
163
164 __INITDATA
165
166 subarch_entries:
167 .long .Ldefault_entry
168 .long xen_entry
169 .long .Ldefault_entry
170 num_subarch_entries = (. - subarch_entries) / 4
171 .previous
172 #else
173 jmp .Ldefault_entry
174 #endif
175
176 #ifdef CONFIG_HOTPLUG_CPU
177
178
179
180
181
182 ENTRY(start_cpu0)
183 movl initial_stack, %ecx
184 movl %ecx, %esp
185 call *(initial_code)
186 1: jmp 1b
187 ENDPROC(start_cpu0)
188 #endif
189
190
191
192
193
194
195
196
197
198 ENTRY(startup_32_smp)
199 cld
200 movl $(__BOOT_DS),%eax
201 movl %eax,%ds
202 movl %eax,%es
203 movl %eax,%fs
204 movl %eax,%gs
205 movl pa(initial_stack),%ecx
206 movl %eax,%ss
207 leal -__PAGE_OFFSET(%ecx),%esp
208
209 #ifdef CONFIG_MICROCODE
210
211 call load_ucode_ap
212 #endif
213
214 .Ldefault_entry:
215 movl $(CR0_STATE & ~X86_CR0_PG),%eax
216 movl %eax,%cr0
217
218
219
220
221
222
223
224
225 pushl $0
226 popfl
227
228
229
230
231
232
233
234 movl $-1,pa(X86_CPUID) # preset CPUID level
235 movl $X86_EFLAGS_ID,%ecx
236 pushl %ecx
237 popfl # set EFLAGS=ID
238 pushfl
239 popl %eax # get EFLAGS
240 testl $X86_EFLAGS_ID,%eax # did EFLAGS.ID remained set?
241 jz .Lenable_paging # hw disallowed setting of ID bit
242
243
244 xorl %eax,%eax
245 cpuid
246 movl %eax,pa(X86_CPUID) # save largest std CPUID function
247
248 movl $1,%eax
249 cpuid
250 andl $~1,%edx # Ignore CPUID.FPU
251 jz .Lenable_paging # No flags or only CPUID.FPU = no CR4
252
253 movl pa(mmu_cr4_features),%eax
254 movl %eax,%cr4
255
256 testb $X86_CR4_PAE, %al # check if PAE is enabled
257 jz .Lenable_paging
258
259
260 movl $0x80000000, %eax
261 cpuid
262
263 subl $0x80000001, %eax
264 cmpl $(0x8000ffff-0x80000001), %eax
265 ja .Lenable_paging
266
267
268 call verify_cpu
269
270 mov $0x80000001, %eax
271 cpuid
272
273 btl $(X86_FEATURE_NX & 31), %edx
274 jnc .Lenable_paging
275
276
277 movl $MSR_EFER, %ecx
278 rdmsr
279
280 btsl $_EFER_NX, %eax
281
282 wrmsr
283
284 .Lenable_paging:
285
286
287
288
289 movl $pa(initial_page_table), %eax
290 movl %eax,%cr3
291 movl $CR0_STATE,%eax
292 movl %eax,%cr0
293 ljmp $__BOOT_CS,$1f
294 1:
295
296 addl $__PAGE_OFFSET, %esp
297
298
299
300
301
302 movl setup_once_ref,%eax
303 andl %eax,%eax
304 jz 1f # Did we do this already?
305 call *%eax
306 1:
307
308
309
310
311 movb $4,X86 # at least 486
312 cmpl $-1,X86_CPUID
313 je .Lis486
314
315
316 xorl %eax,%eax # call CPUID with 0 -> return vendor ID
317 cpuid
318 movl %eax,X86_CPUID # save CPUID level
319 movl %ebx,X86_VENDOR_ID # lo 4 chars
320 movl %edx,X86_VENDOR_ID+4 # next 4 chars
321 movl %ecx,X86_VENDOR_ID+8 # last 4 chars
322
323 orl %eax,%eax # do we have processor info as well?
324 je .Lis486
325
326 movl $1,%eax # Use the CPUID instruction to get CPU type
327 cpuid
328 movb %al,%cl # save reg for future use
329 andb $0x0f,%ah # mask processor family
330 movb %ah,X86
331 andb $0xf0,%al # mask model
332 shrb $4,%al
333 movb %al,X86_MODEL
334 andb $0x0f,%cl # mask mask revision
335 movb %cl,X86_STEPPING
336 movl %edx,X86_CAPABILITY
337
338 .Lis486:
339 movl $0x50022,%ecx # set AM, WP, NE and MP
340 movl %cr0,%eax
341 andl $0x80000011,%eax # Save PG,PE,ET
342 orl %ecx,%eax
343 movl %eax,%cr0
344
345 lgdt early_gdt_descr
346 ljmp $(__KERNEL_CS),$1f
347 1: movl $(__KERNEL_DS),%eax # reload all the segment registers
348 movl %eax,%ss # after changing gdt.
349
350 movl $(__USER_DS),%eax # DS/ES contains default USER segment
351 movl %eax,%ds
352 movl %eax,%es
353
354 movl $(__KERNEL_PERCPU), %eax
355 movl %eax,%fs # set this cpu's percpu
356
357 movl $(__KERNEL_STACK_CANARY),%eax
358 movl %eax,%gs
359
360 xorl %eax,%eax # Clear LDT
361 lldt %ax
362
363 call *(initial_code)
364 1: jmp 1b
365 ENDPROC(startup_32_smp)
366
367 #include "verify_cpu.S"
368
369
370
371
372
373
374
375
376 __INIT
377 setup_once:
378 #ifdef CONFIG_STACKPROTECTOR
379
380
381
382
383
384 movl $gdt_page,%eax
385 movl $stack_canary,%ecx
386 movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
387 shrl $16, %ecx
388 movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
389 movb %ch, 8 * GDT_ENTRY_STACK_CANARY + 7(%eax)
390 #endif
391
392 andl $0,setup_once_ref
393 ret
394
395 ENTRY(early_idt_handler_array)
396 # 36(%esp) %eflags
397 # 32(%esp) %cs
398 # 28(%esp) %eip
399 # 24(%rsp) error code
400 i = 0
401 .rept NUM_EXCEPTION_VECTORS
402 .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
403 pushl $0 # Dummy error code, to make stack frame uniform
404 .endif
405 pushl $i # 20(%esp) Vector number
406 jmp early_idt_handler_common
407 i = i + 1
408 .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
409 .endr
410 ENDPROC(early_idt_handler_array)
411
412 early_idt_handler_common:
413
414
415
416
417 cld
418
419 incl %ss:early_recursion_flag
420
421
422
423 cld
424 pushl %fs
425 pushl %es
426 pushl %ds
427 pushl %eax
428 pushl %ebp
429 pushl %edi
430 pushl %esi
431 pushl %edx
432 pushl %ecx
433 pushl %ebx
434
435
436 movl $(__KERNEL_DS), %ecx
437 movl %ecx, %ds
438 movl %ecx, %es
439
440
441 movl PT_GS(%esp), %edx
442
443
444 movw %gs, PT_GS(%esp)
445
446 movl %esp, %eax
447 call early_fixup_exception
448
449 popl %ebx
450 popl %ecx
451 popl %edx
452 popl %esi
453 popl %edi
454 popl %ebp
455 popl %eax
456 popl %ds
457 popl %es
458 popl %fs
459 popl %gs
460 decl %ss:early_recursion_flag
461 addl $4, %esp
462 iret
463 ENDPROC(early_idt_handler_common)
464
465
466 ENTRY(early_ignore_irq)
467 cld
468 #ifdef CONFIG_PRINTK
469 pushl %eax
470 pushl %ecx
471 pushl %edx
472 pushl %es
473 pushl %ds
474 movl $(__KERNEL_DS),%eax
475 movl %eax,%ds
476 movl %eax,%es
477 cmpl $2,early_recursion_flag
478 je hlt_loop
479 incl early_recursion_flag
480 pushl 16(%esp)
481 pushl 24(%esp)
482 pushl 32(%esp)
483 pushl 40(%esp)
484 pushl $int_msg
485 call printk
486
487 call dump_stack
488
489 addl $(5*4),%esp
490 popl %ds
491 popl %es
492 popl %edx
493 popl %ecx
494 popl %eax
495 #endif
496 iret
497
498 hlt_loop:
499 hlt
500 jmp hlt_loop
501 ENDPROC(early_ignore_irq)
502
503 __INITDATA
504 .align 4
505 GLOBAL(early_recursion_flag)
506 .long 0
507
508 __REFDATA
509 .align 4
510 ENTRY(initial_code)
511 .long i386_start_kernel
512 ENTRY(setup_once_ref)
513 .long setup_once
514
515 #ifdef CONFIG_PAGE_TABLE_ISOLATION
516 #define PGD_ALIGN (2 * PAGE_SIZE)
517 #define PTI_USER_PGD_FILL 1024
518 #else
519 #define PGD_ALIGN (PAGE_SIZE)
520 #define PTI_USER_PGD_FILL 0
521 #endif
522
523
524
525 __PAGE_ALIGNED_BSS
526 .align PGD_ALIGN
527 #ifdef CONFIG_X86_PAE
528 .globl initial_pg_pmd
529 initial_pg_pmd:
530 .fill 1024*KPMDS,4,0
531 #else
532 .globl initial_page_table
533 initial_page_table:
534 .fill 1024,4,0
535 #endif
536 .align PGD_ALIGN
537 initial_pg_fixmap:
538 .fill 1024,4,0
539 .globl swapper_pg_dir
540 .align PGD_ALIGN
541 swapper_pg_dir:
542 .fill 1024,4,0
543 .fill PTI_USER_PGD_FILL,4,0
544 .globl empty_zero_page
545 empty_zero_page:
546 .fill 4096,1,0
547 EXPORT_SYMBOL(empty_zero_page)
548
549
550
551
552 #ifdef CONFIG_X86_PAE
553 __PAGE_ALIGNED_DATA
554
555 .align PGD_ALIGN
556 ENTRY(initial_page_table)
557 .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0
558 # if KPMDS == 3
559 .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0
560 .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0
561 .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x2000),0
562 # elif KPMDS == 2
563 .long 0,0
564 .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0
565 .long pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0
566 # elif KPMDS == 1
567 .long 0,0
568 .long 0,0
569 .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0
570 # else
571 # error "Kernel PMDs should be 1, 2 or 3"
572 # endif
573 .align PAGE_SIZE
574
575 #ifdef CONFIG_PAGE_TABLE_ISOLATION
576
577
578
579
580
581 .fill 1024, 4, 0
582 #endif
583
584 #endif
585
586 .data
587 .balign 4
588 ENTRY(initial_stack)
589
590
591
592
593 .long init_thread_union + THREAD_SIZE - SIZEOF_PTREGS - \
594 TOP_OF_KERNEL_STACK_PADDING;
595
596 __INITRODATA
597 int_msg:
598 .asciz "Unknown interrupt or fault at: %p %p %p\n"
599
600 #include "../../x86/xen/xen-head.S"
601
602
603
604
605
606
607
608
609 .data
610 .globl boot_gdt_descr
611
612 ALIGN
613
614 .word 0 # 32 bit align gdt_desc.address
615 boot_gdt_descr:
616 .word __BOOT_DS+7
617 .long boot_gdt - __PAGE_OFFSET
618
619
620 .word 0 # 32 bit align gdt_desc.address
621 ENTRY(early_gdt_descr)
622 .word GDT_ENTRIES*8-1
623 .long gdt_page
624
625
626
627
628
629 .align L1_CACHE_BYTES
630 ENTRY(boot_gdt)
631 .fill GDT_ENTRY_BOOT_CS,8,0
632 .quad 0x00cf9a000000ffff
633 .quad 0x00cf92000000ffff