Linux双核SMP系统启动流程（Zynq-ARM-CortexA9）

转自：

http://blog.chinaunix.net/uid-20648445-id-3329217.html

3：启动流程：

注：本博文以提出问题，回答问题方式进行记录本人研究双核系统的过程。希望在一步步研究及分析后，能够完整回答各个问题。以下分析基于ARM v7架构Linux代码和XILINX的ZYNQ平台。由于本博文正在更新过程中，还未完成，若对单核启动有兴趣的朋友可以查看如下资源，该资源正是本人前半部分启动需要描述的内容。

    网络资源

问题1-1：双核芯片上电后，是否同时启动的？

答案：双核芯片上电后，并非同时启动，启动代码运行在一个核上，而是一个核处于备用状态。

参见<ug585-Zynq-7000-TRM--P140/1671—6.3.7 Post BootROM State—Starting Code on the CPU 1>

 

图1：CPU1启动工作状态

图2: CPU1启动流程

问题1-2：根据上述参考资料，第二个核是通过CPU0进行设置并引导启动的，Linux是怎么完成这一步的呢？我们从内核入口函数开始研究启动过程。
答案：本部分从Linux系统启动部分开始分析，BOOT启动引导部分暂不做分析。
    1）入口函数
    \linux-xlnx\arch\arm\kernel\head.S

点击(此处)折叠或打开--CodeSegment 1

1. /*
2. * Kernel startup entry point.
3. * ---------------------------
4. *
5. * This is normally called from the decompressor code. The requirements
6. * are: MMU = off, D-cache = off, I-cache = dont care, r0 = 0,
7. * r1 = machine nr, r2 = atags or dtb pointer.
8. *
9. * This code is mostly position independent, so if you link the kernel at
10. * 0xc0008000, you call this at __pa(0xc0008000).
11. *
12. * See linux/arch/arm/tools/mach-types for the complete list of machine
13. * numbers for r1.
14. *
15. * We're trying to keep crap to a minimum; DO NOT add any machine specific
16. * crap here - that's what the boot loader (or in extreme, well justified
17. * circumstances, zImage) is for.
18. */
19. .arm
20. 
    
21. __HEAD
22. ENTRY(stext)
23. 
    
24. THUMB( adr r9, BSYM(1f) ) @ Kernel is always entered in ARM.
25. THUMB( bx r9 ) @ If this is a Thumb-2 kernel,
26. THUMB( .thumb ) @ switch to Thumb now.
27. THUMB(1: )
28. 
    
29. setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode
30. @ and irqs disabled
31. mrc p15, 0, r9, c0, c0 @ get processor id
32. bl __lookup_processor_type @ r5=procinfo r9=cpuid
33. movs r10, r5 @ invalid processor (r5=0)?
34. THUMB( it eq ) @ force fixup-able long branch encoding
35. beq __error_p @ yes, error 'p'
36. 
    
37. /* the machine has no way to get setup when we're using a pod so setup it
38. * up for now this way, if the device tree is expected at the fixed address
39. * then load R2 to find the device tree at that address
40. */
41. #ifdef CONFIG_ARCH_XILINX
42. mov r0,#0x0
43. ldr r1,=MACH_TYPE_XILINX
44. #endif
45. #ifdef CONFIG_XILINX_FIXED_DEVTREE_ADDR
46. mov r2,#0x1000000
47. #endif
48. #ifdef CONFIG_ARM_LPAE
49. mrc p15, 0, r3, c0, c1, 4 @ read ID_MMFR0
50. and r3, r3, #0xf @ extract VMSA support
51. cmp r3, #5 @ long-descriptor translation table format?
52. THUMB( it lo ) @ force fixup-able long branch encoding
53. blo __error_p @ only classic page table format
54. #endif
55. 
    
56. #ifndef CONFIG_XIP_KERNEL
57. adr r3, 2f
58. ldmia r3, {r4, r8}
59. sub r4, r3, r4 @ (PHYS_OFFSET - PAGE_OFFSET)
60. add r8, r8, r4 @ PHYS_OFFSET
61. #else
62. ldr r8, =PHYS_OFFSET @ always constant in this case
63. #endif
64. 
    
65. /*
66. * r1 = machine no, r2 = atags or dtb,
67. * r8 = phys_offset, r9 = cpuid, r10 = procinfo
68. */
69. bl __vet_atags
70. #ifdef CONFIG_SMP_ON_UP
71. bl __fixup_smp
72. #endif
73. #ifdef CONFIG_ARM_PATCH_PHYS_VIRT
74. bl __fixup_pv_table
75. #endif
76. bl __create_page_tables
77. 
    
78. /*
79. * The following calls CPU specific code in a position independent
80. * manner. See arch/arm/mm/proc-*.S for details. r10 = base of
81. * xxx_proc_info structure selected by __lookup_processor_type
82. * above. On return, the CPU will be ready for the MMU to be
83. * turned on, and r0 will hold the CPU control register value.
84. */
85. ldr r13, =__mmap_switched @ address to jump to after
86. @ mmu has been enabled
87. adr lr, BSYM(1f) @ return (PIC) address
88. mov r8, r4 @ set TTBR1 to swapper_pg_dir
89. ARM( add pc, r10, #PROCINFO_INITFUNC )
90. THUMB( add r12, r10, #PROCINFO_INITFUNC )
91. THUMB( mov pc, r12 )
92. 1: b __enable_mmu
93. ENDPROC(stext)
94. .ltorg
95. #ifndef CONFIG_XIP_KERNEL
96. 2: .long .
97. .long PAGE_OFFSET
98. #endif

注：本部分源码为汇编代码，汇编与C语言相互调用参数传递，返回值等相关约定参见<IHI0042D_aapcs.pdf>
问题1-2-1：U-Boot程序如何跳转到该入口函数？
答案：希望后续章节可以研究分析U-Boot源码进行深入分析。 待补充。

问题 1-2-2 ：下列代码为什么意思？THUMB宏是什么定义？ARM宏是什么定义？BSYM宏定义是什么？

点击(此处)折叠或打开--CodeSegment 2

1. THUMB( adr r9, BSYM(1f) ) @ Kernel is always entered in ARM.
2. THUMB( bx r9 ) @ If this is a Thumb-2 kernel,
3. THUMB( .thumb ) @ switch to Thumb now.
4. THUMB(1: )

答案： THUMB宏定义参见文件\linux-xlnx\arch\arm\include\asm\unified.h，分析如下宏定义可知，内核在GCC版本大于4.0后，可通过配置宏CONFIG_THUMB2_KERNEL来选择编译成THUMB指令集内核或ARM指令集内核。在编译成ARM指令集内核时，THUMB宏定义的代码行是不可见的。同理编译成THUMB指令集内核时，ARM宏定义的代码行无效。BSYM宏是为在THUMB指令集时访问标号处理而定义的宏。

点击(此处)折叠或打开--CodeSegment 3

1. #ifdef CONFIG_THUMB2_KERNEL
2. 
   
3. #if __GNUC__ < 4
4. #error Thumb-2 kernel requires gcc >= 4
5. #endif
6. 
   
7. /* The CPSR bit describing the instruction set (Thumb) */
8. #define PSR_ISETSTATE PSR_T_BIT
9. 
   
10. #define ARM(x...)
11. #define THUMB(x...) x
12. #ifdef __ASSEMBLY__
13. #define W(instr) instr.w
14. #define BSYM(sym) sym + 1
15. #endif
16. 
    
17. #else /* !CONFIG_THUMB2_KERNEL */
18. 
    
19. /* The CPSR bit describing the instruction set (ARM) */
20. #define PSR_ISETSTATE 0
21. 
    
22. #define ARM(x...) x
23. #define THUMB(x...)
24. #ifdef __ASSEMBLY__
25. #define W(instr) instr
26. #define BSYM(sym) sym
27. #endif
28. 
    
29. #endif /* CONFIG_THUMB2_KERNEL */

问题 1-2-3 ： THUMB指令集与ARM指令集区别，什么是THUMB-2指令集
答案： 待补充

问题 1-2- 4 ： setmode是什么定义？
答案：setmode定义参见文件 \linux-xlnx\arch\arm\include\asm\assembler.h ，下面代码为一个汇编宏定义。根据不同的指令集采用不同的宏定义，用于设置CPSR。

点击(此处)折叠或打开--CodeSegment 4

1. #ifdef CONFIG_THUMB2_KERNEL
2. .macro setmode, mode, reg
3. mov \reg, #\mode
4. msr cpsr_c, \reg
5. .endm
6. #else
7. .macro setmode, mode, reg
8. msr cpsr_c, #\mode
9. .endm
10. #endif

问题 1-2-5 ： __lookup_processor_type是什么？查找处理器类型？
答案：该汇编函数定义位于文件\linux-xlnx\arch\arm\kernel\head-common.S ，分析如下汇编代码，通过 从__lookup_processor_type_data 中获取处理器信息位置，通过轮询查找该内核是否支持该本处理器。其中r9是通过 processor id。本代码段会进行详细注释，汇编指令
参见< DDI0406C_arm_architecture_reference_manual --P157/2680—A8 Instruction Details—Alphabetical list of instructions>

点击(此处)折叠或打开--CodeSegment 5

1. /*
2. * Read processor ID register (CP#15, CR0), and look up in the linker-built
3. * supported processor list. Note that we can't use the absolute addresses
4. * for the __proc_info lists since we aren't running with the MMU on
5. * (and therefore, we are not in the correct address space). We have to
6. * calculate the offset.
7. *
8. * r9 = cpuid
9. * Returns:
10. * r3, r4, r6 corrupted
11. * r5 = proc_info pointer in physical address space
12. * r9 = cpuid (preserved)
13. */
14. __CPUINIT
15. __lookup_processor_type:
16. adr r3, __lookup_processor_type_data   @ R3指向类型数据指针(此处R3是物理地址，可查看反汇编代码，下图3所示)
    
17. ldmia r3, {r4 - r6}                    @ 读取R3指向地址三个WORD数据到R4,R5,R6
    
18. sub r3, r3, r4                         @ 物理地址-虚拟地址
    
19. add r5, r5, r3            @ 将R5的虚拟地址转换成物理地址，R5为类型数据起始地址
20. add r6, r6, r3            @ 将R6的虚拟地址转换成物理地址，R6为类型数据结束地址
21. 1: ldmia r5, {r3, r4}     @ 从R5指向的地址读取两个WORD到R3,R4
22. and r4, r4, r9            @ 获取判断位
    
23. teq r3, r4                @ 判断本处理器类型是否已找到
    
24. beq 2f                    @ 若已找到，跳转到2：执行
    
25. add r5, r5, #PROC_INFO_SZ @ 若未找到，R5指针增到一个类型数据长度
26. cmp r5, r6                @ 是否已查找完所有处理器类型
    
27. blo 1b
28. mov r5, #0                @ 若未找到本处理器类型信息，返回值R5设置为NULL
    
29. 2: mov pc, lr
30. ENDPROC(__lookup_processor_type)

问题 1-2- 6 ： __lookup_processor_type_data是什么 ？上个问题中R3，R4，R5，R6寄存器是指向什么数据？
答案： __lookup_processor_type_data 数据结构定义位于文件\linux-xlnx\arch\arm\kernel\head-common.S ，参见汇编代码可知，该数据结构有三个数据，数据内容参见代码注释。

点击(此处)折叠或打开--CodeSegment 6

1. /*
2. * Look in <asm/procinfo.h> for information about the __proc_info structure.
3. */
4. .align 2
5. .type __lookup_processor_type_data, %object
6. __lookup_processor_type_data:
7. .long .                              @ 当前内存地址（此处为虚拟地址）
   
8. .long __proc_info_begin              @ 处理器类型信息起始地址
   
9. .long __proc_info_end                @ 处理器类型信息结束地址
10. .size __lookup_processor_type_data, . - __lookup_processor_type_data

通过反汇编，我们可以查看内核编译完成后各个数据结构的具体数据。

图3 反汇编代码

__proc_info_begin，__proc_info_end定义位于\linux-xlnx\arch\arm\kernel\vmlinux.lds.S，它用于存储.proc.info.init段的起始地址及结束地址。

点击(此处)折叠或打开--CodeSegment 7

1. VMLINUX_SYMBOL(__proc_info_begin) = .; \
2. *(.proc.info.init) \
3. VMLINUX_SYMBOL(__proc_info_end) = .;

.proc.info.init段定义位于\linux-xlnx\arch\arm\mm\proc-v7.S，该文件定义了本内核版本支持的ARM v7架构下处理器类型，如下列代码Line 29行表示本版本支持ARM Cortex A5 Processor，Line 39表示本版本支持ARM Cortex A9 Processor等。

点击(此处)折叠或打开--CodeSegment 8

1. .section ".proc.info.init", #alloc, #execinstr
2. 
   
3. /*
4. * Standard v7 proc info content
5. */
6. .macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0
7. ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
8. PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags)
9. ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
10. PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags)
11. .long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | \
12. PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags
13. W(b) \initfunc
14. .long cpu_arch_name
15. .long cpu_elf_name
16. .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_FAST_MULT | \
17. HWCAP_EDSP | HWCAP_TLS | \hwcaps
18. .long cpu_v7_name
19. .long v7_processor_functions
20. .long v7wbi_tlb_fns
21. .long v6_user_fns
22. .long v7_cache_fns
23. .endm
24. 
    
25. #ifndef CONFIG_ARM_LPAE
26. /*
27. * ARM Ltd. Cortex A5 processor.
28. */
29. .type __v7_ca5mp_proc_info, #object
30. __v7_ca5mp_proc_info:
31. .long 0x410fc050
32. .long 0xff0ffff0
33. __v7_proc __v7_ca5mp_setup
34. .size __v7_ca5mp_proc_info, . - __v7_ca5mp_proc_info
35. 
    
36. /*
37. * ARM Ltd. Cortex A9 processor.
38. */
39. .type __v7_ca9mp_proc_info, #object
40. __v7_ca9mp_proc_info:
41. .long 0x410fc090
42. .long 0xff0ffff0
43. __v7_proc __v7_ca9mp_setup
44. .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info
45. #endif /* CONFIG_ARM_LPAE */
46. 
    
47. /*
48. * ARM Ltd. Cortex A7 processor.
49. */
50. .type __v7_ca7mp_proc_info, #object
51. __v7_ca7mp_proc_info:
52. .long 0x410fc070
53. .long 0xff0ffff0
54. __v7_proc __v7_ca7mp_setup, hwcaps = HWCAP_IDIV
55. .size __v7_ca7mp_proc_info, . - __v7_ca7mp_proc_info
56. ......

问题 1-2-7 ： 在上述问题中我们可以看到支持的处理信息数据，但这些数据是什么样的结构呢？ 代表什么意义呢？
答案： .proc.info.init段中数据 是以proc_info_list结构的方式进行连续存放的。该结构体的定义位于\linux-xlnx\arch\arm\include\asm\procinfo.h，

点击(此处)折叠或打开--CodeSegment 9

1. /*
2. * Note! struct processor is always defined if we're
3. * using MULTI_CPU, otherwise this entry is unused,
4. * but still exists.
5. *
6. * NOTE! The following structure is defined by assembly
7. * language, NOT C code. For more information, check:
8. * arch/arm/mm/proc-*.S and arch/arm/kernel/head.S
9. */
10. struct proc_info_list {
11. unsigned int cpu_val;
12. unsigned int cpu_mask;
13. unsigned long __cpu_mm_mmu_flags; /* used by head.S */
14. unsigned long __cpu_io_mmu_flags; /* used by head.S */
15. unsigned long __cpu_flush; /* used by head.S */
16. const char *arch_name;
17. const char *elf_name;
18. unsigned int elf_hwcap;
19. const char *cpu_name;
20. struct processor *proc;
21. struct cpu_tlb_fns *tlb;
22. struct cpu_user_fns *user;
23. struct cpu_cache_fns *cache;
24. };

问题 1-2- 8 ： __vet_atags是什么？
答案：待补充

点击(此处)折叠或打开--CodeSegment 10

1. /* Determine validity of the r2 atags pointer. The heuristic requires
2. * that the pointer be aligned, in the first 16k of physical RAM and
3. * that the ATAG_CORE marker is first and present. If CONFIG_OF_FLATTREE
4. * is selected, then it will also accept a dtb pointer. Future revisions
5. * of this function may be more lenient with the physical address and
6. * may also be able to move the ATAGS block if necessary.
7. *
8. * Returns:
9. * r2 either valid atags pointer, valid dtb pointer, or zero
10. * r5, r6 corrupted
11. */
12. __vet_atags:
13. tst r2, #0x3 @ aligned?
14. bne 1f
15. 
    
16. ldr r5, [r2, #0]
17. #ifdef CONFIG_OF_FLATTREE
18. ldr r6, =OF_DT_MAGIC @ is it a DTB?
19. cmp r5, r6
20. beq 2f
21. #endif
22. cmp r5, #ATAG_CORE_SIZE @ is first tag ATAG_CORE?
23. cmpne r5, #ATAG_CORE_SIZE_EMPTY
24. bne 1f
25. ldr r5, [r2, #4]
26. ldr r6, =ATAG_CORE
27. cmp r5, r6
28. bne 1f
29. 
    
30. 2: mov pc, lr @ atag/dtb pointer is ok
31. 
    
32. 1: mov r2, #0
33. mov pc, lr
34. ENDPROC(__vet_atags)

问题 1-2-9 ：__fixup_smp是什么？
答案：待补充