linux SMP启动代码分析
If one replicates an entire CPU to execute a second thread, then the technique is known as
multi-processing
.
If one replicates only a portion of a CPU to execute a second thread, then the technique is known as
multi-threading
. (关于MP详细的可以查看An Overview of MIPS Multi-Threading
)
红色部分是共用的，蓝色部分是私有的，灰色部分跟实现有关
start_kernel（）
-->boot_cpu_init()
--->int cpu = smp_processor_id();
---->raw_smp_processor_id()
--->#define raw_smp_processor_id() (current_thread_info()->cpu)
---->set_cpu_online(cpu, true);
---->setup_arch()
--->prom_init()
--->board_ebase_setup = &bmips_ebase_setup;
--->register_smp_ops(&
bmips_smp_ops
);
---->plat_smp_setup()
---->
mp_ops->smp_setup();
=
bmips_smp_setup()
#define NR_CPUS CONFIG_NR_CPUS
struct plat_smp_ops
bmips_smp_ops
= {
.smp_setup =
bmips_smp_setup
,
.prepare_cpus = bmips_prepare_cpus,
.boot_secondary = bmips_boot_secondary,
.smp_finish = bmips_smp_finish,
.init_secondary = bmips_init_secondary,
.cpus_done = bmips_cpus_done,
.send_ipi_single = bmips_send_ipi_single,
.send_ipi_mask = bmips_send_ipi_mask,
#ifdef CONFIG_HOTPLUG_CPU
.cpu_disable = bmips_cpu_disable,
.cpu_die = bmips_cpu_die,
#endif
};
static void __init
bmips_smp_setup
(void)
{
/* arbitration priority */
clear_c0_brcm_cmt_ctrl(0x30);
/* NBK and weak order flags */
set_c0_brcm_config_0(0x30000);
/*
* MIPS interrupts 0,1 (SW INT 0,1) cross over to the other thread
* MIPS interrupt 2 (HW INT 0) is the CPU0 L1 controller output
* MIPS interrupt 3 (HW INT 1) is the CPU1 L1 controller output
*/
change_c0_brcm_cmt_intr(0xf8018000,
(0x02 << 27) | (0x03 << 15));
/* single core, 2 threads (2 pipelines) */
max_cpus = 2;
if (!bmips_smp_enabled)
max_cpus = 1;
/* this can be overridden by the BSP */
if (!board_ebase_setup)
board_ebase_setup = &bmips_ebase_setup;
for (i = 0; i < max_cpus; i++) {
__cpu_number_map[i] = 1;
__cpu_logical_map[i] = 1;
set_cpu_possible(i, 1);
set_cpu_present(i, 1);
}
}
/*
* SMTC Linux requires shutting-down microthread scheduling
* during CP0 register read-modify-write sequences.
*/
#define
__BUILD_SET_C0
(name) \
static inline unsigned int \
set_c0_##name(unsigned int set) \
{ \
unsigned int res, new; \
\
res = read_c0_##name(); \
new = res | set; \
write_c0_##name(new); \
\
return res; \
} \
\
static inline unsigned int \
clear_c0_##name(unsigned int clear) \
{ \
unsigned int res, new; \
\
res = read_c0_##name(); \
new = res & ~clear; \
write_c0_##name(new); \
\
return res; \
} \
\
static inline unsigned int \
change_c0_##name(unsigned int change, unsigned int val) \
{ \
unsigned int res, new; \
\
res = read_c0_##name(); \
new = res & ~change; \
new |= (val & change); \
write_c0_##name(new); \
\
return res; \
}
/* BMIPS43xx */
#define read_c0_
brcm_cmt_intr
() __read_32bit_c0_register($22, 1)
#define write_c0_brcm_cmt_intr(val) __write_32bit_c0_register($22, 1, val)
#define read_c0_
brcm_cmt_ctrl
() __read_32bit_c0_register($22, 2)
#define write_c0_brcm_cmt_ctrl(val) __write_32bit_c0_register($22, 2, val)
#define read_c0_
brcm_cmt_local
() __read_32bit_c0_register($22, 3)
#define write_c0_brcm_cmt_local(val) __write_32bit_c0_register($22, 3, val)
__BUILD_SET_C0(
brcm_cmt_intr
)
__BUILD_SET_C0(
brcm_cmt_ctrl
)
start_kernel（）
------>rest_init()
--->kernel_init()
---->smp_init()
---->cpu_up()
----->_cpu_up()
--->__cpu_up()
int __cpuinit __cpu_up(unsigned int cpu)
{
struct task_struct *idle;
/*
* Processor goes to start_secondary(), sets online flag
* The following code is purely to make sure
* Linux can schedule processes on this slave.
*/
if (!cpu_idle_thread[cpu]) {
/*
* Schedule work item to avoid forking user task
* Ported from arch/x86/kernel/smpboot.c
*/
struct create_idle c_idle = {
.cpu = cpu,
.done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
};
INIT_WORK_ONSTACK(&c_idle.work,
do_fork_idle
);
schedule_work(&c_idle.work);
wait_for_completion(&c_idle.done);
idle = cpu_idle_thread[cpu] = c_idle.idle;
if (IS_ERR(idle))
panic(KERN_ERR "Fork failed for CPU %d", cpu);
} else {
idle = cpu_idle_thread[cpu];
init_idle(idle, cpu);
}
mp_ops->
boot_secondary
(cpu, idle);
/*
* Trust is futile. We should really have timeouts ...
*/
while (!cpu_isset(cpu, cpu_callin_map))
udelay(100);
cpu_set(cpu, cpu_online_map);
return 0;
}
static void
bmips_boot_secondary
(int cpu, struct task_struct *idle)
{
bmips_smp_boot_sp = __KSTK_TOS(idle);
bmips_smp_boot_gp = (unsigned long)task_thread_info(idle);
mb();
/*
* Initial boot sequence for secondary CPU:
* bmips_reset_nmi_vec @ a000_0000 ->
* bmips_smp_entry ->
* plat_wired_tlb_setup (cached function call; optional) ->
* start_secondary (cached jump)
*
* Warm restart sequence:
* play_dead WAIT loop ->
* bmips_smp_int_vec @ BMIPS_WARM_RESTART_VEC ->
* eret to play_dead ->
* bmips_secondary_reentry ->
* start_secondary
*/
pr_info("SMP: Booting CPU%d...\n", cpu);
if (cpumask_test_cpu(cpu, &bmips_booted_mask)) {
/* kseg1 might not exist if this CPU enabled XKS01 */
bmips_set_reset_vec(cpu, RESET_FROM_KSEG0);
bmips_send_ipi_single(cpu, 0); //如果cpu已经在运行了，则软复位cpu
} else {
bmips_set_reset_vec(cpu, RESET_FROM_KSEG1); //设置cpu复位的异常向量位置
#if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380)
set_c0_brcm_cmt_ctrl(0x01); //复位cpu
#elif defined(CONFIG_BCM7435A0)
if (cpu & 0x01)
write_c0_brcm_action(ACTION_BOOT_THREAD(cpu));
else {
/*
* core N thread 0 was already booted; just
* pulse the NMI line
*/
bmips_write_zscm_reg(0x210, 0xc0000000);
udelay(10);
bmips_write_zscm_reg(0x210, 0x00);
}
#elif defined(CONFIG_CPU_BMIPS5000)
write_c0_brcm_action(ACTION_BOOT_THREAD(cpu));
#endif
cpumask_set_cpu(cpu, &bmips_booted_mask);
}
}
static void
bmips_set_reset_vec
(int cpu, u32 val)
{
struct reset_vec_info info;
if (current_cpu_type() == CPU_BMIPS5000) {
/* this needs to run from CPU0 (which is always online) */
info.cpu = cpu;
info.val = val;
bmips_set_reset_vec_remote(&info);
} else {
void __iomem *cbr = BMIPS_GET_CBR();
if (cpu == 0)
__raw_writel(val, cbr + BMIPS_RELO_VECTOR_CONTROL_0);
else {
if (current_cpu_type() != CPU_BMIPS4380)
return;
__raw_writel(val, cbr +
BMIPS_RELO_VECTOR_CONTROL_1
);
}
}
__sync();
back_to_back_c0_hazard();
}
复位异常向量是如何安装的呢
static inline void __cpuinit
bmips_nmi_handler_setup
(void)
{
bmips_wr_vec(BMIPS_NMI_RESET_VEC, &
bmips_reset_nmi_vec
,
&bmips_reset_nmi_vec_end);
bmips_wr_vec(BMIPS_WARM_RESTART_VEC, &bmips_smp_int_vec,
&bmips_smp_int_vec_end);
}
void __cpuinit bmips_ebase_setup(void)
{
....
board_nmi_handler_setup = &bmips_nmi_handler_setup;
}
trap_init()
{
....
if (board_nmi_handler_setup)
board_nmi_handler_setup();
}
而trap_init（）是在start_kernel（）中调用的。
cpu复位后进入异常处理执行bmips_reset_nmi_vec，下面来看看bmips_reset_nmi_vec的实现：
arch/mips/kernel/bmips_vec.S
/***********************************************************************
* Reset/NMI vector
* For BMIPS processors that can relocate their exception vectors, this
* entire function gets copied to 0x8000_0000.
***********************************************************************/
NESTED(
bmips_reset_nmi_vec
, PT_SIZE, sp)
.set push
.set noat
.align 4
#ifdef CONFIG_SMP
/* if the NMI bit is clear, assume this is a soft reset */
li k1, (1 << 19)
mfc0 k0, CP0_STATUS
and k0, k1
beqz k0, soft_reset
#if defined(CONFIG_CPU_BMIPS5000)
/* if we're not on core 0, this must be the SMP boot signal */
li k1, (3 << 25)
mfc0 k0, $22
and k0, k1
bnez k0, bmips_smp_entry
#endif
#endif /* CONFIG_SMP */
/* nope, it's just a regular NMI */
SAVE_ALL
move a0, sp
/* clear EXL, ERL, BEV so that TLB refills still work */
mfc0 k0, CP0_STATUS
li k1, ST0_ERL | ST0_EXL | ST0_BEV | ST0_IE
or k0, k1
xor k0, k1
mtc0 k0, CP0_STATUS
BARRIER
/* jump to the NMI handler function */
la k0, nmi_handler
jr k0
RESTORE_ALL
.set mips3
eret
/***********************************************************************
* CPU1 reset vector (used for the initial and warm boot only)
* This is still part of bmips_reset_nmi_vec().
***********************************************************************/
#ifdef CONFIG_SMP
soft_reset:
#if defined(CONFIG_CPU_BMIPS5000) && defined(CONFIG_BCM7435)
/* if running on TP 1, jump to bmips_smp_entry */
mfc0 k0, $22
li k1, (1 << 24)
and k1, k0
bnez k1, bmips_smp_entry
nop
/*
* running on TP0, can not be core 0 (the boot core).
* Check for soft reset. Indicates a warm boot
*/
mfc0 k0, $12
li k1, (1 << 20)
and k0, k1
beqz k0, bmips_smp_entry
/*
* Warm boot.
* Cache init is only done on TP0
*/
la k0, bmips_5xxx_init
jalr k0
nop
#if !defined(CONFIG_BCM7435A0)
b bmips_smp_entry
nop
#else
/* wait for nmi interrupt from start_secondary */
1:
wait
b 1b
nop
#endif
#endif
bmips_smp_entry:
/* set up CP0 STATUS; enable FPU */
li k0, 0x30000000
mtc0 k0, CP0_STATUS
BARRIER
/* set local CP0 CONFIG to make kseg0 cacheable, write-back */
mfc0 k0, CP0_CONFIG
ori k0, 0x07
xori k0, 0x04
mtc0 k0, CP0_CONFIG
#if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380)
/* initialize CPU1's local I-cache */
li k0, 0x80000000
li k1, 0x80010000
mtc0 zero, $28
mtc0 zero, $28, 1
BARRIER
1: cache Index_Store_Tag_I, 0(k0)
addiu k0, 16
bne k0, k1, 1b
#elif defined(CONFIG_CPU_BMIPS5000)
/* set exception vector base */
la k0, ebase
lw k0, 0(k0)
mtc0 k0, $15, 1
BARRIER
#endif
/* jump back to kseg0 in case we need to remap the kseg1 area */
la k0, 1f
jr k0
1:
la k0, bmips_enable_xks01
jalr k0
/* use temporary stack to set up upper memory TLB */
li sp, BMIPS_WARM_RESTART_VEC
la k0, plat_wired_tlb_setup
jalr k0
/* switch to permanent stack and continue booting */
.global bmips_secondary_reentry
bmips_secondary_reentry:
la k0, bmips_smp_boot_sp
lw sp, 0(k0)
la k0, bmips_smp_boot_gp
lw gp, 0(k0)
la k0, start_secondary
jr k0
#endif /* CONFIG_SMP */
.align 4
.global bmips_reset_nmi_vec_end
bmips_reset_nmi_vec_end:
END(bmips_reset_nmi_vec)
.set pop
.previous
arch/mips/kernel/smp.c
/*
* First C code run on the secondary CPUs after being started up by
* the master.
*/
asmlinkage __cpuinit void
start_secondary
(void)
{
unsigned int cpu;
#ifdef CONFIG_MIPS_MT_SMTC
/* Only do cpu_probe for first TC of CPU */
if ((read_c0_tcbind() & TCBIND_CURTC) == 0)
#endif /* CONFIG_MIPS_MT_SMTC */
cpu_probe();
cpu_report();
per_cpu_trap_init();
mips_clockevent_init();
mp_ops->init_secondary();
/*
* XXX parity protection should be folded in here when it's converted
* to an option instead of something based on .cputype
*/
calibrate_delay();
preempt_disable();
cpu = smp_processor_id();
cpu_data[cpu].udelay_val = loops_per_jiffy;
notify_cpu_starting(cpu);
mp_ops->smp_finish();
set_cpu_sibling_map(cpu);
cpu_set(cpu, cpu_callin_map);
synchronise_count_slave();
cpu_idle
();
}