babyos2（30） APIC, local APIC, I/O APIC

Intel从Pentium开始引入APIC（Advanced Programmable Interrupt Controller)，以适应MP（Multiple processor）环境。
Local APIC，在处理器内部
I/O APIC 在PCI-to-ISA bridge（PCI-to-LPC bridge）的LPC控制器内

每个逻辑处理器(logical processor)有自己的local APIC，每个local APIC有一组APIC寄存器，用来控制local和external中断的产生、发送和接收等，也产生IPI（处理器间中断消息）
APIC，xAPIC中local APIC寄存器以内存映射形式映射到物理地址空间。x2APIC则是映射到MSR寄存器组。

简单浏览了下Intel手册中APIC的介绍，不是很好理解，决定一点点测试下，然后再尝试真正使用它来代替8259A做中断控制器。

1.test
1）版本
CPUID.01:EDX[9]表示是否支持APIC
CPUID.01:ECX[21]表示是否支持x2APIC

void local_apic_t::check()
{
    uint32 edx = 0, ecx = 0;
    __asm__ volatile("cpuid" : "=d" (edx), "=c" (ecx) : "a" (0x1) : "memory", "cc");

    console()->kprintf(YELLOW, "**************** check apic **********************\n");
    console()->kprintf(YELLOW, "local_apic check, ecx: %x, edx: %x\n", ecx, edx);
    console()->kprintf(YELLOW, "support APIC:   %s\n", (edx & (1 << 9))  ? "YES" : "NO");
    console()->kprintf(YELLOW, "support x2APIC: %s\n", (ecx & (1 << 21)) ? "YES" : "NO");
}

2）global disable
Using the APIC global enable/disable flag in the IA32_APIC_BASE MSR (MSR address 1BH）

#define rdmsr(msr,val1,val2) \
    __asm__ __volatile__("rdmsr" \
              : "=a" (val1), "=d" (val2) \
              : "c" (msr))

#define wrmsr(msr,val1,val2) \
    __asm__ __volatile__("wrmsr" \
              : /* no outputs */ \
              : "c" (msr), "a" (val1), "d" (val2))

linux的读写msr的代码。

void local_apic_t::global_disable()
{
    uint32 l, h;
    rdmsr(MSR_IA32_APICBASE, l, h);
    l &= ~MSR_IA32_APICBASE_ENABLE;
    wrmsr(MSR_IA32_APICBASE, l, h);
}

3）Local APIC Register Address Map

void local_apic_t::check()
{
    uint32 edx = 0, ecx = 0;
    __asm__ volatile("cpuid" : "=d" (edx), "=c" (ecx) : "a" (0x1) : "memory", "cc");

    console()->kprintf(YELLOW, "**************** check apic **********************\n");
    console()->kprintf(YELLOW, "local_apic check, ecx: %x, edx: %x\n", ecx, edx);
    console()->kprintf(YELLOW, "support APIC:   %s\n", (edx & (1 << 9))  ? "YES" : "NO");
    console()->kprintf(YELLOW, "support x2APIC: %s\n", (ecx & (1 << 21)) ? "YES" : "NO");
    console()->kprintf(YELLOW, "**************** check apic **********************\n");
    console()->kprintf(YELLOW, "\n");

    console()->kprintf(CYAN, "**************** MSR IS_32_APICBASE ***************\n");
    uint32 l, h;
    rdmsr(MSR_IA32_APICBASE, l, h);
    console()->kprintf(CYAN, "MSR IA_32_APICBASE: %x, %x\n", h, l);
    console()->kprintf(CYAN, "**************** MSR IS_32_APICBASE ***************\n");
    console()->kprintf(CYAN, "\n");

    console()->kprintf(WHITE, "******************** local APIC register *****************\n");
    uint64 val = 0;
    val = apic_read(APIC_ID);
    console()->kprintf(WHITE, "APIC ID:                                 %x, %x\n", (uint32) val);
    val = apic_read(APIC_LVR);
    console()->kprintf(WHITE, "APIC version:                            %x, %x\n", (uint32) val);
    val = apic_read(APIC_SPIV);
    console()->kprintf(WHITE, "APIC Spurious interrupt vertor register: %x\n", (uint32) val);
    console()->kprintf(WHITE, "******************** local APIC register *****************\n");
    console()->kprintf(WHITE, "\n");

    console()->kprintf(GREEN, "******************** local vector table *****************\n");
    val = apic_read(APIC_LVT_CMCI);
    console()->kprintf(GREEN, "lvt machine check:       %x\n", (uint32) val);
    val = apic_read(APIC_LVT_TIMER);
    console()->kprintf(GREEN, "lvt timer:               %x\n", (uint32) val);
    val = apic_read(APIC_LVT_THMR);
    console()->kprintf(GREEN, "lvt thermal sensor       %x\n", (uint32) val);
    val = apic_read(APIC_LVT_PMCR);
    console()->kprintf(GREEN, "lvt performance monitor: %x\n", (uint32) val);
    val = apic_read(APIC_LVT_LINT0);
    console()->kprintf(GREEN, "lvt lint0:               %x\n", (uint32) val);
    val = apic_read(APIC_LVT_LINT1);
    console()->kprintf(GREEN, "lvt lint1:               %x\n", (uint32) val);
    val = apic_read(APIC_LVT_ERROR);
    console()->kprintf(GREEN, "lvt error:               %x\n", (uint32) val);
    console()->kprintf(GREEN, "******************** local vector table *****************\n");
    console()->kprintf(GREEN, "\n");

    console()->kprintf(PINK, "*************************** timer ************************\n");
    val = apic_read(APIC_TIMER_ICT);
    console()->kprintf(PINK, "timer initial count register:        %x\n", (uint32) val); 
    val = apic_read(APIC_TIMER_CCT);
    console()->kprintf(PINK, "timer current count register:        %x\n", (uint32) val);
    val = apic_read(APIC_TIMER_DCR);
    console()->kprintf(PINK, "timer divide configuration register: %x\n", (uint32) val);
    console()->kprintf(PINK, "*************************** timer ************************\n");
    console()->kprintf(PINK, "\n");
}

1> IA32_APIC_BASE MSR
读取的结果为0x0000 0000 fee0 0900，它的结构如上图，所以
APIC Base: 0xfee00 （000），这个基地址是可改变的，原因：
This extension of the APIC architecture is provided to help resolve conflicts with memory maps of existing systems and to allow individual processors in an MP system to map their APIC registers to different locations in physical memory.
主要目的是防止跟系统已映射的内存地址冲突，及MP系统中让各个processor有独立的地址映射。

2> local APIC ID
它表示每个逻辑processor在system bus（或APIC bus）上的唯一编号。
因为目前测试在单核处理器上，暂时不管它

3> APIC version

读到的值为：0x0005 0014，表示:
version: 0x14
max lvt entry: 0x05+1
support for EOI-broadcast suppression: 0

4> APIC Spurious interrupt vertor register

读到的值为: 0x0000, 01ff,
EOI-Broadcast Suppression: 0
Focus Processor Checking: 0
APIC Software Enable: 1
Spurious Vector: 0xff

5> local vector table:

local APIC的LVT寄存器能产生中断，LVT LINT0能接收外部8259A的中断请求，
LINT1能接收外部设备的NMI中断请求。

CMCI，表示machine check，是另一个大的主题了，暂不管它
Performance Monitor Counter，是性能监视相关的，暂不管它
Thermal Sensor，可能跟CPU温度相关，暂不管它

local APIC的timer包括：
LVT timer register
initial count register
current count register
divide configuration（clock 频率配置）register

LVT timer只能使用Fixed delivery模式和edge触发。提供3种计数模式：
One-short （一次性）
Periodic（定期）
TSC-deadline（达到TSC值）

LINT0, LINT1读到的结果分别是：0x0000 8700, 0x0000 8400
LINT0, Delivery Mode: 0x7, ExtInt mode
LINT1, Delivery Mode: 0x4, NMI delivery mode
在NMI,SMI, INIT的delivery mode下，LINT0，LINT1固定使用edge触发模式
ExtINT的delivery mode下，固定使用level触发模式。

2.使用APIC timer。
babyos2目前使用的是8254时钟，准备替换成APIC的内部时钟。
使用方法比较简单，divide configuration（clock 频率配置）register里配置timer计数的时钟频率，配置的值为system bus频率处以指定值。

比如0,1,3比特设置为011则时钟频率为system bus频率的1/16.
initial-count跟current-count寄存器为32位，初始化时给initial-count设置一个非零值，它会拷贝到current-count中，并开始递减，到0则产生时钟中断。

void setup_lvt_timer(uint32 clocks)
{
    /* (1 << 17) means timer mode Periodic */
    apic_write(APIC_LVT_TIMER, (1 << 17) | VEC_LOCAL_TIMER);

    /* set timer divide configuration as 111b(0xb's 0,1,3 bit)
     * so the divide value is 1, 
     * so timer's rate will = processor's bus clock / 1
     */
    apic_write(APIC_TIMER_DCR, 0xb);

    /* set timer's initial count as clocks */
    apic_write(APIC_TIMER_ICT, clocks);
}

/* get 8254 timer count, until wraparound */
static void wait_8254_wraparound()
{
    uint32 current_count, prev_count;
    int32 delta = 0;

    current_count = os()->get_arch()->get_8254()->get_timer_count();
    do {
        prev_count = current_count;
        current_count = os()->get_arch()->get_8254()->get_timer_count();
        delta = current_count - prev_count;
    } while (delta < 300);
}

uint32 local_apic_t::calibrate_clock()
{
    uint64 tsc_begin, tsc_end;
    console()->kprintf(CYAN, "calibrating APIC timer...\n");

    /* write a max value to APIC timeout */
    setup_lvt_timer(0xffffffff);

    /* wait 8254 start a new round */
    wait_8254_wraparound();

    /* begin */
    rdtsc64(tsc_begin);
    uint32 apic_begin = apic_read(APIC_TIMER_CCT);

    /* wait */
    for (int i = 0; i < CALIBRATE_LOOP; i++) {
        wait_8254_wraparound();
    }

    /* end */
    rdtsc64(tsc_end);
    uint32 apic_end = apic_read(APIC_TIMER_CCT);

    uint32 clocks = (uint32) (apic_begin - apic_end);
    uint32 tsc_delta = (uint32) ((tsc_end - tsc_begin));

    console()->kprintf(CYAN, "tsc speed: %u.%u MHz.\n", 
            (tsc_delta/CALIBRATE_LOOP) / (1000000/HZ),
            (tsc_delta/CALIBRATE_LOOP) % (1000000/HZ));

    console()->kprintf(CYAN, "bus clock speed: %u.%u MHz.\n", 
            (clocks / CALIBRATE_LOOP) / (1000000/HZ),
            (clocks / CALIBRATE_LOOP) % (1000000/HZ));

    return clocks / CALIBRATE_LOOP;
}

int local_apic_t::init_timer()
{
    uint32 clocks = calibrate_clock();
    setup_lvt_timer(clocks);

    return 0;
}

给它注册一个新的中断处理函数，里面打印下8254中断的tick数，可以发现时钟差不多。

3.通过LINT0, LINT1屏蔽外部中断和NMI
LINT0连接到外部8259中断控制器，可以通过它屏蔽8159中断控制器的外部中断请求，同理可以通过LINT1屏蔽NMI。

int local_apic_t::init()
{
    if (check() != 0) {
        return -1;
    }

    if (init_timer() != 0) {
        return -1;
    }

    /* mask performance monitor counter, LINT0, LINT1 */
    apic_write(APIC_LVT_PMCR,  (1 << 16));
    apic_write(APIC_LVT_LINT0, (1 << 16));
    apic_write(APIC_LVT_LINT1, (1 << 16));

    /* error */
    apic_write(APIC_LVT_ERROR, VEC_ERROR);

    /* ack outstanding interrupts */
    eoi();

    /* enable interrupt on APIC */
    apic_write(APIC_TPR, 0);

    return 0;
}

可以看到，屏蔽LINT0和LINT1之后，8259不会再产生时钟中断，所以tick都是0，同样键盘中断也不能产生，按键盘不会再输出到屏幕，并且因为产生不了IDE硬盘中断，无法读取硬盘，init和shell无法启动。

4.IO APIC

IO APIC通过system bus与local APIC通信。支持24个redirection table。
初始化时将24个表项对应的中断全都disable，其他外部设备准备好开启中断时通过写redirection table启动中断。

/*
 * [email protected]
 * 2018-01-29
 */

#include "io_apic.h"
#include "traps.h"
#include "babyos.h"

#define IO_APIC_REG_ID      0x00
#define IO_APIC_REG_VER     0x01
#define IO_APIC_REG_TABLE   0x10    /* redirection table */

#define INT_DISABLED        0x00010000

void io_apic_t::init()
{
    uint32 count = (read(IO_APIC_REG_VER) >> 16) & 0xff;
    uint32 id = (read(IO_APIC_REG_ID) >> 24);

    console()->kprintf(GREEN, "****************** io apic ****************\n", id);
    console()->kprintf(GREEN, "I/O APIC id: %u\n", id);
    console()->kprintf(GREEN, "I/O APIC num of table: %u\n", count+1);
    console()->kprintf(GREEN, "****************** io apic ****************\n", id);

    for (int i = 0; i <= count; i++) {
        write(IO_APIC_REG_TABLE + 2*i,     INT_DISABLED | (IRQ_0 + i));
        write(IO_APIC_REG_TABLE + 2*i + 1, 0);
    }
}

void io_apic_t::enable_irq(uint32 irq, uint32 cpu_id)
{
    write(IO_APIC_REG_TABLE + 2*irq,     IRQ_0 + irq);
    write(IO_APIC_REG_TABLE + 2*irq + 1, cpu_id << 24);
}

uint32 io_apic_t::read(uint32 reg)
{
    uint32* base = (uint32 *) IO_APIC_BASE;
    *base = reg;
    return *(base + 4);
}

void io_apic_t::write(uint32 reg, uint32 data)
{
    uint32* base = (uint32 *) IO_APIC_BASE;
    *base = reg;
    *(base + 4) = data;
}

可以发现，硬盘中断重新生效（能够从文件系统中读取init, shell 并加载起来），键盘中断重新生效，能够响应输入。

至此，在单核CPU上，成功使用local APIC + I/O APIC 替代了 8259A，使用local APIC的本地时钟替代了8254时钟。

---

下一步的目标：
1.MP（multi processor）的boot
2.MP的配置信息，local APIC，中断控制等
3.MP的schedule