PCIe学习笔记（1）Hot-Plug机制

Hot-Plug Init

PCIe hot-plug是一种支持在不关机情况下从支持的插槽添加或删除设备的功能，PCIe架构定义了一些寄存器以支持原生热插拔。相关寄存器主要分布在Device Capabilities, Slot Capabilities, Slot Control, Slot Status和Slot Capabilities 2。Hot-Plug相关支持寄存器位置如下。

一个Downstream Port支持以下热插拔事件。

• Slot Events:
  • Attention Button Pressed
  • Power Fault Detected
  • MRL Sensor Changed
  • Presence Detect Changed
• Command Completed Events
• Data Link Layer State Changed Events

主要事件都可以通过寄存器控制使能，需要在初始化时配置启用。如果要启用INTx message，需配置如下：

• Command register内的Interrupt Disable位需要设为0
• Slot Control register内的Hot-Plug Interrupt Enable位需要设为1
• 至少使能一个hot-plug event控制寄存器
  如果要启用MSI/MSI-X，则需取消屏蔽关联向量。PME和Hot-Plug事件中断共享同一向量，由PCIE Capabilities的Interrupt Message Number域段控制。
  

Hot-Plug事件产生需要启用PME_En位。

Hot Add Flow

主要处理Presence Detect Changed, Command Completed events（需要在No Command Completed Support为0时，否则硬件支持无通知式自动完成）。
pciehp_handle_presence_or_link_change函数为Presence Detect Changed事件中断服务函数。

void pciehp_handle_presence_or_link_change(struct controller *ctrl, u32 events)
{
    int present, link_active;

    /*
	 * If the slot is on and presence or link has changed, turn it off.
	 * Even if it's occupied again, we cannot assume the card is the same.
	 */
    mutex_lock(&ctrl->state_lock);
    switch (ctrl->state) {
        case BLINKINGOFF_STATE:
            cancel_delayed_work(&ctrl->button_work);
            fallthrough;
        case ON_STATE:
            ctrl->state = POWEROFF_STATE;
            mutex_unlock(&ctrl->state_lock);
            if (events & PCI_EXP_SLTSTA_DLLSC)
                ctrl_info(ctrl, "Slot(%s): Link Down\n",
                      slot_name(ctrl));
            if (events & PCI_EXP_SLTSTA_PDC)
                ctrl_info(ctrl, "Slot(%s): Card not present\n",
                      slot_name(ctrl));
            pciehp_disable_slot(ctrl, SURPRISE_REMOVAL);  //call remove_board
            break;
        default:
            mutex_unlock(&ctrl->state_lock);
            break;
    }

    /* Turn the slot on if it's occupied or link is up */
    mutex_lock(&ctrl->state_lock);
    present = pciehp_card_present(ctrl);
    link_active = pciehp_check_link_active(ctrl);
    if (present <= 0 && link_active <= 0) {
        if (ctrl->state == BLINKINGON_STATE) {
            ctrl->state = OFF_STATE;
            cancel_delayed_work(&ctrl->button_work);
            pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,
                                  INDICATOR_NOOP);
            ctrl_info(ctrl, "Slot(%s): Card not present\n",
                      slot_name(ctrl));
        }
        mutex_unlock(&ctrl->state_lock);
        return;
    }

    switch (ctrl->state) {
        case BLINKINGON_STATE:
            cancel_delayed_work(&ctrl->button_work);
            fallthrough;
        case OFF_STATE:
            ctrl->state = POWERON_STATE;
            mutex_unlock(&ctrl->state_lock);
            if (present)
                ctrl_info(ctrl, "Slot(%s): Card present\n",
                      slot_name(ctrl));
            if (link_active)
                ctrl_info(ctrl, "Slot(%s): Link Up\n",
                      slot_name(ctrl));
            ctrl->request_result = pciehp_enable_slot(ctrl);  //call board_added()
            break;
        default:
            mutex_unlock(&ctrl->state_lock);
            break;
    }
}
pciehp_enable_slot会调用board_added（通过__pciehp_enable_slot）和pciehp_set_indicators
static int pciehp_enable_slot(struct controller *ctrl)
{
    int ret;

    pm_runtime_get_sync(&ctrl->pcie->port->dev);
    ret = __pciehp_enable_slot(ctrl);
    if (ret && ATTN_BUTTN(ctrl))
        /* may be blinking */
        pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,
                              INDICATOR_NOOP);
    pm_runtime_put(&ctrl->pcie->port->dev);

    mutex_lock(&ctrl->state_lock);
    ctrl->state = ret ? OFF_STATE : ON_STATE;
    mutex_unlock(&ctrl->state_lock);

    return ret;
}

board_added完成了pciehp_power_on_slot，pciehp_set_indicators，pciehp_configure_device (Off -> Blink -> On)

static int board_added(struct controller *ctrl)
{
	int retval = 0;
	struct pci_bus *parent = ctrl->pcie->port->subordinate;

	if (POWER_CTRL(ctrl)) {
		/* Power on slot */
		retval = pciehp_power_on_slot(ctrl);
		if (retval)
			return retval;
	}

	pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_BLINK,
			      INDICATOR_NOOP);

	/* Check link training status */
	retval = pciehp_check_link_status(ctrl);
	if (retval)
		goto err_exit;

	/* Check for a power fault */
	if (ctrl->power_fault_detected || pciehp_query_power_fault(ctrl)) {
		ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl));
		retval = -EIO;
		goto err_exit;
	}

	retval = pciehp_configure_device(ctrl);
	if (retval) {
		if (retval != -EEXIST) {
			ctrl_err(ctrl, "Cannot add device at %04x:%02x:00\n",
				 pci_domain_nr(parent), parent->number);
			goto err_exit;
		}
	}

	pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_ON,
			      PCI_EXP_SLTCTL_ATTN_IND_OFF);
	return 0;

err_exit:
	set_slot_off(ctrl);
	return retval;
}

int pciehp_power_on_slot(struct controller *ctrl)
{
	struct pci_dev *pdev = ctrl_dev(ctrl);
	u16 slot_status;
	int retval;

	/* Clear power-fault bit from previous power failures */
	pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status);
	if (slot_status & PCI_EXP_SLTSTA_PFD)
		pcie_capability_write_word(pdev, PCI_EXP_SLTSTA,
					   PCI_EXP_SLTSTA_PFD);
	ctrl->power_fault_detected = 0;

	pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_ON, PCI_EXP_SLTCTL_PCC);
	ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,
		 PCI_EXP_SLTCTL_PWR_ON);

	retval = pciehp_link_enable(ctrl);
	if (retval)
		ctrl_err(ctrl, "%s: Can not enable the link!\n", __func__);

	return retval;
}

Surprise Remove Flow

需要处理AER和DPC events，然后处理hot-plug events。中断触发pciehp_handle_presence_or_link_change，然后pciehp_disable_slot会调用remove_board（通过__pciehp_disable_slot），remove_board主要完成pciehp_power_off_slot和pciehp_set_indicators (On -> Off)。Presense detected需要等待DPC先触发。

static int pciehp_disable_slot(struct controller *ctrl, bool safe_removal)
{
	int ret;

	pm_runtime_get_sync(&ctrl->pcie->port->dev);
	ret = __pciehp_disable_slot(ctrl, safe_removal);
	pm_runtime_put(&ctrl->pcie->port->dev);

	mutex_lock(&ctrl->state_lock);
	ctrl->state = OFF_STATE;
	mutex_unlock(&ctrl->state_lock);

	return ret;
}

static void remove_board(struct controller *ctrl, bool safe_removal)
{
	pciehp_unconfigure_device(ctrl, safe_removal);

	if (POWER_CTRL(ctrl)) {
		pciehp_power_off_slot(ctrl);

		/*
		 * After turning power off, we must wait for at least 1 second
		 * before taking any action that relies on power having been
		 * removed from the slot/adapter.
		 */
		msleep(1000);

		/* Ignore link or presence changes caused by power off */
		atomic_and(~(PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC),
			   &ctrl->pending_events);
	}

	pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,
			      INDICATOR_NOOP);
}

void pciehp_power_off_slot(struct controller *ctrl)
{
	pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_OFF, PCI_EXP_SLTCTL_PCC);
	ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
		 pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,
		 PCI_EXP_SLTCTL_PWR_OFF);
}

NPEM Flow

NPEM(Native PCIe Enclosure Management) 是PCIe标准为NVMe设备定义的扩展LED控制功能，热插拔的LED控制机制由NPEM完成。可以实现在DP内或UP内，用于控制和更新LED状态。软件通过写入NPEM控制寄存器来发出NPEM指令，NPEM控制器根据指令更新LED状态，并使用completed机制指示软件完成。

NPEM支持实现OK/Locate/Fail/Rebuild等可选LED状态。

Linux Reference:
drivers/pci/hotplug /pciehp_hpc.c
driversdrivers/pci/hotplug/pciehp_ctrl.c