zynqmp APU（linux）和RPU(裸机)通过IPI中断实现同步

从内核ipi_mailbox源码触发自己实现APU（linux）和RPU(裸机)核间同步操作

APU核RPU可以通过共享内存交互数据，APU向共享内存写数据后，RPU收到ipi中断,然后读取完数据，将触发中断告知APU。以此实现通步。

需要更改的linux驱动基于platform框架实现如下：

#include 
#include 
#include 
#include 
#include 
#include 
#include  
#include 
#include 
#include 
#include 
#include 
#include 
#include  

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
 
#include <../../arch/arm/mach-zynq/common.h>
#include  

/* IPI agent ID any */
#define IPI_ID_ANY 0xFFUL

/* indicate if ZynqMP IPI mailbox driver uses SMC calls or HVC calls */
#define USE_SMC 0
#define USE_HVC 1

#define IPI_ALL_IRQ_MSK  (0x0F0F0301U)
/* Default IPI SMC function IDs */
#define SMC_IPI_MAILBOX_OPEN		0x82001000U
#define SMC_IPI_MAILBOX_RELEASE		0x82001001U
#define SMC_IPI_MAILBOX_STATUS_ENQUIRY	0x82001002U
#define SMC_IPI_MAILBOX_NOTIFY		0x82001003U
#define SMC_IPI_MAILBOX_ACK		0x82001004U
#define SMC_IPI_MAILBOX_ENABLE_IRQ	0x82001005U
#define SMC_IPI_MAILBOX_DISABLE_IRQ	0x82001006U

/* IPI SMC Macros */
#define IPI_SMC_ENQUIRY_DIRQ_MASK	0x00000001UL /* Flag to indicate if
						      * notification interrupt
						      * to be disabled.
						      */
#define IPI_SMC_ACK_EIRQ_MASK		0x00000001UL /* Flag to indicate if
						      * notification interrupt
						      * to be enabled.
						      */

/* IPI mailbox status */
#define IPI_MB_STATUS_IDLE		0
#define IPI_MB_STATUS_SEND_PENDING	1
#define IPI_MB_STATUS_RECV_PENDING	2

#define IPI_MB_CHNL_TX	0 /* IPI mailbox TX channel */
#define IPI_MB_CHNL_RX	1 /* IPI mailbox RX channel */

struct zynqmp_ipi_message g_msg={
	.len=8,
	.data={0xaa,0xaa,0xaa},
};
#define DRIVER_NAME  "my_ipi"

struct ipi_dev_t{
	int status;
	dev_t devid;
	struct cdev cdev;
	struct class * class;
	struct device *device;
	struct device_node *nd;
	int local_id;
	int remote_id;
	int method;
	int irq;
	spinlock_t spinlock;
};

struct ampipidevice_local 
{
	int cpu0_to_cpu1_ipi;
	int cpu1_to_cpu0_ipi;
	unsigned long mem_start;
	unsigned long mem_end;
	void __iomem *base_addr;
};

//核间中断最重要的接口
 static void zynqmp_ipi_fw_call(struct ipi_dev_t *ipi_dev,
			       unsigned long a0, unsigned long a3,
			       struct arm_smccc_res *res)
{
	struct ipi_dev_t *pdata = ipi_dev;
	unsigned long a1, a2;

	a1 = pdata->local_id;
	a2 = pdata->remote_id;
	if (pdata->method == USE_SMC)
		arm_smccc_smc(a0, a1, a2, a3, 0, 0, 0, 0, res);
	else
		arm_smccc_hvc(a0, a1, a2, a3, 0, 0, 0, 0, res);
}
 

static int myipi_open(struct inode *inode, struct file *filp) 
{ 
	printk("myipi Dev: open success  \r\n"); 
	struct ipi_dev_t *ipi_dev = container_of(inode->i_cdev,struct ipi_dev_t,cdev); //cdev是ipi_dev_t结构内的cdev
	filp->private_data = ipi_dev;
	return 0; 
} 
static ssize_t myipi_write(struct file *filp,const char __user *buf,size_t cnt, loff_t *offt) 
{
	int ret; 
	u8 srcdata[8]={0};
	printk("buf[0]=%d\n",buf[0]);
	ret = copy_from_user(srcdata, buf, cnt);  
	struct ipi_dev_t *ipi_dev =  filp->private_data;
	struct zynqmp_ipi_message *msg = NULL;//srcdata;
	u64 arg0;
	struct arm_smccc_res res;

	//if (msg && msg->len)
	//	memcpy_toio(mchan->req_buf, msg->data, msg->len);
	/* Kick IPI mailbox to send message */
	arg0 = SMC_IPI_MAILBOX_NOTIFY;
	zynqmp_ipi_fw_call(ipi_dev, arg0, 0, &res);   //触发ipi中断给到RPU1 CH2
		
	printk("ampipi Dev: write add: %d  cnt: %d\r\n",srcdata[0],cnt); 

	return cnt;
}
 
static ssize_t myipi_read(struct file *filp,char __user *buf,size_t cnt, loff_t *offt) 
{
	int ret; 
	ret = copy_to_user(buf, (unsigned char*)(g_msg.data) , cnt); 
	g_msg.data[0] = 0x0;
	printk("myipi Dev:%d read : %d  ret: %d\r\n",g_msg.data[0],buf[0],ret); 

	return ret;
}
 static ssize_t myipi_ioctl(struct file *filp,const char __user *buf,size_t cnt, loff_t *offt) 
 {
	return 0;
 }
 static ssize_t myipi_llseek(struct file *filp,const char __user *buf,size_t cnt, loff_t *offt) 
 {
	return 0;
 }
 static ssize_t myipi_fasync(struct file *filp,const char __user *buf,size_t cnt, loff_t *offt) 
 {
	return 0;
 }
 static ssize_t myipi_fasync_release(struct file *filp,const char __user *buf,size_t cnt, loff_t *offt) 
 {
	return 0;
 }
 
 static struct file_operations myipi_fops = 
{ 
	.owner = THIS_MODULE, 
	.open = myipi_open, 
	.write = myipi_write, 
	.read = myipi_read, 
	//.unlocked_ioctl = myipi_ioctl, 
	//.llseek = myipi_llseek, 
	//.fasync = myipi_fasync, 
	//.release = myipi_fasync_release, 
 }; 
 
static irqreturn_t zynqmp_ipi_interrupt(int irq, void *data) //当RPU 触发ipi中断，这里接收到后处理完返回IRQ_HANDLED，后续还是会一直进来，但是RPU1只触发了一次，不知道是为什么？？？
{
	struct ipi_dev_t *pdata = data;
	//struct mbox_chan *chan;
	//struct zynqmp_ipi_mbox *ipi_mbox;
	//struct zynqmp_ipi_mchan *mchan;
	//struct zynqmp_ipi_message *msg;
	u64 arg0, arg3;
	struct arm_smccc_res res;
	int ret, i;

	(void)irq;
	arg0 = SMC_IPI_MAILBOX_STATUS_ENQUIRY;
	arg3 = IPI_SMC_ENQUIRY_DIRQ_MASK;
	
	zynqmp_ipi_fw_call(pdata, arg0, arg3, &res);  //这里会触发中断给RPU，如果不触发，可以屏蔽
	ret = (int)(res.a0 & 0xFFFFFFFF);
	printk("Recv ipi ret=%08x\n",ret);
	//g_msg.data[0] = 0x1e;
	if (ret > 0 && ret & IPI_MB_STATUS_RECV_PENDING) {  
		printk("Recv ipi msg\n");
		g_msg.data[0] = 0x1e;   //用户程序通过这个标志是否非0 ，可以知道RPU是否读取完共享内存数据
		/*if (mchan->is_opened) {
			msg = mchan->rx_buf;
			msg->len = mchan->req_buf_size;
			memcpy_fromio(msg->data, mchan->req_buf,
					  msg->len);
			mbox_chan_received_data(chan, (void *)msg);
			return IRQ_HANDLED;
		}*/
		return IRQ_HANDLED;
	}
	
	return IRQ_NONE;
}
 
static int my_ipi_probe(struct platform_device *pdev)
{
	struct device *dev = &pdev->dev;
	struct device_node *nc, *np = pdev->dev.of_node;

	struct ipi_dev_t *ipi_dev;
	int num_mboxes, ret = -EINVAL;
	struct resource *r_mem; /* IO mem resources */
	
	/*r_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (!r_mem) 
	{
		dev_err(dev, "invalid address\n");
		return -ENODEV;
	}*/
	
	ipi_dev = devm_kzalloc(dev, sizeof(*ipi_dev), GFP_KERNEL);
	
	/* Get the IPI local agents ID */
	ret = of_property_read_u32(np, "xlnx,ipi-id", &ipi_dev->local_id);
	if (ret < 0) {
		dev_err(dev, "No IPI local ID is specified.\n");
		return ret;
	}
	ret = of_property_read_u32(np, "xlnx,ipi-remoteid", &ipi_dev->remote_id);
	if (ret < 0) {
		dev_err(dev, "No IPI remote ID is specified.\n");
		return ret;
	}

	/* IPI IRQ */
	ret = platform_get_irq(pdev, 0);
	if (ret < 0)
		goto free_mbox_dev;

	ipi_dev->irq = ret;
	ret = devm_request_irq(dev, ipi_dev->irq, zynqmp_ipi_interrupt,
			      IRQF_TRIGGER_HIGH, dev_name(dev), ipi_dev);  //中断核tirg触发,当中断注册正常用cat /proc/interrupt中查看注册好的中断名称
	//ret = request_irq( ipi_dev->irq, zynqmp_ipi_interrupt, IRQF_TRIGGER_RISING, "ipi irq", NULL);
	if (ret)
	{
		printk("request irq error!\n");
		return -1;
	}
	if (ret) {
		dev_err(dev, "IRQ %d is not requested successfully.\n",
			ipi_dev->irq);
		goto free_mbox_dev;
	}

	ret = alloc_chrdev_region(&ipi_dev->devid,0, 1, DRIVER_NAME);
	if(ret<0)
	{
		printk("my_ipi aollc_chrdev_region fail\n");
	}
	ipi_dev->cdev.owner = THIS_MODULE;
	cdev_init(&ipi_dev->cdev,&myipi_fops);
	
	ret = cdev_add(&ipi_dev->cdev,ipi_dev->devid,1);
	if(ret<0)
	{
		printk("myipi  cdev_add fail\n");
		goto del_cdev_init;
	}
	ipi_dev->class = class_create(THIS_MODULE,DRIVER_NAME);
	if(IS_ERR(ipi_dev->class))
	{
		printk("myipi class create fail\n");
		goto del_cdev_add; 
	}
	ipi_dev->device = device_create(ipi_dev->class,NULL,ipi_dev->devid,NULL,DRIVER_NAME);
	if(IS_ERR(ipi_dev->device))
	{
		printk("myipi device create fail\n");
		goto del_cdev_class;
		
	}
	printk("myipi device create success\n");
	platform_set_drvdata(pdev, ipi_dev);
	return ret;

free_mbox_dev:
	//zynqmp_ipi_free_mboxes(pdata);
del_cdev_class:
	class_destroy(ipi_dev->class);

del_cdev_add:
	cdev_del(&ipi_dev->cdev);

del_cdev_init:
	unregister_chrdev_region(ipi_dev->devid,1);
	kfree(ipi_dev);

	return ret;

}

static int my_ipi_remove(struct platform_device *pdev)
{
	struct ipi_dev_t *pdata;

	pdata = platform_get_drvdata(pdev);
	//zynqmp_ipi_free_mboxes(pdata);

	return 0;
}

static const struct of_device_id ipi_of_match[] = {
	{.compatible = "xlnx,my_ipi_test"},
	{}	
};
MODULE_DEVICE_TABLE(of,ipi_of_match);

static struct platform_driver my_ipi_driver={
	.probe = my_ipi_probe,
	.remove = my_ipi_remove,
	.driver = {
		.name = "myipi_driver",
		.of_match_table = of_match_ptr(ipi_of_match),
	},

};

//module_platform_driver(my_ipi_driver);
static int __init my_ipi_init(void)
{
	return platform_driver_register(&my_ipi_driver);
}
//subsys_initcall(my_ipi_init);
module_init(my_ipi_init);

static void __exit my_ipi_exit(void)
{
	platform_driver_unregister(&my_ipi_driver);
}
module_exit(my_ipi_exit);

MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("xilinx my ipi driver test");
MODULE_AUTHOR("MYIPI Inc.");

设备树配置：

 myipi_instance: myipimod@0 {
		compatible = "xlnx,my_ipi_test";
 		interrupt-parent = <&gic>;
 		interrupts = <0 30 4>;
		xlnx,ipi-id = <0>;   //APU本地id ，通过xsa设置的ipi通道号
	 	xlnx,ipi-remoteid = <2>;  //RPU本地id ，通过xsa设置的ipi通道号
	};

RPU  ipi中断可以参考官方例子，vitis生成的例子

#include "ipi.h"
#include 
#include 
#include 

#define TEST_MSG_LEN  (2)
#define XIPIPSU_IPI2_TO_IPI0_MASK (0xf000001)  //Ch7~10 Ch0 enable ,PMU Chx disable  查看寄存器手册ug1087 ipi中断寄存器部分
#define XIPIPSU_IPI0_TO_IPI2_MASK (0x0000301)   //指示从哪个通道产生trig中断或是发送trig中断
XIpiPsu* GetaIPIInstancePtr(void)
{
	static XIpiPsu xipiinst;
	return &xipiinst;
}

int preInitIPICtrl(  )
{
	XIpiPsu *xipiinst = GetaIPIInstancePtr();
	XScuGic* pGic = GetScuGicInstancePtr();
	/* Look Up the config data */
	XIpiPsu_Config* CfgPtr = XIpiPsu_LookupConfig(PSU_IPI_DEVICE_ID);

	/* Init with the Cfg Data */
	XIpiPsu_CfgInitialize(xipiinst, CfgPtr, CfgPtr->BaseAddress);

	/* Setup the GIC */
	//SetupInterruptSystem(&GicInst, xipiinst, (xipiinst->Config.IntId));

	XScuGic_InterruptMaptoCpu( pGic, XScuGic_GetCpuID(), (PSU_IPI_INTC_DEVICE_ID) );
	if( XST_SUCCESS == XScuGic_Connect(
			pGic, (PSU_IPI_INTC_DEVICE_ID),
			(Xil_InterruptHandler) IpiIntrHandler,
			xipiinst) );
	else
	{
		printf("interupt fail\n");
	}

	XScuGic_Enable( pGic, (PSU_IPI_INTC_DEVICE_ID) );
	/* Enable reception of IPIs from all CPUs */
	XIpiPsu_InterruptEnable(xipiinst, XIPIPSU_IPI0_TO_IPI2_MASK);   //使能自己所在的ipi通道,APU出发的Trig中断会到这里

	/* Clear Any existing Interrupts */
	XIpiPsu_ClearInterruptStatus(xipiinst, XIPIPSU_IPI0_TO_IPI2_MASK);

	/* Call the test routine */
	return true;
}

void IpiIntrHandler(void *XIpiPsuPtr)  //手动添加中断回调函数
{
	u32 IpiSrcMask; /**< Holds the IPI status register value */
	u32 Index;

	u32 TmpBufPtr[TEST_MSG_LEN] = { 0 }; /**< Holds the received Message, later inverted and sent back as response*/

	u32 SrcIndex;
	XIpiPsu *InstancePtr = (XIpiPsu *) XIpiPsuPtr;

	printf("----++++++++++++++++++++++++++++++++>Enter Interrupt Handler\r\n");

	Xil_AssertVoid(InstancePtr!=NULL);

	IpiSrcMask = XIpiPsu_GetInterruptStatus(InstancePtr);

	/* Poll for each source and send Response (Response = ~Msg) */

	for (SrcIndex = 0U; SrcIndex < InstancePtr->Config.TargetCount;
			SrcIndex++) {

		if (IpiSrcMask & InstancePtr->Config.TargetList[SrcIndex].Mask) {

			/*  Read Incoming Message Buffer Corresponding to Source CPU */
			XIpiPsu_ReadMessage(InstancePtr,
					InstancePtr->Config.TargetList[SrcIndex].Mask, TmpBufPtr,
					TEST_MSG_LEN, XIPIPSU_BUF_TYPE_MSG);

			printf("Message Received:SrcIndex=%d,IpiSrcMask=%08x\r\n",SrcIndex,IpiSrcMask);

			for (Index = 0; Index < TEST_MSG_LEN; Index++) {
				printf("W%d: 0x%08x\r\n", Index, TmpBufPtr[Index]);
			}

			/*Process the Received Message */

			/* Send Response */
			XIpiPsu_WriteMessage(InstancePtr,
					InstancePtr->Config.TargetList[SrcIndex].Mask, TmpBufPtr,
					TEST_MSG_LEN, XIPIPSU_BUF_TYPE_RESP);
			printf("Sent back Inverted Message.=%08x\r\n",InstancePtr->Config.TargetList[SrcIndex].Mask);

			/* Clear the Interrupt Status - This clears the OBS bit on the SRC CPU registers */
			XIpiPsu_ClearInterruptStatus(InstancePtr,
					XIPIPSU_ALL_MASK);
			{
				XIpiPsu_Config *DestCfgPtr;
				DestCfgPtr = XIpiPsu_LookupConfig(PSU_IPI_DEVICE_ID);
				//XIpiPsu_TriggerIpi(InstancePtr, DestCfgPtr->BitMask);  //触发自身的通道中断
        //通过读写ocm地址可以传递数据 0xfffcexxx
        
				XIpiPsu_TriggerIpi(InstancePtr,XIPIPSU_IPI2_TO_IPI0_MASK);  //出发ipi中断到APU所在的通道
			}
		}
	}
	XIpiPsu_ClearInterruptStatus(InstancePtr,
						XIPIPSU_ALL_MASK);
	printf("<----Exit Interrupt Handler\r\n");
}

bool IPIStartSend(int32_t BaseAddr,int32_t* Data,uint32_t size,int32_t offset)
{
	return true;
}

void IPIStop()
{

}

void IPIReset()
{
	XIpiPsu_Reset(GetaIPIInstancePtr());
}

rpu ipi.h

#ifndef _POW_DRIVER_IPI_H_
#define _POW_DRIVER_IPI_H_

#include "xipipsu.h"
#include "xipipsu_hw.h"
#include "driver/intc/intc.h"

#define PSU_IPI_DEVICE_ID		XPAR_XIPIPSU_0_DEVICE_ID /* ipi device Id */
#define PSU_IPI_INTC_DEVICE_ID	XPAR_XIPIPSU_0_INT_ID

XIpiPsu* GetaIPIInstancePtr(void);

int preInitIPICtrl( );
void IpiIntrHandler(void *XIpiPsuPtr);
uint32_t DoIpiTest();

bool IPIStartSend(int32_t BaseAddr,int32_t* Data,uint32_t size,int32_t offset);

void IPIStop();

void IPIReset();

#endif

makefile 编写 

CC = aarch64-linux-gnu-
target:
		$(CC)gcc -o testipi testipi.c -lpthread -static
		
clean:
		rm -rf testipi