Cavium SSL加速卡驱动移植到Ubuntu
因为cavium ssl加速卡的驱动指定的系统是Fedora，而我们的系统是Ubuntu，所以肯定要移植了，看一下Release_Notes.txt里面的Supported Features：
Supported Features
==================
System architecture
-------------------
- 32/64 bit Intel/AMD platforms
- Little Endian
- SMP and Non-SMP
Operating System and Kernel versions
------------------------------------
OS:
- Fedora Core 12
Kernel Versions:
- v2.6.31
- v2.6.32
- v2.6.341、源码
需要修改的文件有cavium_proc.c、linux_main.c、linux_sysdep.h、Makefile、Makefile.Linux，用Beyond Compare比较如下：
static int
proc_write_timeout(struct file *file, const char *buffer,
unsigned long count, void *data)
{
char str[count+1], *strend;
Uint32 timeout;
CAVIUM_MOD_INC_USE_COUNT;
if(cavium_copy_in(str, buffer, count))
{
cavium_error("cavium_copy_out failed\n");
return -EFAULT;
}
str[count] = '\0';
timeout = simple_strtoul(str, &strend, 0);
if (timeout > (2*60*60))
{
cavium_error("Timeout too high. The maximum value is 7200\n");
return -EFAULT;
}
cavium_command_timeout = timeout*HZ;
CAVIUM_MOD_DEC_USE_COUNT;
return count;
}
static int
proc_read_timeout(char *page, char **start, off_t off, int count,
int *eof, void *data)
{
int len = 0;
CAVIUM_MOD_INC_USE_COUNT;
len = sprintf(page + len, "%d\n", (cavium_command_timeout/HZ));
CAVIUM_MOD_DEC_USE_COUNT;
return len;
}
int cavium_init_proc(void)
{
int rv = 0;
/* create directory /proc/cavium */
cavium_dir = proc_mkdir("cavium", NULL);
if(cavium_dir == NULL) {
rv = -ENOMEM;
goto out;
}
#if LINUX_VERSION_CODE < KERNEL_VERSION (2,6,30)
cavium_dir->owner = THIS_MODULE;
#endif
/* create timeout file */
timeout_file = create_proc_entry("command_timeout", 0644, cavium_dir);
if(timeout_file == NULL)
{
rv = -ENOMEM;
goto out;
}
timeout_file->data = &cavium_command_timeout;
timeout_file->read_proc = proc_read_timeout;
timeout_file->write_proc = proc_write_timeout;
#if LINUX_VERSION_CODE < KERNEL_VERSION (2,6,30)
timeout_file->owner = THIS_MODULE;
#endif
……
}而在3.10以及3.10之后的内核版本proc文件系统是这么用的：
static ssize_t proc_write_timeout(struct file *file, const char __user *buffer,
size_t count, loff_t *ppos)
{
Uint32 timeout;
unsigned long val;
int err;
CAVIUM_MOD_INC_USE_COUNT;
err = kstrtoul_from_user(buffer, count, 0, &val);
if (err)
return err;
timeout = val;
if (timeout > (2*60*60)) {
cavium_error("Timeout too high. The maximum value is 7200\n");
return -EFAULT;
}
cavium_command_timeout = timeout*HZ;
CAVIUM_MOD_DEC_USE_COUNT;
return count;
}
static int proc_read_timeout(struct seq_file *m, void *v)
{
int len = 0;
CAVIUM_MOD_INC_USE_COUNT;
len = seq_printf(m, "%d\n", (cavium_command_timeout/HZ));
CAVIUM_MOD_DEC_USE_COUNT;
return len;
}
static struct file_operations cavium_proc_timeout_operations = {
.open = open_proc_timeout,
.read = seq_read,
.llseek = seq_lseek,
.write = proc_write_timeout,
};
int cavium_init_proc(void)
{
int rv = 0;
/* create directory /proc/cavium */
cavium_dir = proc_mkdir("cavium", NULL);
if(cavium_dir == NULL) {
rv = -ENOMEM;
goto out;
}
timeout_file = proc_create("command_timeout", 0644, cavium_dir, &cavium_proc_timeout_operations);
if(timeout_file == NULL)
{
rv = -ENOMEM;
goto out;
}
……
}文件linux_main.c改得不多，主要是加几个宏定义就可以了，这是因为在内核版本3.7.0之后内核去掉了devinit、devinitdata、devexit_p这几个宏，我们加上这几个宏的定义就可以了：
#if LINUX_VERSION_CODE > KERNEL_VERSION (3,7,0)
#define __devinit
#define __devinitdata
#define __devexit_p(x) x
#endif
文件driver\linux\Makefile做了只能在2.6内核上编译的限制，要在新版Ubuntu（内核版本3.13）上编译，肯定要去掉这个限制：
文件Makefile.Linux也做了一些修改：
2、编译
经过上面源码的修改，再看看linux_install_driver_pf.txt基本上make clean和make一下就可以了，如果还出错那就再根据出错信息去做相应的修改就可以了。编译还是很简单的。
3、加载
编译成功了的话，在编译目录的bin底下应该有驱动模块：
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK# ls -l bin/
total 0
lrwxrwxrwx 1 root root 69 Apr 14 14:29 csp1_init -> /root/work/cavium/Cavium_Networks/NITROX-SDK/utils/csp1_init*
lrwxrwxrwx 1 root root 71 Apr 14 14:29 init_nitrox -> /root/work/cavium/Cavium_Networks/NITROX-SDK/utils/init_nitrox*
lrwxrwxrwx 1 root root 77 Apr 14 14:29 pkp_drv.ko -> /root/work/cavium/Cavium_Networks/NITROX-SDK/driver/linux/pkp_drv.ko
然后在bin目录底下执行命令：./init_nitrox ssl就可以加载驱动模块了，如果不出意外，基本上就完事了，但在Ubuntu一些版本上总是出现硬件无法分配资源以及驱动初始化失败的问题：
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# lspci -vvv -s 05:00.0
05:00.0 Network and computing encryption device: Cavium Networks Device 0011 (rev 01)
Subsystem: Cavium Networks Device 0011
Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- SERR- (64-bit, prefetchable)
Capabilities: [40] Power Management version 3
Flags: PMEClk- DSI- D1- D2- AuxCurrent=0mA PME(D0-,D1-,D2-,D3hot-,D3cold-)
Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=0 PME-
Capabilities: [50] MSI: Enable- Count=1/1 Maskable+ 64bit+
Address: 0000000000000000 Data: 0000
Masking: 00000000 Pending: 00000000
Capabilities: [70] Express (v2) Endpoint, MSI 00
DevCap: MaxPayload 256 bytes, PhantFunc 0, Latency L0s <1us, L1 <8us
ExtTag+ AttnBtn- AttnInd- PwrInd- RBE+ FLReset+
DevCtl: Report errors: Correctable- Non-Fatal+ Fatal+ Unsupported+
RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+ FLReset-
MaxPayload 256 bytes, MaxReadReq 512 bytes
DevSta: CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr- TransPend-
LnkCap: Port #0, Speed 5GT/s, Width x16, ASPM L0s L1, Exit Latency L0s <1us, L1 <64us
ClockPM- Surprise- LLActRep- BwNot-
LnkCtl: ASPM Disabled; RCB 64 bytes Disabled- CommClk+
ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
LnkSta: Speed 5GT/s, Width x8, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
DevCap2: Completion Timeout: Not Supported, TimeoutDis+, LTR-, OBFF Not Supported
DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis+, LTR-, OBFF Disabled
LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
LnkSta2: Current De-emphasis Level: -6dB, EqualizationComplete-, EqualizationPhase1-
EqualizationPhase2-, EqualizationPhase3-, LinkEqualizationRequest-
Capabilities: [b0] MSI-X: Enable- Count=19 Masked-
Vector table: BAR=0 offset=00001000
PBA: BAR=0 offset=00002000
Capabilities: [100 v2] Advanced Error Reporting
UESta: DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
UEMsk: DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt+ UnxCmplt+ RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
UESvrt: DLP+ SDES+ TLP+ FCP+ CmpltTO+ CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC+ UnsupReq- ACSViol-
CESta: RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr-
CEMsk: RxErr+ BadTLP+ BadDLLP+ Rollover+ Timeout+ NonFatalErr+
AERCap: First Error Pointer: 00, GenCap+ CGenEn- ChkCap+ ChkEn-
Capabilities: [140 v1] Alternative Routing-ID Interpretation (ARI)
ARICap: MFVC- ACS-, Next Function: 0
ARICtl: MFVC- ACS-, Function Group: 0
Capabilities: [150 v1] Single Root I/O Virtualization (SR-IOV)
IOVCap: Migration-, Interrupt Message Number: 000
IOVCtl: Enable- Migration- Interrupt- MSE- ARIHierarchy+
IOVSta: Migration-
Initial VFs: 64, Total VFs: 64, Number of VFs: 0, Function Dependency Link: 00
VF offset: 1, stride: 1, Device ID: 0011
Supported Page Size: 00000553, System Page Size: 00000001
Region 0: Memory at 0000000000000000 (64-bit, prefetchable)
VF Migration: offset: 00000000, BIR: 0
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# dmesg
[ 239.840522] VFCOUNT 0
[ 239.840604] requested mem region for bar 0 cannot be allocated
[ 239.840610] Cavium Init failed for device
[ 239.840644] pkp: probe of 0000:05:00.0 failed with error -12
[ 239.840697] requested mem region for bar 0 cannot be allocated
[ 239.840702] Cavium Init failed for device
[ 239.840719] pkp: probe of 0000:43:00.0 failed with error -12
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# 
这个问题困扰我很长时间，看内核pci驱动初始化代码也没找出是啥问题，后来峰哥在https://bugzilla.kernel.org/show_bug.cgi?id=10461上看到人家只要加一个内核参数就能解决问题了，在我们系统试了一下果然也能解决掉这个问题，加的内核启动参数是pci=nocrs，这个参数对应到代码是在arch/x86/pci/common.c和arch/x86/pci/acpi.c文件中有用到，最后体现在acpi.c的函数pci_acpi_scan_root中：
如果加了内核启动参数pci=nocrs的话，应该执行的是上图else分支，看样子是先释放一些资源再添加一些资源，具体原因就没有往更深层次去挖掘了，因为这个需要熟悉pci一整套流程和玩法才好理解，这个还是比较复杂的，目前功力尚未达到这个境界，先这么着吧。添加内核启动参数只需要在/boot/grub/grub.cfg加一下就可以了：
修改完重启之后加载驱动：
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# ./init_nitrox ssl
insmod ./pkp_drv.ko ssl=0
./csp1_init ssl=0
File: ../microcode/boot_mc2_n3.out
Version = CN35x-MC-Boot-0001
Code length = 14060 Data length = 200
0: name=../microcode/boot_mc2_n3.out, index=0, core=0
File: ../microcode/main_ssl2_n3.out
Version = CNN35x-MC-SSL-0008
Code length = 59620 Data length = 320
1: name=../microcode/main_ssl2_n3.out, index=1, core=0
Microcode Load Succeed
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# dmesg
[ 4151.317917] VFCOUNT 0
[ 4151.318649] Nitrox3 found at Bus 5 Slot 0
[ 4151.318827] Using memory-mapped bar for device 0x177D:0x11
[ 4151.320505] bar 0: d9200000
[ 4151.331446] SERDES_TUNING register value is already set: 0x458cc58
[ 4151.334542] detect_n3_sku: Invalid zip mask 0x0000000f
[ 4151.431688] part number=CNN35XX has 16 cores
[ 4151.431691] Finished Initializing this device
[ 4151.431693] Total Number of Nitrox Devices: 1
[ 4151.432981] Loaded Cavium Driver --- 5.03-
[ 4152.456226] got boot microcode
[ 4152.456243] Code type = 02, code size = 36ec, data size = c8
[ 4152.456246] got mainline microcode
[ 4152.456269] Code type = 01, code size = e8e4, data size = 140
[ 4152.812956] UEN = 0xffff
[ 4152.812960] Final Core Mask = 0x0000ffff
[ 4152.812962] Loaded Boot microcode
[ 4154.315289] Loaded Main microcode
[ 4154.350374] Loading of Microcodes successful
[ 4154.370396]
[ 4154.370396] Core Frequency 500 MHz
root@BJDEVR720:~/work/cavium/Cavium_Networks/NITROX-SDK/bin#
root@rdserver5:~/work/cavium/Cavium_Networks/NITROX-SDK/bin# lspci -vvv -s 05:00.0
05:00.0 Network and computing encryption device: Cavium Networks Device 0011 (rev 01)
Subsystem: Cavium Networks Device 0011
Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- SERR- 可以看出两个Region所在行是正常值而不是ignored了。
整个移植过程完成。