使用SG_IO发送SCSI 指令测试底层驱动的scatter-gather 功能

http://gmd20.blog.163.com/blog/static/16843923201002274341552/

Linux中的sg驱动 （“The Linux SCSI Generic (sg) Driver“ http://sg.danny.cz/sg/index.html） 是一个通用SCSI设备的模型，应用程序通过定义的“SG_IO ioctl ”可以像scsi磁盘等设备发送自定义的scsi指令集。  2.6版本的，这个ioctl 将在  block驱动层的 （http://lxr.linux.no/#linux+v2.6.32/block/scsi_ioctl.c 文件的 
static int  sg_io(struct  request_queue * q, struct  gendisk * bd_disk, struct  sg_io_hdr * hdr,  fmode_t  mode)

函数中得到处理，然后组成 request发送的底层的 scsi驱动。
使用的关键数据结构在  /include/scsi/sg.h 里面定义 

  83typedef struct sg_io_hdr
  84{
  85    int interface_id;           /* [i] 'S' for SCSI generic (required) */
  86    int dxfer_direction;        /* [i] data transfer direction  */
  87    unsigned char cmd_len;      /* [i] SCSI command length ( <= 16 bytes) */
  88    unsigned char mx_sb_len;    /* [i] max length to write to sbp */
  89    unsigned short iovec_count; /* [i] 0 implies no scatter gather */
  90    unsigned int dxfer_len;     /* [i] byte count of data transfer */
  91    void __user *dxferp;        /* [i], [*io] points to data transfer memory
  92                                              or scatter gather list */
  93    unsigned char __user *cmdp; /* [i], [*i] points to command to perform */
  94    void __user *sbp;           /* [i], [*o] points to sense_buffer memory */
  95    unsigned int timeout;       /* [i] MAX_UINT->no timeout (unit: millisec) */
  96    unsigned int flags;         /* [i] 0 -> default, see SG_FLAG... */
  97    int pack_id;                /* [i->o] unused internally (normally) */
  98    void __user * usr_ptr;      /* [i->o] unused internally */
  99    unsigned char status;       /* [o] scsi status */
 100    unsigned char masked_status;/* [o] shifted, masked scsi status */
 101    unsigned char msg_status;   /* [o] messaging level data (optional) */
 102    unsigned char sb_len_wr;    /* [o] byte count actually written to sbp */
 103    unsigned short host_status; /* [o] errors from host adapter */
 104    unsigned short driver_status;/* [o] errors from software driver */
 105    int resid;                  /* [o] dxfer_len - actual_transferred */
 106    unsigned int duration;      /* [o] time taken by cmd (unit: millisec) */
 107    unsigned int info;          /* [o] auxiliary information */
 108} sg_io_hdr_t;  /* 64 bytes long (on i386) */

关于具体用法，可以在“The Linux SCSI Generic (sg) HOWTO http://tldp.org/HOWTO/SCSI-Generic-HOWTO/index.html” 找到一些说明，“sg3_utils（http://sg.danny.cz/sg/sg3_utils.html）”的源码包里面的代码很不错的参考例 子，提供了各种scsi指令的封装代码，简单的scsi指令测试也可以直接诶使用sg3_utils里面的命令就可以了。  更多信息可以参考http://sg.danny.cz/sg/index.html 站点。


scatter-gather DMA
是一些磁盘或者网络硬件的扩展特性，可以让硬件设备在一次DMA映射里面访问不连续的多块内存，利用这个特性，可以直接让用户空间的地址直接映射DMA上 面去，减少传输过程中的数据复制量，提高系统西能。 “ Understanding the linux kernel ”一书的“block device” 一章的开头部分有介绍。



如果构建多段的 scatter-gather scsi request，可以参考sg3_utils 中的example目录下的sg_iovec_tst 那个例子。我简单修改一下源代码，发送一个自定义的scatter-gather列表的read10 来读磁盘的scsi request到设备的代码。其中 单个request的扇区总数和 scatter-gather列表个数的限制和block驱动层数和scsi3层驱动的scsi_host等参数设置有关，scatter-gather 列表还可能依赖于硬件实现。 


#include <unistd.h>
#include <signal.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "sg_lib.h"
#include "sg_io_linux.h"

/* Test co de for D. Gilbert's extensions to the Linux OS SCSI generic ("sg")
device driver.
*  Copyright (C) 2003-2007 D. Gilbert
*  This program is free software; you can redistribute it and/or modify
*  it under the terms of the GNU General Public License as published by
*  the Free Software Foundation; either version 2, or (at your option)
*  any later version.

This program will read a certain number of blocks of a given block size
from a given sg device node and write what is retrieved out to a
normal file. The purpose is to test the sg_iovec mechanism within the
sg_io_hdr structure.

Version 0.12 (20070121)
*/


#define ME "sg_iovec_tst: "

#define A_PRIME 509
#define IOVEC_ELEMS 2048

#define SENSE_BUFF_LEN 32
#define DEF_TIMEOUT 40000       /* 40,000 milliseconds */

struct sg_iovec iovec[IOVEC_ELEMS];

/* Returns 0 if everything ok */
int sg_read(int sg_fd, unsigned char * buff, int num_blocks, int from_block,
int bs)
{
unsigned char rdCmd[10] = {READ_10, 0, 0, 0, 0, 0, 0, 0, 0, 0};
unsigned char senseBuff[SENSE_BUFF_LEN];
struct sg_io_hdr io_hdr;
int dxfer_len = bs * num_blocks;
int k=0, pos =0, rem=0;

rdCmd[2] = (unsigned char)((from_block >> 24) & 0xff);
rdCmd[3] = (unsigned char)((from_block >> 16) & 0xff);
rdCmd[4] = (unsigned char)((from_block >> 8) & 0xff);
rdCmd[5] = (unsigned char)(from_block & 0xff);
rdCmd[7] = (unsigned char)((num_blocks >> 8) & 0xff);
rdCmd[8] = (unsigned char)(num_blocks & 0xff);

/*
for (k = 0, pos = 0, rem = dxfer_len; k < IOVEC_ELEMS; ++k) {
iovec[k].iov_base = buff + pos;
iovec[k].iov_len = (rem > A_PRIME) ? A_PRIME : rem;
if (rem <= A_PRIME)
break;
pos += A_PRIME;
rem -= A_PRIME;
}

*/
//create 32 x 509 scatter-gather list , so remain ">128kb da ta"  was split into the second request.

for (k = 0; k < 32; ++k) {
iovec[k].iov_base = buff + pos;
iovec[k].iov_len = 509;
pos += 512;
}

//put the remain into 2 scatter-gather list , so we don't exceed maxnium 64 segments.
rem = dxfer_len - (k+1) * 509 ;

iovec[k++].iov_base = buff + pos;
iovec[k++].iov_len = (rem /2) -3;

pos += rem /2 ;
iovec[k++].iov_base = buff + pos;
iovec[k++].iov_len = rem - (rem /2) + 3;


if (k >= IOVEC_ELEMS) {
fprintf(stderr, "Can't fit dxfer_len=%d bytes in iovec\n", dxfer_len);
return -1;
}
memset(&io_hdr, 0, sizeof(sg_io_hdr_t));
io_hdr.interface_id = 'S';
io_hdr.cmd_len = sizeof(rdCmd);
io_hdr.cmdp = rdCmd;
io_hdr.dxfer_direction = SG_DXFER_FROM_DEV;
io_hdr.dxfer_len = dxfer_len;
io_hdr.iovec_count = k + 1;
io_hdr.dxferp = iovec;
io_hdr.mx_sb_len = SENSE_BUFF_LEN;
io_hdr.sbp = senseBuff;
io_hdr.timeout = DEF_TIMEOUT;
io_hdr.pack_id = from_block;

if (ioctl(sg_fd, SG_IO, &io_hdr)) {
perror("reading (SG_IO) on sg device, error");
return -1;
}
switch (sg_err_category3(&io_hdr)) {
case SG_LIB_CAT_CLEAN:
break;
case SG_LIB_CAT_RECOVERED:
fprintf(stderr, "Recovered error while reading block=%d, num=%d\n",
from_block, num_blocks);
break;
case SG_LIB_CAT_UNIT_ATTENTION:
fprintf(stderr, "Unit attention\n");
return -1;
default:
sg_chk_n_print3("reading", &io_hdr, 1);
return -1;
}
return 0;
}


int main(int argc, char * argv[])
{
int sg_fd, fd, res, j, m, dxfer_len;
unsigned int k, num;
int do_help = 0;
int blk_size = 512;
int count = 0;
char * sg_file_name = 0;
char * out_file_name = 0;
unsigned char * buffp;

for (j = 1; j < argc; ++j) {
if (0 == strncmp("-b=", argv[j], 3)) {
m = 3;
num = sscanf(argv[j] + m, "%d", &blk_size);
if ((1 != num) || (blk_size <= 0)) {
printf("Couldn't decode number after '-b' switch\n");
sg_file_name = 0;
break;
}
}
else if (0 == strncmp("-c=", argv[j], 3)) {
m = 3;
num = sscanf(argv[j] + m, "%d", &count);
if (1 != num) {
printf("Couldn't decode number after '-c' switch\n");
sg_file_name = 0;
break;
}
}
else if (0 == strcmp("-h", argv[j]))
do_help = 1;
else if (*argv[j] == '-') {
printf("Unrecognized switch: %s\n", argv[j]);
sg_file_name = 0;
break;
}
else if (NULL == sg_file_name)
sg_file_name = argv[j];
else
out_file_name = argv[j];
}
if ((NULL == sg_file_name) || (NULL == out_file_name) || (0 == count)) {
printf("Usage: sg_iovec_tst [-h] [-b=num] -c=num <generic_device> "
"<output_filename>\n");
printf("  where: -h       this usage message\n");
printf("         -b=num   block size (default 512 Bytes)\n");
printf("         -c=num   count of blocks to transfer\n");
printf(" reads from <generic_device> and sends to <output_filename>\n");
return 1;
}

sg_fd = open(sg_file_name, O_RDONLY);
if (sg_fd < 0) {
perror(ME "sg device node open error");
return 1;
}
/* Don't worry, being very careful not to write to a none-sg file ... */
res = ioctl(sg_fd, SG_GET_VERSION_NUM, &k);
if ((res < 0) || (k < 30000)) {
printf(ME "not a sg device, or driver prior to 3.x\n");
return 1;
}
fd = open(out_file_name, O_WRONLY | O_CREAT, 0666);
if (fd < 0) {
perror(ME "output file open error");
return 1;
}



count = 512 -1 ;  //don't exceed 256kb IO
blk_size = 512;   

dxfer_len = count * blk_size;  //最终生成的request的大小

const int inter_space= 1024; 

buffp = (unsigned char *)malloc(dxfer_len + inter_space);
if (buffp) {
if (0 == sg_read(sg_fd, buffp, count, 0, blk_size)) {
if (write(fd, buffp, dxfer_len) < 0)
perror(ME "output write failed");
}
free(buffp);
}
res = close(fd);
if (res < 0) {
perror(ME "output file close error");
close(sg_fd);
return 1;
}
res = close(sg_fd);
if (res < 0) {
perror(ME "sg device close error");
return 1;
}
return 0;
}