iscsi kernel模块-网络部分
接口就是ioctl 没什么好说的
定义了一个上下文的 static DECLARE_MUTEX(ioctl_sem); 用来限制一次只能调用一个ioctl
主要分为
Session / Conn/ volume 增加 减少
Param的配置修改
先看临时一下 iet_socket_bind
struct iscsi_conn {
struct list_head list; /* list entry in session list */
struct iscsi_session *session; /* owning session */
u16 cid;
unsigned long state;
u32 stat_sn;
u32 exp_stat_sn;
int hdigest_type;
int ddigest_type;
struct list_head poll_list;
struct file *file;
/*通用 BSD socket 关键是socket->sk<L2 L3>*/
struct socket *sock;
//...
};
static void iet_socket_bind(struct iscsi_conn *conn)
{
int opt = 1;
mm_segment_t oldfs;
struct iscsi_session *session = conn->session;
struct iscsi_target *target = session->target;
dprintk(D_GENERIC, "%llu\n", (unsigned long long) session->sid);
/*userspace传递的是struct file得到对应的sock */
conn->sock = SOCKET_I(conn->file->f_dentry->d_inode);
conn->sock->sk->sk_user_data = conn;
/*下面iscsi用了自己socket线程处理唤醒 方法，和kernel独立开*/
write_lock_bh(&conn->sock->sk->sk_callback_lock);
/*sk_state_change: callback to indicate change in the state of the sock*/
target->nthread_info.old_state_change = conn->sock->sk->sk_state_change;
conn->sock->sk->sk_state_change = iet_state_change;
/*sk_data_ready: callback to indicate there is data to be processed*/
target->nthread_info.old_data_ready = conn->sock->sk->sk_data_ready;
conn->sock->sk->sk_data_ready = iet_data_ready;
target->nthread_info.old_write_space = conn->sock->sk->sk_write_space;
conn->sock->sk->sk_write_space = iet_write_space;
write_unlock_bh(&conn->sock->sk->sk_callback_lock);
/*没什么好说的 其实应该用int kernel_setsockopt(struct socket *sock, int level, int optname,
char *optval, unsigned int optlen)*/
oldfs = get_fs();
set_fs(get_ds());
conn->sock->ops->setsockopt(conn->sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt));
set_fs(oldfs);
}
非常著名的函数： 优化的源泉
void sock_init_data(struct socket *sock, struct sock *sk)
初始化 sock 结构中和IP相关的可扩展接口部分
void sock_init_data(struct socket *sock, struct sock *sk)
{
//...
sk->sk_state_change = sock_def_wakeup;
/*主要执行 wake_up_interruptible_sync_poll 唤醒socket_wq 的线程 固定CPU*/
sk->sk_data_ready = sock_def_readable;
sk->sk_write_space = sock_def_write_space;
sk->sk_error_report = sock_def_error_report;
sk->sk_destruct = sock_def_destruct;
//...
}
struct socket_wq {
wait_queue_head_t wait;
struct fasync_struct *fasync_list;
struct rcu_head rcu;
} ____cacheline_aligned_in_smp;
static void sock_def_wakeup(struct sock *sk)
{
struct socket_wq *wq;
rcu_read_lock();
wq = rcu_dereference(sk->sk_wq);
/*如果有等待数据的进程就返回ture*/
if (wq_has_sleeper(wq))
wake_up_interruptible_all(&wq->wait);
rcu_read_unlock();
}
static void iet_data_ready(struct sock *sk, int len)
{
struct iscsi_conn *conn = sk->sk_user_data;
struct iscsi_target *target = conn->session->target;
/*就是调用 wake_up_process(info->task); struct task_struct *task; 通知他们起来处理*/
nthread_wakeup(target);
target->nthread_info.old_data_ready(sk, len);
}
static int is_data_available(struct iscsi_conn *conn)
{
int avail, res;
mm_segment_t oldfs;
struct socket *sock = conn->sock;
oldfs = get_fs();
set_fs(get_ds());
/*这里作者很懒啊 ，直接通过FIONREAD 来得到数据情况*/
res = sock->ops->ioctl(sock, SIOCINQ, (unsigned long) &avail);
set_fs(oldfs);
return (res >= 0) ? avail : res;
}
下面看看 add_conn
static int add_conn(struct iscsi_target *target, unsigned long ptr)
{
//...
err = copy_from_user(&info, (void *) ptr, sizeof(info));
if (err)
return -EFAULT;
/*找到对应会话*/
session = session_lookup(target, info.sid);
if (!session)
return -ENOENT;
return conn_add(session, &info);
}
int conn_add(struct iscsi_session *session, struct conn_info *info)
{
//...
/*如果有了 就删掉 重新生成*/
conn = conn_lookup(session, info->cid);
if (conn)
conn_close(conn);
err = iet_conn_alloc(session, info);
if (!err && conn)
err = -EEXIST;
return err;
}
iet_conn_alloc 初始化iscsi_conn 结构体 ，所有的核心操作都和其有关
static int iet_conn_alloc(struct iscsi_session *session, struct conn_info *info)
{
struct iscsi_conn *conn;
dprintk(D_SETUP, "%#Lx:%u\n", (unsigned long long) session->sid, info->cid);
conn = kzalloc(sizeof(*conn), GFP_KERNEL);
if (!conn)
return -ENOMEM;
//...
spin_lock_init(&conn->list_lock);
atomic_set(&conn->nr_cmnds, 0);
atomic_set(&conn->nr_busy_cmnds, 0);
INIT_LIST_HEAD(&conn->pdu_list);
INIT_LIST_HEAD(&conn->write_list);
INIT_LIST_HEAD(&conn->poll_list);
init_timer(&conn->nop_timer); /*连接超时器*/
list_add(&conn->list, &session->conn_list);
set_bit(CONN_ACTIVE, &conn->state);
/*用户空间建立的连接传下来处理 */
conn->file = fget(info->fd);
iet_socket_bind(conn);
/*把自己加到该session 的激活链接中*/
list_add(&conn->poll_list, &session->target->nthread_info.active_conns);
nthread_wakeup(conn->session->target);
return 0;
}
add_session()->session_add()-> iet_session_alloc
和上面类似 不过建立了一个 命令的hash 数组链表
{
//...
for (i = 0; i < ARRAY_SIZE(session->cmnd_hash); i++)
INIT_LIST_HEAD(&session->cmnd_hash[i]);
spin_lock_init(&session->ua_hash_lock);
for (i = 0; i < ARRAY_SIZE(session->ua_hash); i++)
INIT_LIST_HEAD(&session->ua_hash[i]);
list_for_each_entry(vol, &target->volumes, list)
/* power-on, reset, or bus device reset occurred */
ua_establish_for_session(session, vol->lun, 0x29, 0x0);
//...
}
然后来看看干活的进程
static long ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
//...
if (cmd == ADD_TARGET) {
err = add_target(arg);
goto done;
}
//...
}
add_target()->通过用户入参target_info 初始化
target_add()->必要的检验
iscsi_target_create()->开始target线程 完成结构体初始化
target_thread_start()->
int nthread_start(struct iscsi_target *target)
{
//...
task = kthread_run(istd, target, "istd%d", target->tid);
//...
}
这是最主要关键的后台进程
static int istd(void *arg)
{
struct iscsi_target *target = arg;
struct network_thread_info *info = &target->nthread_info;
struct iscsi_conn *conn, *tmp;
/*进入调度*/
__set_current_state(TASK_RUNNING);
do {
spin_lock_bh(&info->nthread_lock);
__set_current_state(TASK_INTERRUPTIBLE);
/*数据没有好就让出调度 （这里要注意设置顺序，防止丢失事件）*/
if (!test_bit(D_DATA_READY, &info->flags)) {
spin_unlock_bh(&info->nthread_lock);
schedule();
spin_lock_bh(&info->nthread_lock);
}
__set_current_state(TASK_RUNNING);
clear_bit(D_DATA_READY, &info->flags);
spin_unlock_bh(&info->nthread_lock);
target_lock(target, 0);
list_for_each_entry_safe(conn, tmp, &info->active_conns, poll_list) {
if (test_bit(CONN_ACTIVE, &conn->state))
process_io(conn);/*处理IO事物*/
else
close_conn(conn);
}
target_unlock(target);
} while (!kthread_should_stop());
return 0;
}
先回顾一下 socket-> ops 的一个函数指针结构
struct proto_ops {
//...
ssize_t (*sendpage) (struct socket *sock, struct page *page,
int offset, size_t size, int flags);
ssize_t (*splice_read)(struct socket *sock, loff_t *ppos,
struct pipe_inode_info *pipe, size_t len, unsigned int flags);
//
...
}
在 tcp4中主要是
struct proto tcp_prot = {
.sendpage = tcp_sendpage,
}
Tcp_sendpage 也就是do_tcp_sendpages 先把页面拆分后
放到sk_write_queue 中然后按需通过 __tcp_push_pending_frames (多页) tcp_push_one(单页 头部) 发送出去。
static void process_io(struct iscsi_conn *conn)
{
struct iscsi_target *target = conn->session->target;
int res, wakeup = 0;
/*从网络上接受过来的数据请求*/
res = recv(conn);
//...
/*将带数据相应包发送出去*/
res = send(conn);
/*如果还有数据就等待下次处理*/
if (!list_empty(&conn->write_list) || conn->write_cmnd) {
conn_reset_nop_timer(conn);
wakeup = 1;
}
//...
return;
}
函数很简单没什么好说的 看看recv
static int recv(struct iscsi_conn *conn) --> case RX_DATA: do_recv()
这里作者还是很懒
直接 res = sock_recvmsg(conn->sock, &msg, len, MSG_DONTWAIT | MSG_NOSIGNAL);
然后就是 send
static int send(struct iscsi_conn *conn) -> 先是 case TX_INIT:
cmnd_tx_start(cmnd); case ISCSI_OP_SCSI_RSP:
static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
将 iscsi_conn-> write_tcmnd->tio 该tio 通过随后的 do_send()--->write_data()-->sendpage
发送出去