Linux那些事儿之我是Block层(7)scsi命令的前世今生(一)

现在我们块设备也有了,队列也有了,要提交请求也就可以开始提交了.那就让我们来研究一下如何提交请求如何处理请求吧.不过哥们儿有言在先,出错处理的那些乱七八糟的代码咱们就不理睬了.

仍然以scsi磁盘举例,最初scsi这边发送的是scsi命令,可是从block走就得变成request,然而走到usb-storage那边又得变回scsi命令.换言之,这整个过程scsi命令要变两次身.

首先让我们从sd那边很常用的一个函数开始,我们来看scsi命令是如何在光天化日之下被偷梁换柱的变成了request,这个函数就是scsi_execute_req().来自drivers/scsi/scsi_lib.c:

    216 int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,

    217                      int data_direction, void *buffer, unsigned bufflen,

    218                      struct scsi_sense_hdr *sshdr, int timeout, int retries)

    219 {

    220         char *sense = NULL;

    221         int result;

    222

    223         if (sshdr) {

    224                 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);

    225                 if (!sense)

    226                         return DRIVER_ERROR << 24;

    227         }

    228         result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,

    229                               sense, timeout, retries, 0);

    230         if (sshdr)

    231                 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);

    232

    233         kfree(sense);

    234         return result;

    235 }

这里面最需要关注的就是一个函数,scsi_execute(),来自同一个文件.

    164 /**

    165  * scsi_execute - insert request and wait for the result

    166  * @sdev:       scsi device

    167  * @cmd:        scsi command

    168  * @data_direction: data direction

    169  * @buffer:     data buffer

    170  * @bufflen:    len of buffer

    171  * @sense:      optional sense buffer

    172  * @timeout:    request timeout in seconds

    173  * @retries:    number of times to retry request

    174  * @flags:      or into request flags;

    175  *

    176  * returns the req->errors value which is the scsi_cmnd result

    177  * field.

    178  **/

    179 int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,

    180                  int data_direction, void *buffer, unsigned bufflen,

    181                  unsigned char *sense, int timeout, int retries, int flags)

    182 {

    183         struct request *req;

    184         int write = (data_direction == DMA_TO_DEVICE);

    185         int ret = DRIVER_ERROR << 24;

    186

    187         req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);

    188

    189         if (bufflen &&  blk_rq_map_kern(sdev->request_queue, req,

    190                                         buffer, bufflen, __GFP_WAIT))

    191                 goto out;

    192

    193         req->cmd_len = COMMAND_SIZE(cmd[0]);

    194         memcpy(req->cmd, cmd, req->cmd_len);

    195         req->sense = sense;

    196         req->sense_len = 0;

    197         req->retries = retries;

    198         req->timeout = timeout;

    199         req->cmd_type = REQ_TYPE_BLOCK_PC;

    200         req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;

    201

    202         /*

    203          * head injection *required* here otherwise quiesce won't work

204          */

    205         blk_execute_rq(req->q, NULL, req, 1);

    206

    207         ret = req->errors;

    208  out:

    209         blk_put_request(req);

    210

    211         return ret;

    212 }

首先被调用的是blk_get_request.来自block/ll_rw_blk.c:

   2215 struct request *blk_get_request(request_queue_t *q, int rw, gfp_t gfp_mask)

   2216 {

   2217         struct request *rq;

   2218

   2219         BUG_ON(rw != READ && rw != WRITE);

   2220

   2221         spin_lock_irq(q->queue_lock);

   2222         if (gfp_mask & __GFP_WAIT) {

   2223                 rq = get_request_wait(q, rw, NULL);

   2224         } else {

   2225                 rq = get_request(q, rw, NULL, gfp_mask);

   2226                 if (!rq)

   2227                         spin_unlock_irq(q->queue_lock);

   2228         }

   2229         /* q->queue_lock is unlocked at this point */

   2230

   2231         return rq;

   2232 }

注意到我们调用这个函数的时候,第二个参数确实是__GFP_WAIT.所以2223行会被执行.get_request_wait()来自同一个文件:

   2173 static struct request *get_request_wait(request_queue_t *q, int rw_flags,

   2174                                         struct bio *bio)

   2175 {

   2176         const int rw = rw_flags & 0x01;

   2177         struct request *rq;

   2178

   2179         rq = get_request(q, rw_flags, bio, GFP_NOIO);

   2180         while (!rq) {

   2181                 DEFINE_WAIT(wait);

   2182                 struct request_list *rl = &q->rq;

   2183

   2184                 prepare_to_wait_exclusive(&rl->wait[rw], &wait,

   2185                                 TASK_UNINTERRUPTIBLE);

   2186

   2187                 rq = get_request(q, rw_flags, bio, GFP_NOIO);

   2188

   2189                 if (!rq) {

   2190                         struct io_context *ioc;

   2191

   2192                         blk_add_trace_generic(q, bio, rw, BLK_TA_SLEEPRQ);

   2193

   2194                         __generic_unplug_device(q);

   2195                         spin_unlock_irq(q->queue_lock);

   2196                         io_schedule();

   2197

   2198                         /*

   2199                          * After sleeping, we become a "batching" process and

   2200                          * will be able to allocate at least one request, and

   2201                          * up to a big batch of them for a small period time.

   2202                          * See ioc_batching, ioc_set_batching

   2203                          */

   2204                         ioc = current_io_context(GFP_NOIO, q->node);

   2205                         ioc_set_batching(q, ioc);

   2206

   2207                         spin_lock_irq(q->queue_lock);

   2208                 }

   2209                 finish_wait(&rl->wait[rw], &wait);

   2210         }

   2211

   2212         return rq;

   2213 }

而真正被调用的又是get_request(),仍然是来自同一个文件.

   2063 /*

   2064  * Get a free request, queue_lock must be held.

   2065  * Returns NULL on failure, with queue_lock held.

   2066  * Returns !NULL on success, with queue_lock *not held*.

   2067  */

   2068 static struct request *get_request(request_queue_t *q, int rw_flags,

   2069                                    struct bio *bio, gfp_t gfp_mask)

   2070 {

   2071         struct request *rq = NULL;

   2072         struct request_list *rl = &q->rq;

   2073         struct io_context *ioc = NULL;

   2074         const int rw = rw_flags & 0x01;

   2075         int may_queue, priv;

   2076

   2077         may_queue = elv_may_queue(q, rw_flags);

   2078         if (may_queue == ELV_MQUEUE_NO)

   2079                 goto rq_starved;

   2080

   2081         if (rl->count[rw]+1 >= queue_congestion_on_threshold(q)) {

   2082                 if (rl->count[rw]+1 >= q->nr_requests) {

   2083                         ioc = current_io_context(GFP_ATOMIC, q->node);

   2084                         /*

   2085                          * The queue will fill after this allocation, so set

   2086                          * it as full, and mark this process as "batching".

   2087                          * This process will be allowed to complete a batch of

   2088                          * requests, others will be blocked.

   2089                          */

   2090                         if (!blk_queue_full(q, rw)) {

   2091                                 ioc_set_batching(q, ioc);

   2092                                 blk_set_queue_full(q, rw);

   2093                         } else {

   2094                                 if (may_queue != ELV_MQUEUE_MUST

   2095                                                 && !ioc_batching(q, ioc)) {

   2096                                         /*

   2097                                          * The queue is full and the allocating

   2098                                          * process is not a "batcher", and not

   2099                                          * exempted by the IO scheduler

   2100                                          */

   2101                                         goto out;

   2102                                 }

   2103                         }

   2104                 }

   2105                 blk_set_queue_congested(q, rw);

   2106         }

   2107

   2108         /*

   2109          * Only allow batching queuers to allocate up to 50% over the defined

   2110          * limit of requests, otherwise we could have thousands of requests

   2111          * allocated with any setting of ->nr_requests

   2112          */

   2113         if (rl->count[rw] >= (3 * q->nr_requests / 2))

   2114                 goto out;

   2115

   2116         rl->count[rw]++;

   2117         rl->starved[rw] = 0;

   2118

   2119         priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);

   2120         if (priv)

   2121                 rl->elvpriv++;

   2122

   2123         spin_unlock_irq(q->queue_lock);

   2124

   2125         rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);

   2126         if (unlikely(!rq)) {

   2127                 /*

   2128                  * Allocation failed presumably due to memory. Undo anything

   2129                  * we might have messed up.

   2130                  *

   2131                  * Allocating task should really be put onto the front of the

   2132                  * wait queue, but this is pretty rare.

   2133                  */

   2134                 spin_lock_irq(q->queue_lock);

   2135                 freed_request(q, rw, priv);

   2136

   2137                 /*

   2138                  * in the very unlikely event that allocation failed and no

   2139                  * requests for this direction was pending, mark us starved

   2140                  * so that freeing of a request in the other direction will

   2141                  * notice us. another possible fix would be to split the

   2142                  * rq mempool into READ and WRITE

   2143                  */

   2144 rq_starved:

   2145                 if (unlikely(rl->count[rw] == 0))

   2146                         rl->starved[rw] = 1;

   2147

   2148                 goto out;

   2149         }

   2150

   2151         /*

   2152          * ioc may be NULL here, and ioc_batching will be false. That's

   2153          * OK, if the queue is under the request limit then requests need

   2154          * not count toward the nr_batch_requests limit. There will always

   2155          * be some limit enforced by BLK_BATCH_TIME.

   2156          */

   2157         if (ioc_batching(q, ioc))

   2158                 ioc->nr_batch_requests--;

   2159

   2160         rq_init(q, rq);

   2161

   2162         blk_add_trace_generic(q, bio, rw, BLK_TA_GETRQ);

   2163 out:

   2164         return rq;

   2165 }

这个elv_may_queue来自block/elevator.c:

    848 int elv_may_queue(request_queue_t *q, int rw)

    849 {

    850         elevator_t *e = q->elevator;

    851

    852         if (e->ops->elevator_may_queue_fn)

    853                 return e->ops->elevator_may_queue_fn(q, rw);

    854

    855         return ELV_MQUEUE_MAY;

    856 }

属于我们的那个elevator_t结构体变量是当初我们在elevator_init()中调用elevator_alloc()申请的.它的ops显然是和具体我们采用了哪种电梯有关系的.这里我们为了简便起见,做一个最不要脸的选择,选择”noop”,这种最简单最原始的机制.再一次贴出它的elevator_type.

     87 static struct elevator_type elevator_noop = {

     88         .ops = {

     89              .elevator_merge_req_fn          = noop_merged_requests,

     90                 .elevator_dispatch_fn           = noop_dispatch,

     91                 .elevator_add_req_fn            = noop_add_request,

     92                 .elevator_queue_empty_fn        = noop_queue_empty,

     93                 .elevator_former_req_fn         = noop_former_request,

     94                 .elevator_latter_req_fn         = noop_latter_request,

     95                 .elevator_init_fn               = noop_init_queue,

     96                 .elevator_exit_fn               = noop_exit_queue,

     97         },

     98         .elevator_name = "noop",

     99         .elevator_owner = THIS_MODULE,

    100 };

是不是觉得很开心. 对于我们选择的这种noop的电梯,elevator_may_queue_fn根本就没有定义哎.虽然我们这样做很无耻,但是谁叫我们不幸生在现在的中国呢?只要我们够作践,够胆大,够无耻,够疯狂,所谓的道德底线不是”大底”,重心可以下移,完全有向下突破的机会.

带着一个返回值ELV_MQUEUE_MAY,我们返回到get_request()中来.rl又是什么呢?2072行我们让它指向了q->rq.在这样一个危急关头,我不得不搬出一个复杂的结构体了,它就是request_queue,或者叫request_queue_t,定义于include/linux/blkdev.h:

     38 struct request_queue;

     39 typedef struct request_queue request_queue_t;

    360 struct request_queue

    361 {

    362         /*

    363          * Together with queue_head for cacheline sharing

    364          */

    365         struct list_head        queue_head;

    366         struct request          *last_merge;

    367         elevator_t              *elevator;

    368

    369         /*

    370          * the queue request freelist, one for reads and one for writes

    371          */

    372         struct request_list     rq;

    373

    374         request_fn_proc         *request_fn;

    375         make_request_fn         *make_request_fn;

    376         prep_rq_fn              *prep_rq_fn;

    377         unplug_fn               *unplug_fn;

    378         merge_bvec_fn           *merge_bvec_fn;

    379         issue_flush_fn          *issue_flush_fn;

    380         prepare_flush_fn        *prepare_flush_fn;

    381         softirq_done_fn         *softirq_done_fn;

    382

    383         /*

    384          * Dispatch queue sorting

    385          */

    386         sector_t                end_sector;

    387         struct request          *boundary_rq;

    388

    389         /*

    390          * Auto-unplugging state

    391          */

    392         struct timer_list       unplug_timer;

    393         int                     unplug_thresh;  /* After this many requests */

394         unsigned long           unplug_delay;   /* After this many jiffies */

    395         struct work_struct      unplug_work;

    396

    397         struct backing_dev_info backing_dev_info;

    398

    399         /*

    400          * The queue owner gets to use this for whatever they like.

    401          * ll_rw_blk doesn't touch it.

    402          */

    403         void                    *queuedata;

    404

    405         /*

    406          * queue needs bounce pages for pages above this limit

    407          */

    408         unsigned long           bounce_pfn;

    409         gfp_t                   bounce_gfp;

    410

    411         /*

    412          * various queue flags, see QUEUE_* below

    413          */

    414         unsigned long           queue_flags;

    415

    416         /*

    417          * protects queue structures from reentrancy. ->__queue_lock should

    418          * _never_ be used directly, it is queue private. always use

    419          * ->queue_lock.

    420          */

    421         spinlock_t              __queue_lock;

    422         spinlock_t              *queue_lock;

    423

    424         /*

    425          * queue kobject

    426          */

    427         struct kobject kobj;

    428

    429         /*

    430          * queue settings

    431          */

    432         unsigned long           nr_requests;    /* Max # of requests */

    433         unsigned int            nr_congestion_on;

    434         unsigned int            nr_congestion_off;

435         unsigned int            nr_batching;

    436

    437         unsigned int            max_sectors;

    438         unsigned int            max_hw_sectors;

    439         unsigned short          max_phys_segments;

    440         unsigned short          max_hw_segments;

    441         unsigned short          hardsect_size;

    442         unsigned int            max_segment_size;

    443

    444         unsigned long           seg_boundary_mask;

    445         unsigned int            dma_alignment;

    446

    447         struct blk_queue_tag    *queue_tags;

    448

    449         unsigned int            nr_sorted;

    450         unsigned int            in_flight;

    451

    452         /*

    453          * sg stuff

    454          */

    455         unsigned int            sg_timeout;

    456         unsigned int            sg_reserved_size;

    457         int                     node;

    458 #ifdef CONFIG_BLK_DEV_IO_TRACE

    459         struct blk_trace        *blk_trace;

    460 #endif

    461         /*

    462          * reserved for flush operations

    463          */

    464         unsigned int            ordered, next_ordered, ordseq;

    465         int                     orderr, ordcolor;

    466         struct request          pre_flush_rq, bar_rq, post_flush_rq;

    467         struct request          *orig_bar_rq;

    468         unsigned int            bi_size;

    469

    470         struct mutex            sysfs_lock;

    471 };

这里我们看到了rq其实是struct request_list结构体变量.这个结构体定义于同一个文件.

    131 struct request_list {

    132         int count[2];

    133         int starved[2];

    134         int elvpriv;

    135         mempool_t *rq_pool;

    136         wait_queue_head_t wait[2];

    137 };

不过这些我们现在都不想看,我们想看的只有其中的几个函数,第一个是2125行blk_alloc_request().来自ll_rw_blk.c:

   1970 static struct request *

   1971 blk_alloc_request(request_queue_t *q, int rw, int priv, gfp_t gfp_mask)

   1972 {

   1973         struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);

   1974

   1975         if (!rq)

   1976                 return NULL;

   1977

   1978         /*

   1979          * first three bits are identical in rq->cmd_flags and bio->bi_rw,

   1980          * see bio.h and blkdev.h

   1981          */

   1982         rq->cmd_flags = rw | REQ_ALLOCED;

   1983

   1984         if (priv) {

   1985                 if (unlikely(elv_set_request(q, rq, gfp_mask))) {

   1986                         mempool_free(rq, q->rq.rq_pool);

   1987                         return NULL;

   1988                 }

   1989                 rq->cmd_flags |= REQ_ELVPRIV;

   1990         }

   1991

   1992         return rq;

   1993 }

其它我们不懂没有关系,至少我们从1972行可以看出这里申请了一个struct request的结构体指针,换句话说,此前,我们已经有了请求队列,但是没有实质性的元素,从这一刻起,我们有了一个真正的request.虽然现在还没有进入到队伍中去,但这只是早晚的事儿了.

下一个rq_init().

    238 static void rq_init(request_queue_t *q, struct request *rq)

    239 {

    240         INIT_LIST_HEAD(&rq->queuelist);

    241         INIT_LIST_HEAD(&rq->donelist);

    242

    243         rq->errors = 0;

    244         rq->bio = rq->biotail = NULL;

    245         INIT_HLIST_NODE(&rq->hash);

    246         RB_CLEAR_NODE(&rq->rb_node);

    247         rq->ioprio = 0;

    248         rq->buffer = NULL;

    249         rq->ref_count = 1;

    250         rq->q = q;

    251         rq->special = NULL;

    252         rq->data_len = 0;

    253         rq->data = NULL;

    254         rq->nr_phys_segments = 0;

    255         rq->sense = NULL;

    256         rq->end_io = NULL;

    257         rq->end_io_data = NULL;

    258         rq->completion_data = NULL;

    259 }

这个函数在干什么不用我说,浦东金杨新村卖麻辣烫的大妈都知道,对刚申请的rq进行初始化.

然后,get_request()就开开心心的返回了,正常情况下,get_request_wait()也会跟着返回,再接着,blk_get_request()也就返回了.我们也带着申请好初始化好的req回到scsi_execute()中去,而接下来一段代码就是我们最关心的,对req的真正的赋值.比如req->cmd_len,req->cmd等等,就是这样被赋上的.换言之,我们的scsi命令就是这样被request拖下水的,从此它们之间不再是以前那种”水留不住落花的漂泊,落花走不进水的世界”的关系,而是沦落到了一荣俱荣一损俱损狼狈为奸的关系.

至此,完成了第一次变身,从scsi命令到request的变身.