MR-2.输入格式（InputFormat）FileInputFormat源码分析

FileInputFormat 是InputFormat一个实现类。主要做两个方面的功能

（1）          为一个job作业提供文件输入，FileInputFormat提供四个静态方法来设置job作业的输入路径

public static void addInputPath( Job j ob, Path path)
public static void addInputPaths( Job j ob, String commaSeparatedPaths)
public static void setInputPaths( Job j ob, Path. . . inputPaths)

public static void setInputPaths( Job j ob, String commaSeparatedPaths)

（2）          对输入的文件生成Input files。

  /** 
   * 对输入的文件生成一个FileSplits列表
   */
  public List getSplits(JobContext job) throws IOException {
    Stopwatch sw = new Stopwatch().start();
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
    long maxSize = getMaxSplitSize(job);

    // generate splits
    List splits = new ArrayList();
	//获取所有输入文件列表
    List files = listStatus(job);
    for (FileStatus file: files) {
      Path path = file.getPath();
      long length = file.getLen();
      if (length != 0) {
		//获取文件file对应的block信息
        BlockLocation[] blkLocations;
        if (file instanceof LocatedFileStatus) {
          blkLocations = ((LocatedFileStatus) file).getBlockLocations();
        } else {
          FileSystem fs = path.getFileSystem(job.getConfiguration());
          blkLocations = fs.getFileBlockLocations(file, 0, length);
        }
	        if (isSplitable(job, path)) {//文件可切分，形成多个FileSplits
          long blockSize = file.getBlockSize();
		  //计算每个InputSplit的大小
          long splitSize = computeSplitSize(blockSize, minSize, maxSize);

          long bytesRemaining = length;
          while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, splitSize,
                        blkLocations[blkIndex].getHosts(),
                        blkLocations[blkIndex].getCachedHosts()));
            bytesRemaining -= splitSize;
          }
          if (bytesRemaining != 0) {
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
                       blkLocations[blkIndex].getHosts(),
                       blkLocations[blkIndex].getCachedHosts()));
          }
        } else { // 不可切分整个文件形成一个FileSplit,让MapTask处理
          splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
                      blkLocations[0].getCachedHosts()));
        }
      } else { 
        //创建一个长度为0的FileSplit
        splits.add(makeSplit(path, 0, length, new String[0]));
      }
    }
    // Save the number of input files for metrics/loadgen
    job.getConfiguration().setLong(NUM_INPUT_FILES, files.size());
    sw.stop();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Total # of splits generated by getSplits: " + splits.size()
          + ", TimeTaken: " + sw.elapsedMillis());
    }
    return splits;
  }

其中，把InputSplit的切片文件生成records键值对由FileInputFormat的子类来实现。下面以TextInputFormat为例，分析FileInputFormat类中的中的createRecordReader方法如何来切分文件生成Record的k/v键值对。

（1）          Mapper中的run方法分析

public void run(Context context) throws IOException, InterruptedException {
    setup(context);
    try {
	  //这里的context.nextKeyValue方法底层是RecordReader类中nextKeyValue来实现
	  //由于FileInputFormat中通过createRecordReader方法调用通过LineRecordReader来重写RecordReader
	  //中的nextKeyValue方法，故该方法的调用是LineRecordReader中的nextKeyValue方法
      while (context.nextKeyValue()) {
        map(context.getCurrentKey(), context.getCurrentValue(), context);
      }
    } finally {
      cleanup(context);
    }
  }

（2）          LineRecordReader中的nextKeyValue方法

/**
 * 该方法的目的：每次调用nextKeyValue，获取对应的key/value键值对
 */
public boolean nextKeyValue() throws IOException {
    //这里说明TextInputFormat切分出来的k/v的数据类型为LongWritable/Text
	//从此说明对TextInputFormat格式处理的文本，map函数输入k/v 类型是LongWritable/Text
    if (key == null) {
      key = new LongWritable();
    }
	//pos 每行记录的偏移量，例如: 文本处理，每行的开始位置就是pos
    key.set(pos);
    if (value == null) {
      value = new Text();
    }
    int newSize = 0;
    // We always read one extra line, which lies outside the upper
    // split limit i.e. (end - 1)
    while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) {
      if (pos == 0) {
        newSize = skipUtfByteOrderMark();
      } else {
		//通过输入流SplitLineReader in 读取每行赋值给value
        newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos));
        pos += newSize;
      }
      if ((newSize == 0) || (newSize < maxLineLength)) {
        break;
      }

      // line too long. try again
      LOG.info("Skipped line of size " + newSize + " at pos " + 
               (pos - newSize));
    }
    if (newSize == 0) {
      key = null;
      value = null;
      return false;
    } else {
      return true;
    }
  }