【hadoop】FileInputFormat中getSplits()方法

Mapreduce是hadoop的并行计算框架。框架可以分为maptask，shuffle和reducetask阶段。

在maptask阶段，会根据Driver中关于InputFormat的Job配置信息对整个文件进行切分，根据切片文件数量，分配同等数量的maptask。然后根据“规则”读取切片文件，并以key-value的形式写入到环形缓冲区。

默认的切分、读取由TextInputFormat类实现（job.setInputFormatClass(TextInputFormat.class)）。TextInputFormat类继承FileInputFormat类。覆写了createRecordReader方法，创建了LineRecordReader类对象。其中，父类FileInputFormat的getSplits方法实现了切分，LineRecordReader类实现了读取规则（按行读取，key为行的偏移量，value为行的内容）。本文主要分析实现切片的代码。

getSplits源码（hadoop-3.2.1）

public List<InputSplit> getSplits(JobContext job) throws IOException {
    StopWatch sw = new StopWatch().start();
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
    long maxSize = getMaxSplitSize(job);

    // generate splits
    List<InputSplit> splits = new ArrayList<InputSplit>();
    List<FileStatus> files = listStatus(job);

    boolean ignoreDirs = !getInputDirRecursive(job)
      && job.getConfiguration().getBoolean(INPUT_DIR_NONRECURSIVE_IGNORE_SUBDIRS, false);
    for (FileStatus file: files) {
      if (ignoreDirs && file.isDirectory()) {
        continue;
      }
      Path path = file.getPath();
      long length = file.getLen();
      if (length != 0) {
        BlockLocation[] blkLocations;
        if (file instanceof LocatedFileStatus) {
          blkLocations = ((LocatedFileStatus) file).getBlockLocations();
        } else {
          FileSystem fs = path.getFileSystem(job.getConfiguration());
          blkLocations = fs.getFileBlockLocations(file, 0, length);
        }
        if (isSplitable(job, path)) {
          long blockSize = file.getBlockSize();
          long splitSize = computeSplitSize(blockSize, minSize, maxSize);

          long bytesRemaining = length;
          while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, splitSize,
                        blkLocations[blkIndex].getHosts(),
                        blkLocations[blkIndex].getCachedHosts()));
            bytesRemaining -= splitSize;
          }

          if (bytesRemaining != 0) {
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
                       blkLocations[blkIndex].getHosts(),
                       blkLocations[blkIndex].getCachedHosts()));
          }
        } else { // not splitable
          if (LOG.isDebugEnabled()) {
            // Log only if the file is big enough to be splitted
            if (length > Math.min(file.getBlockSize(), minSize)) {
              LOG.debug("File is not splittable so no parallelization "
                  + "is possible: " + file.getPath());
            }
          }
          splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
                      blkLocations[0].getCachedHosts()));
        }
      } else { 
        //Create empty hosts array for zero length files
        splits.add(makeSplit(path, 0, length, new String[0]));
      }
    }
    // Save the number of input files for metrics/loadgen
    job.getConfiguration().setLong(NUM_INPUT_FILES, files.size());
    sw.stop();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Total # of splits generated by getSplits: " + splits.size()
          + ", TimeTaken: " + sw.now(TimeUnit.MILLISECONDS));
    }
    return splits;
  }

核心逻辑：

1. 确定切片大小（splitSize）

//获取splitSize的最小范围，默认返回值为1字节
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));

//获取splitSize的最大范围，默认为返回值为Long.MAX_VALUE
long maxSize = getMaxSplitSize(job);

//获取block的大小
long blockSize = file.getBlockSize();

//默认配置下，splitSize就是文件的blockSize
long splitSize = computeSplitSize(blockSize, minSize, maxSize);

protected long computeSplitSize(long blockSize, long minSize, long maxSize) {
	return Math.max(minSize, Math.min(maxSize, blockSize));
}

2. 切分文件

long bytesRemaining = length;

//SPLIT_SLOP的值为1.1。如果未切分文件的大小比splitSize 还多10%，就继续切分。否则，停止切分
while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
	int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
	splits.add(makeSplit(path, length-bytesRemaining, splitSize,
               blkLocations[blkIndex].getHosts(),
               blkLocations[blkIndex].getCachedHosts()));
	bytesRemaining -= splitSize;
}

//将可能的最后一个切片文件，也放入splits列表
if (bytesRemaining != 0) {
            int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
            splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
                       blkLocations[blkIndex].getHosts(),
                       blkLocations[blkIndex].getCachedHosts()));
          }

3. 获取切片文件信息列表（splits）

splits.add(makeSplit(path, length-bytesRemaining, splitSize,
               blkLocations[blkIndex].getHosts(),
               blkLocations[blkIndex].getCachedHosts()));