Yarn下分片和分块源代码分析
public class FileSplit extends InputSplit implements Writable {
private Path file;
private long start;
private long length;
private String[] hosts;
public FileSplit() {
}
public FileSplit(Path file, long start, long length, String[] hosts) {
this.file = file;
this.start = start;
this.length = length;
this.hosts = hosts;
}
public Path getPath() {
return this.file;
}
public long getStart() {
return this.start;
}
public long getLength() {
return this.length;
}
public String toString() {
return this.file + ":" + this.start + "+" + this.length;
}
public void write(DataOutput out) throws IOException {
Text.writeString(out, this.file.toString());
out.writeLong(this.start);
out.writeLong(this.length);
}
public void readFields(DataInput in) throws IOException {
this.file = new Path(Text.readString(in));
this.start = in.readLong();
this.length = in.readLong();
this.hosts = null;
}
public String[] getLocations() throws IOException {
if (this.hosts == null) {
return new String[0];
}
return this.hosts;
}
}
代码比较简单， 四部分组成 文件路径 ，启始位置，长度，Host列表
Host为什么是个列表
看分片的时候创建函数
splits.add(makeSplit(path, length - bytesRemaining,
splitSize, blkLocations[blkIndex].getHosts()));
再来看块的源代码
public class BlockLocation { private String[] hosts; private String[] names; private String[] topologyPaths; private long offset; private long length; private boolean corrupt; public BlockLocation() { this(new String[0], new String[0], 0L, 0L); } public BlockLocation(String[] names, String[] hosts, long offset, long length) { this(names, hosts, offset, length, false); } public BlockLocation(String[] names, String[] hosts, long offset, long length, boolean corrupt) { if (names == null) this.names = new String[0]; else { this.names = names; } if (hosts == null) this.hosts = new String[0]; else { this.hosts = hosts; } this.offset = offset; this.length = length; this.topologyPaths = new String[0]; this.corrupt = corrupt; } public BlockLocation(String[] names, String[] hosts, String[] topologyPaths, long offset, long length) { this(names, hosts, topologyPaths, offset, length, false); } public BlockLocation(String[] names, String[] hosts, String[] topologyPaths, long offset, long length, boolean corrupt) { this(names, hosts, offset, length, corrupt); if (topologyPaths == null) this.topologyPaths = new String[0]; else this.topologyPaths = topologyPaths; } public String[] getHosts() throws IOException { if ((this.hosts == null) || (this.hosts.length == 0)) { return new String[0]; } return this.hosts; } public String[] getNames() throws IOException { if ((this.names == null) || (this.names.length == 0)) { return new String[0]; } return this.names; } public String[] getTopologyPaths() throws IOException { if ((this.topologyPaths == null) || (this.topologyPaths.length == 0)) { return new String[0]; } return this.topologyPaths; } public long getOffset() { return this.offset; } public long getLength() { return this.length; } public boolean isCorrupt() { return this.corrupt; } public void setOffset(long offset) { this.offset = offset; } public void setLength(long length) { this.length = length; } public void setCorrupt(boolean corrupt) { this.corrupt = corrupt; } public void setHosts(String[] hosts) throws IOException { if (hosts == null) this.hosts = new String[0]; else this.hosts = hosts; } public void setNames(String[] names) throws IOException { if (names == null) this.names = new String[0]; else this.names = names; } public void setTopologyPaths(String[] topologyPaths) throws IOException { if (topologyPaths == null) this.topologyPaths = new String[0]; else this.topologyPaths = topologyPaths; } public String toString() { StringBuilder result = new StringBuilder(); result.append(this.offset); result.append(','); result.append(this.length); if (this.corrupt) { result.append("(corrupt)"); } for (String h : this.hosts) { result.append(','); result.append(h); } return result.toString(); } }