4-1、自定义InputFormat 类代码
在编写MapReduce的时候，自带的输入格式有时候满足不了我们的需求，这个时候就要自己来定制InputFprmat、InputSplit和RecordReader。
MapReduce自带的输入类型都是基于HDFS的，这个例子的功能是，不从HDFS上面读取输入内容，在内存中随机生成100个（0-1）float型的小数，然后求这100个小数的最大值。
1、类 FindMaxValueInputFormat 继承抽象类 InputFormat：
主要实现的方法：
public abstract List<InputSplit> getSplits(JobContext context)
public abstract RecordReader<K,V> createRecordReader(InputSplit split,TaskAttemptContext context)
package inputformat;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
public class FindMaxValueInputFormat extends InputFormat<IntWritable, ArrayWritable>{
public static float[] floatvalues;
/**
* 返回一个InputSplit 集合
* 这个例子一共有两个InputSplit，两个map
* 随机产生100个 0-1 的数组，放到float数组里面
*/
@Override
public List<InputSplit> getSplits(JobContext context) throws IOException,
InterruptedException {
int NumOfValues = context.getConfiguration().getInt("NumOfValues", 100);
floatvalues = new float[NumOfValues];
Random rand =new Random();
for (int i = 0; i < NumOfValues; i++) {
floatvalues[i]=rand.nextFloat();
}
int NumSplits = context.getConfiguration().getInt("mapred.map.tasks", 2);
int beg=0;
int length =(int) Math.floor(NumOfValues/NumSplits);
ArrayList<InputSplit> splits =new ArrayList<InputSplit>();
int end = length-1;
for (int i = 0; i < NumSplits-1; i++) {
FindMaxValueInputSplit split =new FindMaxValueInputSplit(beg,end);
splits.add(split);
beg=end+1;
end=end+length-1;
}
FindMaxValueInputSplit split=new FindMaxValueInputSplit(beg,NumOfValues-1);
splits.add(split);
return splits;
}
/**
* 自定义 RecordReader
*/
@Override
public RecordReader<IntWritable, ArrayWritable> createRecordReader(
InputSplit split, TaskAttemptContext context) throws IOException,
InterruptedException {
return new FindMaxValueRecordReader();
}
}
2、类 FindMaxValueInputSplit 继承 InputSplit 实现 Writable ：
package inputformat;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
public class FindMaxValueInputSplit extends InputSplit implements Writable{
private int m_StartIndex;
private int m_EndIndex;
private ArrayWritable m_FloatArray=new ArrayWritable(FloatWritable.class);
public FindMaxValueInputSplit(){};
/**
* 这个自定义分类主要记录了Map函数的开始索引和结束索引，第一个map处理前50个小数，第二个map后50个小数
* @param start 开始位置
* @param end 结束位置
*/
public FindMaxValueInputSplit(int start,int end){
m_StartIndex=start;
m_EndIndex=end;
int len=m_EndIndex-m_StartIndex+1;
int index=m_StartIndex;
FloatWritable[] result = new FloatWritable[len];
for (int i = 0; i < result.length; i++) {
float f=FindMaxValueInputFormat.floatvalues[index];
FloatWritable fW =new FloatWritable();
fW.set(f);
result[i]=fW;
index++;
}
m_FloatArray.set(result);
}
public void write(DataOutput out) throws IOException {
out.writeInt(this.m_StartIndex);
out.writeInt(this.m_EndIndex);
this.m_FloatArray.write(out);
}
public void readFields(DataInput in) throws IOException {
this.m_StartIndex=in.readInt();
this.m_EndIndex=in.readInt();
this.m_FloatArray.readFields(in);
}
@Override
public long getLength() throws IOException, InterruptedException {
return (this.m_EndIndex-this.m_StartIndex+1);
}
@Override
public String[] getLocations() throws IOException, InterruptedException {
return new String[]{"hadoop-2","hadoop-1"};
}
public int getM_StartIndex(){
return m_StartIndex;
}
public int getM_EndIndex(){
return m_EndIndex;
}
public ArrayWritable getM_FloatArray(){
return m_FloatArray;
}
}
3、类 FindMaxValueRecordReader 继承 RecordReader ：
package inputformat;
import java.io.IOException;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
/**
* 这个自定义 RecordReader 类，定义输入到map函数的输入格式
* Key为偏移量
* Value为float数组，长度为50
* @author Administrator
*
*/
public class FindMaxValueRecordReader extends RecordReader<IntWritable,ArrayWritable>{
private int m_End;
private int m_Index;
private int m_Start;
private IntWritable key=null;
private ArrayWritable value=null;
private FindMaxValueInputSplit fmvsplit=null;
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
fmvsplit=(FindMaxValueInputSplit)split;
this.m_Start=fmvsplit.getM_StartIndex();
this.m_End=fmvsplit.getM_EndIndex();
this.m_Index=this.m_Start;
}
/**
* 输出的key为 IntWritable
* 输出的value为 ArrayWritable
*/
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if (key == null) {
key=new IntWritable();
}
if (value == null) {
value = new ArrayWritable(FloatWritable.class);
}
if (m_Index <= m_End) {
key.set(m_Index);
value=fmvsplit.getM_FloatArray();
m_Index=m_End+1;
return true;
}else {
return false;
}
}
@Override
public IntWritable getCurrentKey() throws IOException, InterruptedException {
return key;
}
@Override
public ArrayWritable getCurrentValue() throws IOException,
InterruptedException {
return value;
}
@Override
public float getProgress() throws IOException, InterruptedException {
if (this.m_Index == this.m_End) {
return 0.0f;
}else {
return Math.min(1.0f, (this.m_Index - this.m_Start)/(float)(this.m_End-this.m_Start));
}
}
@Override
public void close() throws IOException {
}
}
4、mapper类 FindMaxValueMapper：
package inputformat;
import java.io.IOException;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Mapper;
/**
* Map函数的输入格式有所改变
* @author Administrator
*
*/
public class FindMaxValueMapper extends Mapper<IntWritable, ArrayWritable,IntWritable, FloatWritable>{
private final static IntWritable one =new IntWritable(1);
@Override
protected void map(
IntWritable key,
ArrayWritable value,
Mapper<IntWritable, ArrayWritable, IntWritable, FloatWritable>.Context context)
throws IOException, InterruptedException {
FloatWritable[] floatArray =(FloatWritable[])value.toArray();
float maxfloat=floatArray[0].get();
float tmp;
/**
* 求一个InputSplit中的最大值
*/
for (int i = 0; i < floatArray.length; i++) {
tmp=floatArray[i].get();
if (tmp>maxfloat) {
maxfloat=tmp;
}
}
/**
* 把一个map中的最大值输出出来
*/
context.write(one, new FloatWritable(maxfloat));
}
}
5、Reducer 类 FindMaxValueReducer ：
package inputformat;
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
/**
* Ruducer比较两个Map函数输出的最大值，结果输出在HDFS上面
* 这个例子就比较两个值，有几个Map比较几个
* @author Administrator
*
*/
public class FindMaxValueReducer extends Reducer<IntWritable, FloatWritable, Text, FloatWritable>{
@SuppressWarnings("rawtypes")
@Override
protected void reduce(
IntWritable k2,
Iterable<FloatWritable> v2s,
Reducer<IntWritable, FloatWritable, Text, FloatWritable>.Context context)
throws IOException, InterruptedException {
Iterator it = v2s.iterator();
float maxfloat=0,tmp;
/**
* 取第一个数
*/
if (it.hasNext()) {
maxfloat=((FloatWritable)(it.next())).get();
}else {
//集合为空时，输出迭代失败信息
context.write(new Text("Max float value : "), null);
return;
}
/**
* 求最大值
*/
while (it.hasNext()) {
tmp=((FloatWritable)(it.next())).get();
if (tmp > maxfloat) {
maxfloat = tmp;
}
}
//把最大的那个值输出来
context.write(new Text("Max float value : "), new FloatWritable(maxfloat));
}
}
6、驱动类 MaxValueDriver ：
package inputformat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider.Text;
/**
* 驱动类
* @author Administrator
*
*/
public class MaxValueDriver {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job=Job.getInstance(conf, MaxValueDriver.class.getSimpleName());
job.setJarByClass(MaxValueDriver.class);
job.setNumReduceTasks(1);
job.setMapperClass(FindMaxValueMapper.class);
job.setReducerClass(FindMaxValueReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(FloatWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FloatWritable.class);
job.setInputFormatClass(FindMaxValueInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
// FileInputFormat.setInputPaths(job, args[0]);
FileOutputFormat.setOutputPath(job, new Path(args[0]));
job.waitForCompletion(true);
}
}
第一次运行：
[hadoop@hadoop ~]$ hdfs dfs -ls /user/hadoop/output_inputformat1 Found 2 items -rw-r--r-- 3 hadoop supergroup 0 2015-09-04 15:09 /user/hadoop/output_inputformat1/_SUCCESS -rw-r--r-- 3 hadoop supergroup 29 2015-09-04 15:09 /user/hadoop/output_inputformat1/part-r-00000 [hadoop@hadoop ~]$ hdfs dfs -text /user/hadoop/output_inputformat1/part-r-00000 Max float value : 0.9981093 [hadoop@hadoop ~]$
第二次运行：
[hadoop@hadoop ~]$ hdfs dfs -ls /user/hadoop/output_inputformat2 Found 2 items -rw-r--r-- 3 hadoop supergroup 0 2015-09-05 11:43 /user/hadoop/output_inputformat2/_SUCCESS -rw-r--r-- 3 hadoop supergroup 29 2015-09-05 11:43 /user/hadoop/output_inputformat2/part-r-00000 [hadoop@hadoop ~]$
查看两次运行内容：
[hadoop@hadoop ~]$ hdfs dfs -text /user/hadoop/output_inputformat1/part-r-00000 Max float value : 0.9981093 [hadoop@hadoop ~]$ hdfs dfs -text /user/hadoop/output_inputformat2/part-r-00000 Max float value : 0.9966892 [hadoop@hadoop ~]$