使用MRUnit和TestNG进行单元测试

 

MRUnit

　　MRUnit是一款由Couldera公司开发的专门针对Hadoop中编写MapReduce单元测试的框架。

 

定义Map逻辑

import java.io.IOException;



import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.Mapper;





public class WordMapper extends Mapper<LongWritable, Text, Text, Text> 

{

    @Override

    public void map(LongWritable key, Text value, Context context) throws InterruptedException, IOException 

    {

        String[] line = value.toString().split(" ");

        context.write(new Text(line[0]), new Text(line[1]));

    }

}

 

定义Reduce逻辑

import java.io.IOException;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;





public class WordReducer extends Reducer<Text, Text, Text, Text> 

{

    public void reduce(Text key, Iterable<Text> values, Context context) throws InterruptedException, IOException 

    {

        for (Text value : values) {

            context.write(value, key);

        }

    }

}

 

单元测试示例

import java.util.*;



import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.*;

import org.apache.hadoop.mrunit.mapreduce.*;

import org.apache.hadoop.mrunit.types.Pair;

import org.testng.annotations.*;





public class UserForAreaDistributeTest 

{

    private Mapper<LongWritable, Text, Text, Text> mapper;

    private MapDriver<LongWritable, Text, Text, Text> mapDriver;

    

    private Reducer<Text, Text, Text, Text> reducer;

    private ReduceDriver<Text, Text, Text, Text> reduceDriver;

    

    

    @BeforeClass

    public void setUp()

    {

        mapper = new WordMapper();

        mapDriver = new MapDriver(mapper);

        

        reducer = new WordReducer();

        reduceDriver = new ReduceDriver(reducer);

    }

    

    @Test

    public void testMap() 

    {

        mapDriver.withInput(new Pair(new LongWritable(1L), new Text("abc def")));

        mapDriver.withOutput(new Text("abc"), new Text("def"));

        mapDriver.runTest();

    }

    

    @Test

    public void testReduce()

    {

        List<Text> list = new ArrayList<Text>();

        list.add(new Text("value1"));

        list.add(new Text("value2"));

        

        reduceDriver.withInput(new Text("key"), list);

        reduceDriver.withOutput(new Text("value1"), new Text("key"));

        reduceDriver.withOutput(new Text("value2"), new Text("key"));

        reduceDriver.runTest();

    }

}