mapreduce作业单元测试

用官方提供的MRunit包进行对MapReducer作业进行测试

http://mrunit.apache.org/

先要下载对应的测试包，分为1.0和2.0的hadoop版本

 

来自：

https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial

 

Mapper

 

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>
{

    private Text status = new Text();

    private final static IntWritable addOne = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException
    {

        //655209;1;796764372490213;804422938115889;6 is the Sample record format
        String[] line = value.toString().split(";");
        // If record is of SMS CDR
        if (Integer.parseInt(line[1]) == 1)
        {
            status.set(line[4]);
            context.write(status, addOne);
        }
    }

}

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;



public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>

{



    private Text status = new Text();



    private final static IntWritable addOne = new IntWritable(1);



    @Override

    protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException

    {



        //655209;1;796764372490213;804422938115889;6 is the Sample record format

        String[] line = value.toString().split(";");

        // If record is of SMS CDR

        if (Integer.parseInt(line[1]) == 1)

        {

            status.set(line[4]);

            context.write(status, addOne);

        }

    }



}

 

 

Reducer:

 

 

import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import flex.messaging.io.ArrayList;

public class SMSCDRMapperReducerTest
{
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    @Before
    public void setup()
    {
        SMSCDRMapper mapper = new SMSCDRMapper();
        SMSCDRReducer reducer = new SMSCDRReducer();
        mapDriver = MapDriver.newMapDriver(mapper);

        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }

    @Test
    public void testMapper()
    {
        mapDriver.withInput(new LongWritable(), new Text(
        "655209;1;796764372490213;804422938115889;6"));
        mapDriver.withOutput(new Text("6"), new IntWritable(1));
        mapDriver.runTest();
    }

    @Test
    public void testReducer()
    {
        List<IntWritable> values = new ArrayList();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("6"), values);
        reduceDriver.withOutput(new Text("6"), new IntWritable(2));
        reduceDriver.runTest();
    }
}

import java.util.List;



import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.mapreduce.MapDriver;

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;

import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;

import org.junit.Before;

import org.junit.Test;



import flex.messaging.io.ArrayList;



public class SMSCDRMapperReducerTest

{

    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;

    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;

    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;

    @Before

    public void setup()

    {

        SMSCDRMapper mapper = new SMSCDRMapper();

        SMSCDRReducer reducer = new SMSCDRReducer();

        mapDriver = MapDriver.newMapDriver(mapper);

        

        reduceDriver = ReduceDriver.newReduceDriver(reducer);

        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);

    }

    

    @Test

    public void testMapper()

    {

        mapDriver.withInput(new LongWritable(), new Text(

        "655209;1;796764372490213;804422938115889;6"));

        mapDriver.withOutput(new Text("6"), new IntWritable(1));

        mapDriver.runTest();

    }

    

    @Test

    public void testReducer()

    {

        List<IntWritable> values = new ArrayList();

        values.add(new IntWritable(1));

        values.add(new IntWritable(1));

        reduceDriver.withInput(new Text("6"), values);

        reduceDriver.withOutput(new Text("6"), new IntWritable(2));

        reduceDriver.runTest();

    }

}

 

Test测试类

 

 

import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import flex.messaging.io.ArrayList;

public class SMSCDRMapperReducerTest
{
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    @Before
    public void setup()
    {
        SMSCDRMapper mapper = new SMSCDRMapper();
        SMSCDRReducer reducer = new SMSCDRReducer();
        mapDriver = MapDriver.newMapDriver(mapper);

        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }

    @Test
    public void testMapper()
    {
        mapDriver.withInput(new LongWritable(), new Text(
        "655209;1;796764372490213;804422938115889;6"));
        mapDriver.withOutput(new Text("6"), new IntWritable(1));
        mapDriver.runTest();
    }

    @Test
    public void testReducer()
    {
        List<IntWritable> values = new ArrayList();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("6"), values);
        reduceDriver.withOutput(new Text("6"), new IntWritable(2));
        reduceDriver.runTest();
    }
}

import java.util.List;



import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.mapreduce.MapDriver;

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;

import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;

import org.junit.Before;

import org.junit.Test;



import flex.messaging.io.ArrayList;



public class SMSCDRMapperReducerTest

{

    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;

    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;

    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;

    @Before

    public void setup()

    {

        SMSCDRMapper mapper = new SMSCDRMapper();

        SMSCDRReducer reducer = new SMSCDRReducer();

        mapDriver = MapDriver.newMapDriver(mapper);

        

        reduceDriver = ReduceDriver.newReduceDriver(reducer);

        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);

    }

    

    @Test

    public void testMapper()

    {

        mapDriver.withInput(new LongWritable(), new Text(

        "655209;1;796764372490213;804422938115889;6"));

        mapDriver.withOutput(new Text("6"), new IntWritable(1));

        mapDriver.runTest();

    }

    

    @Test

    public void testReducer()

    {

        List<IntWritable> values = new ArrayList();

        values.add(new IntWritable(1));

        values.add(new IntWritable(1));

        reduceDriver.withInput(new Text("6"), values);

        reduceDriver.withOutput(new Text("6"), new IntWritable(2));

        reduceDriver.runTest();

    }

}

 

转自http://piaoling.iteye.com/blog/1672683