mapreduce作业单元测试

用官方提供的MRunit包进行对MapReducer作业进行测试

http://mrunit.apache.org/

先要下载对应的测试包,分为1.0和2.0的hadoop版本

 

来自:

https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial

 

Mapper

 

Java代码 复制代码  收藏代码
  1. import org.apache.hadoop.io.IntWritable;   
  2. import org.apache.hadoop.io.LongWritable;   
  3. import org.apache.hadoop.io.Text;   
  4. import org.apache.hadoop.mapreduce.Mapper;   
  5.   
  6. public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>   
  7. {   
  8.   
  9.     private Text status = new Text();   
  10.   
  11.     private final static IntWritable addOne = new IntWritable(1);   
  12.   
  13.     @Override  
  14.     protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException   
  15.     {   
  16.   
  17.         //655209;1;796764372490213;804422938115889;6 is the Sample record format   
  18.         String[] line = value.toString().split(";");   
  19.         // If record is of SMS CDR   
  20.         if (Integer.parseInt(line[1]) == 1)   
  21.         {   
  22.             status.set(line[4]);   
  23.             context.write(status, addOne);   
  24.         }   
  25.     }   
  26.   
  27. }  
import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;



public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>

{



    private Text status = new Text();



    private final static IntWritable addOne = new IntWritable(1);



    @Override

    protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException

    {



        //655209;1;796764372490213;804422938115889;6 is the Sample record format

        String[] line = value.toString().split(";");

        // If record is of SMS CDR

        if (Integer.parseInt(line[1]) == 1)

        {

            status.set(line[4]);

            context.write(status, addOne);

        }

    }



}

 

 

Reducer:

 

 

Java代码 复制代码  收藏代码
  1. import java.util.List;   
  2.   
  3. import org.apache.hadoop.io.IntWritable;   
  4. import org.apache.hadoop.io.LongWritable;   
  5. import org.apache.hadoop.io.Text;   
  6. import org.apache.hadoop.mrunit.mapreduce.MapDriver;   
  7. import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;   
  8. import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;   
  9. import org.junit.Before;   
  10. import org.junit.Test;   
  11.   
  12. import flex.messaging.io.ArrayList;   
  13.   
  14. public class SMSCDRMapperReducerTest   
  15. {   
  16.     MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;   
  17.     ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;   
  18.     MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;   
  19.     @Before  
  20.     public void setup()   
  21.     {   
  22.         SMSCDRMapper mapper = new SMSCDRMapper();   
  23.         SMSCDRReducer reducer = new SMSCDRReducer();   
  24.         mapDriver = MapDriver.newMapDriver(mapper);   
  25.            
  26.         reduceDriver = ReduceDriver.newReduceDriver(reducer);   
  27.         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);   
  28.     }   
  29.        
  30.     @Test  
  31.     public void testMapper()   
  32.     {   
  33.         mapDriver.withInput(new LongWritable(), new Text(   
  34.         "655209;1;796764372490213;804422938115889;6"));   
  35.         mapDriver.withOutput(new Text("6"), new IntWritable(1));   
  36.         mapDriver.runTest();   
  37.     }   
  38.        
  39.     @Test  
  40.     public void testReducer()   
  41.     {   
  42.         List<IntWritable> values = new ArrayList();   
  43.         values.add(new IntWritable(1));   
  44.         values.add(new IntWritable(1));   
  45.         reduceDriver.withInput(new Text("6"), values);   
  46.         reduceDriver.withOutput(new Text("6"), new IntWritable(2));   
  47.         reduceDriver.runTest();   
  48.     }   
  49. }  
import java.util.List;



import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.mapreduce.MapDriver;

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;

import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;

import org.junit.Before;

import org.junit.Test;



import flex.messaging.io.ArrayList;



public class SMSCDRMapperReducerTest

{

    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;

    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;

    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;

    @Before

    public void setup()

    {

        SMSCDRMapper mapper = new SMSCDRMapper();

        SMSCDRReducer reducer = new SMSCDRReducer();

        mapDriver = MapDriver.newMapDriver(mapper);

        

        reduceDriver = ReduceDriver.newReduceDriver(reducer);

        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);

    }

    

    @Test

    public void testMapper()

    {

        mapDriver.withInput(new LongWritable(), new Text(

        "655209;1;796764372490213;804422938115889;6"));

        mapDriver.withOutput(new Text("6"), new IntWritable(1));

        mapDriver.runTest();

    }

    

    @Test

    public void testReducer()

    {

        List<IntWritable> values = new ArrayList();

        values.add(new IntWritable(1));

        values.add(new IntWritable(1));

        reduceDriver.withInput(new Text("6"), values);

        reduceDriver.withOutput(new Text("6"), new IntWritable(2));

        reduceDriver.runTest();

    }

}

 

Test测试类

 

 

Java代码 复制代码  收藏代码
  1. import java.util.List;   
  2.   
  3. import org.apache.hadoop.io.IntWritable;   
  4. import org.apache.hadoop.io.LongWritable;   
  5. import org.apache.hadoop.io.Text;   
  6. import org.apache.hadoop.mrunit.mapreduce.MapDriver;   
  7. import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;   
  8. import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;   
  9. import org.junit.Before;   
  10. import org.junit.Test;   
  11.   
  12. import flex.messaging.io.ArrayList;   
  13.   
  14. public class SMSCDRMapperReducerTest   
  15. {   
  16.     MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;   
  17.     ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;   
  18.     MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;   
  19.     @Before  
  20.     public void setup()   
  21.     {   
  22.         SMSCDRMapper mapper = new SMSCDRMapper();   
  23.         SMSCDRReducer reducer = new SMSCDRReducer();   
  24.         mapDriver = MapDriver.newMapDriver(mapper);   
  25.            
  26.         reduceDriver = ReduceDriver.newReduceDriver(reducer);   
  27.         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);   
  28.     }   
  29.        
  30.     @Test  
  31.     public void testMapper()   
  32.     {   
  33.         mapDriver.withInput(new LongWritable(), new Text(   
  34.         "655209;1;796764372490213;804422938115889;6"));   
  35.         mapDriver.withOutput(new Text("6"), new IntWritable(1));   
  36.         mapDriver.runTest();   
  37.     }   
  38.        
  39.     @Test  
  40.     public void testReducer()   
  41.     {   
  42.         List<IntWritable> values = new ArrayList();   
  43.         values.add(new IntWritable(1));   
  44.         values.add(new IntWritable(1));   
  45.         reduceDriver.withInput(new Text("6"), values);   
  46.         reduceDriver.withOutput(new Text("6"), new IntWritable(2));   
  47.         reduceDriver.runTest();   
  48.     }   
  49. }  
import java.util.List;



import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.mapreduce.MapDriver;

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;

import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;

import org.junit.Before;

import org.junit.Test;



import flex.messaging.io.ArrayList;



public class SMSCDRMapperReducerTest

{

    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;

    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;

    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;

    @Before

    public void setup()

    {

        SMSCDRMapper mapper = new SMSCDRMapper();

        SMSCDRReducer reducer = new SMSCDRReducer();

        mapDriver = MapDriver.newMapDriver(mapper);

        

        reduceDriver = ReduceDriver.newReduceDriver(reducer);

        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);

    }

    

    @Test

    public void testMapper()

    {

        mapDriver.withInput(new LongWritable(), new Text(

        "655209;1;796764372490213;804422938115889;6"));

        mapDriver.withOutput(new Text("6"), new IntWritable(1));

        mapDriver.runTest();

    }

    

    @Test

    public void testReducer()

    {

        List<IntWritable> values = new ArrayList();

        values.add(new IntWritable(1));

        values.add(new IntWritable(1));

        reduceDriver.withInput(new Text("6"), values);

        reduceDriver.withOutput(new Text("6"), new IntWritable(2));

        reduceDriver.runTest();

    }

}

 

转自http://piaoling.iteye.com/blog/1672683

你可能感兴趣的:(mapreduce)