一个使用MRUnit测试mapreduce程序例子

mrunit测试mapreduce程序很方便，具体的流程和解释，给一个连接：

https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial

下面把自己写的例子代码贴一下，做个笔记：

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import java.util.regex.*;

public class RetrievePasswdMapper extends Mapper<Object, Text, Text, Text>{
    
    private String retrieveByMobile = "\\[Retrieve Password Service\\]retrieve by mobile";
    private String retrieveByEmail = "\\[Retrieve Password Service\\]retrieve by email";
    private String mobileNotExist = "\\[Retrieve Password Service\\]mobile not exist";
    private String emailNotExist = "\\[Retrieve Password Service\\]email not exist";
    private String sendMobileVcode = "\\[Retrieve Password Service\\]send mobile vcode";
    private String sendEmailVcode = "\\[Retrieve Password Service\\]send email vcode";
    private String successByMobile = "\\[Retrieve Password Service\\]success by mobile";
    private String successByEmail = "\\[Retrieve Password Service\\]success by email";
    private String failByMobile = "\\[Retrieve Password Service\\]fail by mobile";
    private String failByEmail = "\\[Retrieve Password Service\\]fail by email";
    
    Pattern p1=Pattern.compile(retrieveByMobile);
    Pattern p2=Pattern.compile(retrieveByEmail);
    Pattern p3=Pattern.compile(mobileNotExist);
    Pattern p4=Pattern.compile(emailNotExist);
    Pattern p5=Pattern.compile(sendMobileVcode);
    Pattern p6=Pattern.compile(sendEmailVcode);
    Pattern p7=Pattern.compile(successByMobile);
    Pattern p8=Pattern.compile(successByEmail);
    Pattern p9=Pattern.compile(failByMobile);
    Pattern p10=Pattern.compile(failByEmail);
    
    private Text reportDate = new Text();
    private Text word = new Text();
            
    public void map(Object key, Text value, Context context
                    ) throws IOException, InterruptedException {
        String line = value.toString();
        reportDate.set(line.substring(1, 11).trim());
        if (p1.matcher(line).find()){
//            System.out.println("retrieveByMobile");
//            word.set("retrieveByMobile");
            context.write(reportDate, new Text("retrieveByMobile"));
        } else if (p2.matcher(line).find()){
            System.out.println("retrieveByEmail");
            context.write(reportDate, new Text("retrieveByEmail"));
        }else if (p3.matcher(line).find()){
            System.out.println("");
            context.write(reportDate, new Text("mobileNotExist"));
        }else if (p4.matcher(line).find()){
            System.out.println("");
            context.write(reportDate, new Text("emailNotExist"));
        }else if (p5.matcher(line).find()){
            System.out.println("");
            context.write(reportDate, new Text("sendMobileVcode"));
        }else if (p6.matcher(line).find()){
            System.out.println("");
            context.write(reportDate, new Text("sendEmailVcode"));
        }else if (p7.matcher(line).find()){
            context.write(reportDate, new Text("successByMobile"));
        }else if (p8.matcher(line).find()){
            System.out.println("");
            context.write(reportDate, new Text("successByEmail"));
        }else if (p9.matcher(line).find()){
            System.out.println("");
            context.write(reportDate, new Text("failByMobile"));
        }else if (p10.matcher(line).find()){
            System.out.println("");
            context.write(reportDate, new Text("failByEmail"));
        }
    }   
    
  /*  public static void main(String[] args)
    {
        String re = "[2012-08-29 00:18:39  INFO GetPasswordService:202]  [Retrieve Password Service]send mobile vcode.mobile=189XXXX0798";
        String sendMobileVcode = "\\[Retrieve Password Service\\]send mobile ";
        Pattern p=Pattern.compile(sendMobileVcode);
        Matcher m=p.matcher(re);
        System.out.println("result is "+Pattern.matches(sendMobileVcode, re));
    }*/

}

public class RetrievePasswdReducer extends Reducer<Text,Text,Text,Text> {
   
    public void reduce(Text key, Iterable<Text> values,
                       Context context
                       ) throws IOException, InterruptedException {
      int[] result = new int[10];
      for (Text val : values) {
        if (val.toString().equals("retrieveByMobile")){
            result[0]++;
        }
        else if (val.toString().equals("retrieveByEmail"))
            result[1]++;
        else if (val.toString().equals("mobileNotExist"))
            result[2]++;
        else if (val.toString().equals("emailNotExist"))
            result[3]++;
        else if (val.toString().equals("sendMobileVcode"))
            result[4]++;
        else if (val.toString().equals("sendEmailVcode"))
            result[5]++;
        else if (val.toString().equals("successByMobile"))
            result[6]++;
        else if (val.toString().equals("successByEmail"))
            result[7]++;
        else if (val.toString().equals("failByMobile"))
            result[8]++;
        else if (val.toString().equals("failByEmail"))
            result[9]++;
      }
      String sum = result[0]+", "+result[1]+", "+result[2]+", "+
      result[3]+", "+result[4]+", "+result[5]+", "+
      result[6]+", "+result[7]+", "+result[8]+", "+result[9];
      System.out.println(sum);
      System.out.println("sum");
      context.write(key, new Text(sum));
    }
  }

public class RetrievePasswd {

    public static void main(String[] args) {
        Configuration conf = new Configuration();
        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
        if (otherArgs.length != 2) {
          System.err.println("Usage: wordcount <in> <out>");
          System.exit(2);
        }
        try {
            Job job = new Job(conf, "word count");
            job.setJarByClass(RetrievePasswd.class);
            job.setMapperClass(RetrievePasswdMapper.class);
            job.setCombinerClass(RetrievePasswdReducer.class);
            job.setReducerClass(RetrievePasswdReducer.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
            FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
            FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
            System.exit(job.waitForCompletion(true) ? 0 : 1);
        } catch (Exception e)
        {
            e.printStackTrace();
        }
      }
    

}

mrunit测试：

import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

public class RetrievePasswdTest {
    MapDriver<Object, Text, Text, Text> mapDriver;
    ReduceDriver<Text, Text, Text, Text> reduceDriver;
    MapReduceDriver<Object, Text, Text, Text, Text, Text> mapReduceDriver;
    
    @Before
    public void setUp() {
        RetrievePasswdMapper mapper = new RetrievePasswdMapper();
        RetrievePasswdReducer reducer = new RetrievePasswdReducer();
        mapDriver = MapDriver.newMapDriver(mapper);
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }
    
    @Test
    public void testMapper() {
        mapDriver.withInput(new Object(), new Text(
                "[2012-08-29 00:18:39  INFO GetPasswordService:202]  [Retrieve Password Service]send mobile vcode.mobile=189XXXX0798"
                ));
        mapDriver.withOutput(new Text("2012-08-29"), new Text("sendMobileVcode"));
        mapDriver.runTest();
        
    }
    @Test
    public void testReducer() {
        List<Text> values = new ArrayList<Text>();
        values.add(new Text("retrieveByMobile"));
        values.add(new Text("retrieveByEmail"));
        values.add(new Text("retrieveByEmail"));
        values.add(new Text(""));
        reduceDriver.withInput(new Text("2012-08-29"), values);
        reduceDriver.withOutput(new Text("2012-08-29"), new Text("1, 2, 0, 0, 0, 0, 0, 0, 0, 0"));
        reduceDriver.runTest();
    }


}