hadoop MultipleOutputs

 

MultipleOutputs: 

　　write data to multiple files with customized name, can be used for both map and reduce phase.

http://www.lichun.cc/blog/2013/11/how-to-use-hadoop-multipleoutputs/

public static class MyMap extends

            Mapper<LongWritable, Text, Text, DoubleWritable> {

        MultipleOutputs<Text, DoubleWritable> mos;



        public void map(LongWritable inKey, Text inValue, Context context)

                throws IOException, InterruptedException {



            mos.write(map_out_file, NullWritable.get(), new Text(name));



        }



        @Override

        public void setup(Context context) {

            mos = new MultipleOutputs<Text, DoubleWritable>(context);

        }



        @Override

        protected void cleanup(Context context) throws IOException,

                InterruptedException {

            mos.close();

        }



    }

example

package a5p2;



import java.io.IOException;

import java.util.StringTokenizer;



import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.DoubleWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.RawComparator;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;



public class ClassAvg2 {

    public static final String map_out_file = "mapOutFileIndividualStudentAverage";

    public static final String reduce_out_file = "reduceOutFileClassAverage";



    public static class AvgMap extends

            Mapper<LongWritable, Text, Text, DoubleWritable> {

        MultipleOutputs<Text, DoubleWritable> mos;



        public void map(LongWritable inKey, Text inValue, Context context)

                throws IOException, InterruptedException {



            String line = inValue.toString();

            StringTokenizer myToken = new StringTokenizer(line);

            String name = myToken.nextToken();

            int cnt = 0;

            double sum = 0;

            double avg;

            while (myToken.hasMoreTokens()) {

                sum += Float.parseFloat(myToken.nextToken());

                cnt++;

            }

            avg = sum / cnt;

            context.write(new Text(name), new DoubleWritable(avg));

            mos.write(map_out_file, NullWritable.get(), new Text(name + " "

                    + avg));



        }



        @Override

        public void setup(Context context) {

            mos = new MultipleOutputs<Text, DoubleWritable>(context);

        }



        @Override

        protected void cleanup(Context context) throws IOException,

                InterruptedException {

            mos.close();

        }



    }



    public static class AvgReduce extends

            Reducer<Text, DoubleWritable, Text, DoubleWritable> {

        MultipleOutputs<Text, DoubleWritable> mos;



        public void reduce(Text key, Iterable<DoubleWritable> inValues,

                Context context) throws IOException, InterruptedException {



            double classSum = 0;

            int cnt = 0;

            for (DoubleWritable dw : inValues) {

                classSum += dw.get();

                cnt++;

            }

            double classAvg = classSum / cnt;

            mos.write(reduce_out_file, NullWritable.get(), new Text(

                    "Class average: " + classAvg));

            // context.write(new Text("class average"), new DoubleWritable(

            // classAvg));



        }



        @Override

        public void setup(Context context) {

            mos = new MultipleOutputs<Text, DoubleWritable>(context);

        }



        @Override

        protected void cleanup(Context context) throws IOException,

                InterruptedException {

            mos.close();

        }



    }



    public static class AvgGroupComparator implements RawComparator<Text> {



        public int compare(Text t1, Text t2) {

            return 0;

        }



        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {

            return 0;

        }

    }



    public static void main(String[] args) throws IOException,

            ClassNotFoundException, InterruptedException {

        Configuration conf = new Configuration();

        Job job = new Job(conf, "class avg");

        job.setJarByClass(ClassAvg2.class);



        // mapper

        job.setMapperClass(AvgMap.class);

        job.setGroupingComparatorClass(AvgGroupComparator.class);

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(DoubleWritable.class);



        // reducer

        job.setReducerClass(AvgReduce.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(DoubleWritable.class);



        // input and output

        FileInputFormat.addInputPath(job, new Path(args[0]));

        FileOutputFormat.setOutputPath(job, new Path(args[1]));



        MultipleOutputs.addNamedOutput(job, map_out_file,

                TextOutputFormat.class, NullWritable.class, Text.class);

        MultipleOutputs.addNamedOutput(job, reduce_out_file,

                TextOutputFormat.class, NullWritable.class, Text.class);



        System.exit(job.waitForCompletion(true) ? 0 : 1);



    }



}