Hadoop MapReduce入门

基本类型

BooleanWritalbe:标准的布尔类型
ByteWritable : 字节类型
DoubleWritable : 双字节
FloatWritable ： 浮点型
IntWritable ： 整型
LongWritable ：长整型
ObjectWritable：Obejct
Text ： UTF8 存储
NullWritable : 当

源码：

package org.bigdata.util;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/** * 统计字符串出现次数 * * @author wwhhf * */
public class WordCountMapReduce {

    public static class WordCountMapper extends
            Mapper<LongWritable, Text, Text, LongWritable> {

        @Override
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            String values[] = value.toString().split(" ");
            for (int i = 0, len = values.length; i < len; i++) {
                context.write(new Text(values[i]), new LongWritable(1));
            }
        }

    }

    public static class WordCountCombiner extends
            Reducer<Text, LongWritable, Text, LongWritable> {

        @Override
        protected void reduce(Text key, Iterable<LongWritable> values,
                Context context) throws IOException, InterruptedException {
            Long sum = 0L;
            for (LongWritable value : values) {
                sum = sum + value.get();
            }
            context.write(key, new LongWritable(sum));
        }

    }

    public static class WordCountReducer extends
            Reducer<Text, LongWritable, Text, LongWritable> {

        @Override
        protected void reduce(Text key, Iterable<LongWritable> values,
                Context context) throws IOException, InterruptedException {
            Long sum = 0L;
            for (LongWritable value : values) {
                sum = sum + value.get();
            }
            context.write(key, new LongWritable(sum));
        }

    }

    public static class TextDescComparator extends WritableComparator {

        protected TextDescComparator() {
            super(Text.class, true);
        }

        @Override
        public int compare(WritableComparable a, WritableComparable b) {
            return -super.compare(a, b);
        }

    }

    public static void main(String[] args) {
        try {

            Configuration cfg = HadoopCfg.getConfiguration();
            Job job = Job.getInstance(cfg);
            job.setJobName("Word Count");
            job.setJarByClass(WordCountMapReduce.class);

            // mapper
            job.setMapperClass(WordCountMapper.class);
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(LongWritable.class);

            // combiner
            job.setCombinerClass(WordCountCombiner.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(LongWritable.class);

            // reducer
            job.setReducerClass(WordCountReducer.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(LongWritable.class);

            job.setSortComparatorClass(TextDescComparator.class);

            FileInputFormat.addInputPath(job, new Path("/input/input.txt"));
            FileOutputFormat.setOutputPath(job, new Path("/output/"));

            System.exit(job.waitForCompletion(true) ? 0 : 1);

        } catch (IllegalStateException | IllegalArgumentException
                | ClassNotFoundException | IOException | InterruptedException e) {
            e.printStackTrace();
        }
    }

}

package org.bigdata.util;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.NlpAnalysis;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.bigdata.util.WeatherMapReduce.WeatherMapper;
import org.bigdata.util.WeatherMapReduce.WeatherReducer;

/** * 统计红楼梦出现最多的名字 * * @author wwhhf * */
public class RedHouseMapReduce {

    /** * 分词 * * @param s * @return */
    private static List<String> parse(String s) {
        List<Term> terms = NlpAnalysis.parse(s);
        List<String> words = new ArrayList<String>();
        for (Term term : terms) {
            // 词性为人名
            if ("nr".equals(term.getNatureStr())) {
                words.add(term.getName());
            }
        }
        return words;
    }

    private static class RedHouseMapper extends
            Mapper<LongWritable, Text, Text, LongWritable> {

        @Override
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            List<String> names = parse(value.toString());
            for (String name : names) {
                context.write(new Text(name), new LongWritable(1));
            }
        }
    }

    private static class RedHouseReducer extends
            Reducer<Text, LongWritable, Text, LongWritable> {

        @Override
        protected void reduce(Text key, Iterable<LongWritable> values,
                Context context) throws IOException, InterruptedException {
            Long sum = 0L;
            for (LongWritable value : values) {
                sum = sum + value.get();
            }
            context.write(key, new LongWritable(sum));
        }

    }

    public static void main(String[] args) {
        try {

            Configuration cfg = HadoopCfg.getConfiguration();
            Job job = Job.getInstance(cfg);
            job.setJobName("RedHourse");
            job.setJarByClass(RedHouseMapReduce.class);

            // mapper
            job.setMapperClass(RedHouseMapper.class);
            job.setMapOutputKeyClass(LongWritable.class);
            job.setMapOutputValueClass(Text.class);

            // reducer
            job.setReducerClass(RedHouseReducer.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(LongWritable.class);

            FileInputFormat.addInputPath(job, new Path("/input/red.txt"));
            FileOutputFormat.setOutputPath(job, new Path("/red_out/"));

            System.exit(job.waitForCompletion(true) ? 0 : 1);

        } catch (IllegalStateException | IllegalArgumentException
                | ClassNotFoundException | IOException | InterruptedException e) {
            e.printStackTrace();
        }
    }
}