mapreduce (三) MapReduce实现倒排索引(二)

hadoop api

http://hadoop.apache.org/docs/r1.0.4/api/org/apache/hadoop/mapreduce/Reducer.html

改变一下需求：要求“文档词频列表”是经过排序的，即 出现次数高的再前

思路：


代码：

package proj;



import java.io.IOException;

import java.util.HashMap;

import java.util.Map;



import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

import org.apache.hadoop.util.GenericOptionsParser;



public class InvertedIndexSortByFreq {



    // 将词分为<word:num,docid>

    public static class InvertedIndexMapper extends

            Mapper<Object, Text, Text, Text> {



        private Text keyInfo = new Text();

        private Text valInfo = new Text();

        private FileSplit split;



        public void map(Object key, Text value, Context context)

                throws IOException, InterruptedException {

            String[] tokens = value.toString().split(" ");

            split = (FileSplit) context.getInputSplit();

            String docid = split.getPath().getName();

            Map<String, Integer> map = new HashMap<String, Integer>();

            for (String token : tokens) {

                if (map.containsKey(token)) {

                    Integer newInt = new Integer(map.get(token) + 1);

                    map.put(token, newInt);

                } else {

                    map.put(token, 1);

                }

            }

            for (String k : map.keySet()) {

                Integer num = map.get(k);

                keyInfo.set(k + ":" + num);

                valInfo.set(docid);

                context.write(keyInfo, valInfo);

            }

        }

    }



    public static class InvertedIndexPartioner extends

            HashPartitioner<Text, Text> {



        private Text term = new Text();



        public int getPartition(Text key, Text value, int numReduceTasks) {

            term.set(key.toString().split(":")[0] + ":" + value);

            return super.getPartition(term, value, numReduceTasks);

        }

    }



    // 组合成倒排索引文档

    public static class InvertedIndexReducer extends

            Reducer<Text, Text, Text, Text> {

        private Text keyInfo = new Text();

        

        private Text valInfo = new Text();



        private String tPrev = null;



        private StringBuffer buff = new StringBuffer();



        public void reduce(Text key, Iterable<Text> values, Context context)

                throws IOException, InterruptedException {



            String[] tokens = key.toString().split(":");

            String current = tokens[0];



            if (tPrev == null) {

                tPrev = current;

                for (Text val : values) {

                    buff.append(tokens[1] + ":" + val.toString() + ";");

                }

            }

            

            if(tPrev.equals(current)){

                for (Text val : values) {

                    buff.append(tokens[1] + ":" + val.toString() + ";");

                }

            }else{

                keyInfo.set(tPrev);

                valInfo.set(buff.toString());

                context.write(keyInfo,valInfo);

                tPrev = current;

                buff = new StringBuffer();

                for (Text val : values) {

                    buff.append(tokens[1] + ":" + val.toString() + ";");

                }

            }

        }

        

        public void cleanup(Context context) throws IOException, InterruptedException{

            keyInfo.set(tPrev);

            valInfo.set(buff.toString());

            context.write(keyInfo,valInfo);

            super.cleanup(context);

        }

        

        

    }



    public static void main(String[] args) throws IOException,

            ClassNotFoundException, InterruptedException {

        Configuration conf = new Configuration();

        String[] otherArgs = new GenericOptionsParser(conf, args)

                .getRemainingArgs();

        Job job = new Job(conf, "InvertedIndex");

        job.setJarByClass(InvertedIndex.class);

        job.setMapperClass(InvertedIndexMapper.class);

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(Text.class);

        job.setPartitionerClass(InvertedIndexPartioner.class);

        job.setReducerClass(InvertedIndexReducer.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(Text.class);

        FileInputFormat.addInputPath(job, new Path(otherArgs[0]));

        FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }



}