Hadoop Mapreduce编程 MapJoin实现

1.Mapper端设计

package com.mycat.mapd_movie_mapjoin;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

public class RatingMapper extends Mapper {
    Map map=new HashMap<>();
    IntWritable mk=new IntWritable();
    Text mv=new Text();
    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        Path[] cacheFiles = context.getLocalCacheArchives();
        String path=cacheFiles[0].toString();
        BufferedReader bf=new BufferedReader(new FileReader(path));
        String line=null;
        while((line=bf.readLine())!=null){
            String[] movies = line.split("::");
            map.put(Integer.parseInt(movies[0].trim()),movies[1]+"\t"+movies[2]);
        }
    }

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] lines = value.toString().split("::");
        Integer joinKey=Integer.parseInt(lines[0].trim()   );
        if(map.containsKey(joinKey)){
            String res=map.get(joinKey)+lines[0]+"\t"+lines[2]+"\t"+lines[3];
            mk.set(joinKey);
            mv.set(res);
            context.write(mk,mv);
        }
    }
}

2.Driver端设计

package com.mycat.mapd_movie_mapjoin;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;


/**
 * reduce端join的缺陷：-----适合大表和大表关联
 *      1）数据倾斜---分区分布不均匀
 *      2）因为reduce端采用的集合，数据量大的时候，可能会产生OOM
 *      3）reducetask本身并行度不高,导致性能比较低----经验值是：DataNode数量*0.95
 */
public class RatingDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException {
        System.setProperty("HADOOP_USER_NAME","hadoop");
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs:/mkmg/");
        Job job = Job.getInstance(conf);

        job.setJarByClass(RatingDriver.class);

        job.setMapperClass(RatingMapper.class);

        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(Text.class);

        job.setNumReduceTasks(0);

        job.addCacheArchive(new URI("/movie/movies.dat"));// hdfs

        FileInputFormat.addInputPath(job,new Path("/rate"));
        FileSystem fs=FileSystem.get(conf);
        Path out=new Path("/mapjoin_out");
        if(fs.exists(out)){
            fs.delete(out,true);
        }
        FileOutputFormat.setOutputPath(job,out);

        job.waitForCompletion(true);
    }
}