马士兵 day5_Java开发Map/Reduce

day5_Java开发Map/Reduce

本地运行

• job_local

package com.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Test {
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);

        job.setMapperClass(WordMapper.class);
        job.setReducerClass(WordReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

        //数据文件在本地
        FileInputFormat.setInputPaths(job, "/Users/lujiafeng/GitRepositories/bigdata/hadoop/test/test.txt");
        //输出文件在本地
        FileOutputFormat.setOutputPath(job, new Path("/Users/lujiafeng/GitRepositories/bigdata/hadoop/test/out/"));

        //true表示将运行进度等信息及时输出给用户，false的话只是等待作业结束
        job.waitForCompletion(true);
    }

}

把hdfs中的文件拉到本地来运行

• 需要启动hdfs
• 注意：这里是把hdfs文件拉到本地来运行，如果观察输出的话会观察到jobID带有local字样
• 同时这样运行的时候是不需要yarn的。

package com.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Test {
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);

        job.setMapperClass(WordMapper.class);
        job.setReducerClass(WordReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

        FileInputFormat.setInputPaths(job, "hdfs://master:9000/input/");
        FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9000/output2/"));

        //true表示将运行进度等信息及时输出给用户，false的话只是等待作业结束
        job.waitForCompletion(true);
    }

}

在远程服务器执行

• 临时设置System.setProperty("HADOOP_USER_NAME", "root");
• 把Mapper和Reducer打包成jar包上传到hdfs
• 使用maven打包成jar文件

package com.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * Created by lujiafeng on 2018/10/7.
 */
public class Test {
    public static void main(String[] args) throws Exception {

        //System.out.println(System.getProperty("HADOOP_USER_NAME"));
        System.setProperty("HADOOP_USER_NAME", "root");
        //System.out.println(System.getProperty("HADOOP_USER_NAME"));

        Configuration conf = new Configuration();

        //添加如下内容后下面的文件输入输出位置可以使用相对路径
        conf.set("fs.defaultFS", "hdfs://master:9000/");

        //注意jar文件所在目录
        conf.set("mapreduce.job.jar", "target/hadoopTest-1.0-SNAPSHOT.jar");
        conf.set("mapreduce.framework.name", "yarn");
        conf.set("yarn.resourcemanager.hostname", "master");
        conf.set("mapreduce.app-submission.cross-platform", "true");

        Job job = Job.getInstance(conf);

        job.setMapperClass(WordMapper.class);
        job.setReducerClass(WordReducer.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);


        FileInputFormat.setInputPaths(job, "hdfs://master:9000/input/");
        FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9000/output7"));

        //true表示将运行进度等信息及时输出给用户，false的话只是等待作业结束
        job.waitForCompletion(true);
    }
}