08.hadoop-maven-intellij-win10环境搭建

一.软件环境

1. Intellij Idea
2. Maven
3. Hadoop分布式环境
4. JDK 1.8

二.win10下环境配置

2.1 JDK安装环境变量添加（自行安装）

2.2 maven安装（自行安装）

2.3 Intellij Idea安装（自行安装）

2.4 hadoop安装

QQ图片20180508144543.png

QQ图片20180508144909.png

QQ图片20180508145022.png

三.创建maven工程

1. 在Intellij中点击File->New->Project，在弹出的对话框中选择Maven，JDK选择1.8，点击Next
   
   

   QQ图片20180508145459.png
   
   
   

   QQ图片20180508145608.png
   
   
   

   QQ图片20180508145643.png
2. java环境检查是否一致
   file -> project structure
   
   

   QQ图片20180508151127.png
   
   file ->setting
   
   

   QQ图片20180508151233.png
3. 修改pom.xml文件

    4.0.0

    
    com.guider.hadoop
    hadoop
    1.0-SNAPSHOT
    jar

    hadoop
    http://maven.apache.org

    
    
        UTF-8
        1.8
        1.8
    

    
    
        
            nexus-aliyun
            Nexus aliyun
            http://maven.aliyun.com/nexus/content/groups/public
        
    


    
        
            org.apache.hadoop
            hadoop-client
            2.5.0
        

        
            junit
            junit
            4.11
            test
        
    


    
        
            
                
                    maven-clean-plugin
                    3.0.0
                
                
                
                    maven-resources-plugin
                    3.0.2
                
                
                    maven-compiler-plugin
                    3.7.0
                
                
                    maven-surefire-plugin
                    2.20.1
                
                
                    maven-jar-plugin
                    3.0.2
                
                
                    maven-install-plugin
                    2.5.2
                
                
                    maven-deploy-plugin
                    2.8.2
                
            
        
    

4. 复制配置文件到资源目录下

   
   
   

   QQ图片20180508150157.png

5. 复制DiskChecker到org.apache.hadoop.util包下面

   
   
   

   QQ图片20180508152913.png

四. 编写WordCount代码

package com.guider.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;

public class WordCount extends Configured implements Tool {
    // input -> map -> shuffle -> output
    // mapper，输入数据变成键值对，一行转化为一条
    //1. Map claaa
    public static class WordCountMapper extends Mapper {
        private Text mapOutputKey = new Text();
        private IntWritable mapOutputValue = new IntWritable();

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            //1.将读取的文件变成，偏移量+内容
            String linevalue = value.toString();
            System.out.println("linevalue----" + linevalue);
            //2.根据“ ”某种规则划分我们的单词，并处理
            String[] strs = linevalue.split(" ");
            for (String str : strs) {
                //key:单词， value:1
                mapOutputKey.set(str);
                mapOutputValue.set(1);
                context.write(mapOutputKey, mapOutputValue);
                System.out.println("str----" + str);
            }
            //3.将结果传递出处
        }
    }

    //2. Reduce class
    //reducer，map的输出就是reduce的输入
    public static class WordCountReduce extends Reducer {
        private IntWritable outputValue = new IntWritable();
        @Override
        protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
            //汇总
            int sum = 0;
            for (IntWritable value : values){
                sum += value.get();
            }
            outputValue.set(sum);
            context.write(key,outputValue);
        }
    }

    //3. job class
    public int run(String[] args) throws Exception {
        //获取我们的配置
        Configuration conf = new Configuration();
        //Configuration conf = this.getConf();

        Job job = Job.getInstance(conf, this.getClass().getSimpleName());
        //设置input与output
        Path inpath = new Path(args[0]);
        FileInputFormat.addInputPath(job, inpath);
        Path outpath = new Path(args[1]);
        FileOutputFormat.setOutputPath(job, outpath);

        //设置map与
        //需要设置的内容类 + 输出key与value
        job.setMapperClass(WordCountMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        //shuffle优化
//        job.setPartitionerClass(cls);
//        job.setSortComparatorClass(cls);
//        job.setCombinerClass(cls);
//        job.setGroupingComparatorClass(cls);

        //设置reduce
        job.setReducerClass(WordCountReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //将job交给Yarn
        boolean issucess = job.waitForCompletion(true);
        return issucess ? 0 : 1;
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        //参数
        args = new String[] {
                "hdfs://bigguider22.com:8020/user/root/mapreduce/input",
                "hdfs://bigguider22.com:8020/user/root/mapreduce/output1"
        };
        //跑我们的任务
        int status = new WordCount().run(args);
        //int status = ToolRunner.run(conf,new WordCount(),args);
        System.exit(status);
    }
}