Mapreduce 各部门工资统计实操

Mapreduce 各部门工资统计

0.本地文件内容

分别是
人物代号，人物名字，上级领导，上级代号，入职时间，正常工资，奖金，工作部门号

1.Mapper部分

package com.mr.emp;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class EmpMapper extends Mapper<LongWritable, Text, IntWritable,IntWritable> {
     
    IntWritable key2 = new IntWritable();
    IntWritable value2 = new IntWritable();

    @Override
    protected void map(LongWritable key1, Text v1, Context context) throws IOException, InterruptedException {
     
        //数据格式：111,agatha,jane,222,1980/12/21,4000,,20
        //1.分词，按照逗号分隔，取出工资和部门号
        String data = v1.toString();
        //["111"，"agatha"，"aa","222","1980/12/21","4000","","20"]
        String[] split = data.split(",");
        //2、通过context以即<20,4000>
        String salary = split[5];
        String deptNo= split[7];
        int salaryInt = Integer.parseInt(salary);
        int deptNoInt = Integer.parseInt(deptNo);
        key2.set(deptNoInt);
        value2.set(salaryInt);
        context.write(key2,value2);
    }
}

2.Reducer部分

package com.mr.emp;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class EmpReducer extends Reducer<IntWritable,IntWritable,IntWritable,IntWritable> {
     
   IntWritable value4 = new IntWritable();
   IntWritable key4;
    @Override
    protected void reduce(IntWritable key3, Iterable<IntWritable> value3, Context context) throws IOException, InterruptedException {
     
        //求和操作：<20,[200,4000]>
        int sum=0;
        for(IntWritable value : value3){
     
            sum += value.get();
        }
        //context写出
        key4=key3;
        value4.set(sum);
        context.write(key4,value4);
    }
}

3.Job部分

package com.mr.emp;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Arrays;

public class EmpJob {
     
    public static void main(String[] args) throws Exception {
     
        //1,job对象
        Job job = Job.getInstance(new Configuration());
        job.setJarByClass(EmpJob.class);

        job.setMapperClass(EmpMapper.class);
        job.setMapOutputKeyClass(IntWritable.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setReducerClass(EmpReducer.class);
        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(IntWritable.class);

        FileInputFormat.setInputPaths(job,new Path("D:\\emp.csv"));
        FileOutputFormat.setOutputPath(job,new Path("D:\\empoutput\\"));

        boolean completion = job.waitForCompletion(true);
        System.out.println("运行结果： " + completion);
    }
}

运行结果