Hadoop 求最大值 最小值 BiggestSmallest

Hadoop 求最大值 最小值

 

1、源代码

package com.dtspark.hadoop.hellomapreduce;
import java.io.IOException;
import java.util.Iterator;
import java.util.StringTokenizer;
 
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

import com.dtspark.hadoop.hellomapreduce.BiggestSmallest.BiggestSmallestMapper;

public class BiggestSmallest {
    /*
     * 将文件中的每行数据作为输出的key
     */
  public static class BiggestSmallestMapper
     extends Mapper<Object, Text, Text,LongWritable>{
 
     private final LongWritable data = new LongWritable(0);
  private final Text keyFoReducer=new Text("keyFoReducer");  
  
     public void map(Object key, Text value, Context context
                 ) throws IOException, InterruptedException {
      System.out.println("Map Method Invoked!");
 
         data.set(Long.parseLong(value.toString())); 
         System.out.println(data);
         context.write(keyFoReducer,data);
        
        }
    }

  /*
   * 将key输出
   */
public static  class BiggestSmallestReducer
    extends Reducer<Text, LongWritable,Text,LongWritable> {
 
 
 private long maxValue =Long.MIN_VALUE;
 private long minValue =Long.MAX_VALUE;
 
 
   public void reduce(Text key, Iterable<LongWritable> values,
                    Context context
                    ) throws IOException, InterruptedException {
  System.out.println("Reduce Method Invoked!");
 
  for(LongWritable item : values){
   if (item.get()>maxValue){
    maxValue=item.get();
   }
  
   if (item.get()<minValue){
    minValue=item.get();
   }
  
  }
  context.write(new Text("maxValue"), new LongWritable(maxValue));
  context.write(new Text("minValue"), new LongWritable(minValue));
 
  }
   
  
  
}

public static void main(String[] args) throws Exception {
 Configuration conf = new Configuration();
 String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
 if (otherArgs.length < 2) {
   System.err.println("Usage: DefferentData <in> [<in>...] <out>");
   System.exit(2);
 }
 Job job = new Job(conf, "Average");
 job.setJarByClass(BiggestSmallest.class);
 job.setMapperClass(BiggestSmallestMapper.class);
 job.setCombinerClass(BiggestSmallestReducer.class);//加快效率
 job.setReducerClass(BiggestSmallestReducer.class);
 job.setOutputKeyClass(Text.class);
 job.setOutputValueClass(LongWritable.class);
 for (int i = 0; i < otherArgs.length - 1; ++i) {
   FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
 }
 FileOutputFormat.setOutputPath(job,
   new Path(otherArgs[otherArgs.length - 1]));
 System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}

 

2、数据文件

[root@master IMFdatatest]#pwd
/usr/local/IMFdatatest
[root@master IMFdatatest]#cat dataBiggestsmallest.txt
1
33
55
66
77
45
34567
50
88776
345
5555555
23
32

3、上传集群

[root@master IMFdatatest]#hadoop dfs -put dataBiggestsmallest.txt /library

 

4、运行参数

hdfs://192.168.2.100:9000/library/dataBiggestsmallest.txt
hdfs://192.168.2.100:9000/library/outputdataBiggestsmallest10

 

5、运行结果

[root@master IMFdatatest]#hadoop dfs -cat /library/outputdataBiggestsmallest10/part-r-00000
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.

16/02/12 20:03:12 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
maxValue        5555555
minValue        5555555
maxValue        5555555
minValue        1
[root@master IMFdatatest]#

 

这里有点奇怪，单独打印最大值 或者最小值都是正确的 只有一行

如果同时打印，就重复了