Hadoop分析NCDC气象数据

气象数据准备：

1. 下载1993年到2003年每年的部分气象数据

ftp://ftp3.ncdc.noaa.gov/pub/data/noaa/

2. 下载的文件格式是*.gz，用zcat命令将其解压并上传到hdfs

zcat *.gz>1993.txt
bin/hadoop fs -put 1993.txt /feixu/input

3. 查看hdfs上的气象数据，总大小约1.2G

 

MapReduce代码：

package com.oracle.temperature;
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class AvgTemperatureMapper extends Mapper {
	
    private static final int MISSING = 9999;
    @Override
    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 	
        String line = value.toString();
        String year = line.substring(15, 19);
        int airTemperature;
        
        if (line.charAt(87) == '+') { // parseInt doesn't like leading plus signs
            airTemperature = Integer.parseInt(line.substring(88, 92));
        } else {
            airTemperature = Integer.parseInt(line.substring(87, 92));
        }
        
        String quality = line.substring(92, 93);
        if (airTemperature != MISSING && quality.matches("[01459]")) {
          context.write(new Text(year), new IntWritable(airTemperature));
        }
    }
}

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class AvgTemperatureReducer extends Reducer {
	
    @Override
    public void reduce(Text key, Iterable values, Context context)  throws IOException, InterruptedException {
        long totalNum = 0;
    	long totalValue = 0;
    	int average = 0;
        for (IntWritable value : values) {
        	totalNum ++;
        	totalValue +=value.get();
        }
        
        average = (int)(totalValue/totalNum);
        context.write(key, new IntWritable(average));
    }
}

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class AvgTemperature {
	
    public static void main(String[] args) throws Exception {
        if (args.length != 2) {
             System.err.println("Usage: AverageTemperature  
"); 
             System.exit(-1);
        }
        
        Job job = new Job();
        job.setJarByClass(AvgTemperature.class);
        job.setJobName("Average temperature");
        
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        
        job.setMapperClass(AvgTemperatureMapper.class);
        job.setReducerClass(AvgTemperatureReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

运行JOB

 

Map Count: 29

Reduce Count: 1

Map Input Records: 4,781,181

Reduce Output Records: 21