总目录:https://www.jianshu.com/p/e406a9bc93a9
Hadoop - 子目录:https://www.jianshu.com/p/9428e443b7fd
天气案例
经典案例
myclient.java :客户端
package com.SL.tq;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.fs.Path;
public class MyClient {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration(true);
Job job = Job.getInstance(conf);
job.setJarByClass(com.SL.tq.MyClient.class);
//-----conf-----
//++map:
// 输入格式化:job.setInputFormatClass(ooxx.class);
job.setMapperClass(TMaper.class);
job.setMapOutputKeyClass(TQ.class);
job.setMapOutputValueClass(IntWritable.class);
job.setPartitionerClass(TPartitioner.class);
job.setSortComparatorClass(TsortComparator.class);
// job.setCombinerClass(TCombiner.class);
//++reduce:
job.setGroupingComparatorClass(TGroupingComparator.class);
job.setReducerClass(TReducer.class);
//++输入输出
Path input = new Path("hdfs://192.168.110.110:9000/data/tq/input");
FileInputFormat.addInputPath(job,input);
Path output = new Path("hdfs://192.168.110.110:9000/data/tq/output");
if(output.getFileSystem(conf).exists(output)) {
output.getFileSystem(conf).delete(output,true);
}
FileOutputFormat.setOutputPath(job,output);
job.setNumReduceTasks(2);
//--------------
job.waitForCompletion(true);
}
}
TMaper.java : map程序
package com.SL.tq;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.text.ParseException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class TMaper extends Mapper{
TQ mkey = new TQ();
IntWritable mval = new IntWritable();
@Override
protected void map(LongWritable key, Text value, Mapper.Context context)
throws IOException, InterruptedException {
// 1949-10-01 14:21:02 34c
try{
String[] strs = StringUtils.split(value.toString(),"\t");
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
Date date = sdf.parse(strs[0]);
Calendar cal = Calendar.getInstance();
cal.setTime(date);
mkey.setYear(cal.get(Calendar.YEAR));
mkey.setMonth(cal.get(Calendar.MONTH)+1);
mkey.setDay(cal.get(Calendar.DAY_OF_MONTH));
int wd = Integer.parseInt(strs[1].substring(0,strs[1].length()-1));
mkey.setWd(wd);
mval.set(wd);
System.out.println(mkey+" "+mval);
context.write(mkey, mval);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
TPartitioner.java :分区器
package com.SL.tq;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Partitioner;
public class TPartitioner extends Partitioner {
@Override
public int getPartition(TQ key, IntWritable value, int numPartitions) {
return key.hashCode() % numPartitions;
}
}
TsortComparator.java :排序比较器
package com.SL.tq;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
public class TsortComparator extends WritableComparator{
public TsortComparator() {
super(TQ.class,true);
}
@Override
public int compare(WritableComparable a, WritableComparable b){
TQ t1 = (TQ)a;
TQ t2 = (TQ)b;
int c1 = Integer.compare(t1.getYear(), t2.getYear());
if (c1 == 0) {
int c2 = Integer.compare(t1.getMonth(),t2.getMonth());
if(c2 == 0) {
return Integer.compare(t1.getWd(),t2.getWd());
}
return c2;
}
return c1;
}
}
TQ.java :序列化
package com.SL.tq;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;
public class TQ implements WritableComparable{
private int year;
private int month;
private int day;
private int wd;
public int getYear() {
return year;
}
public void setYear(int year) {
this.year = year;
}
public int getMonth() {
return month;
}
public void setMonth(int month) {
this.month = month;
}
public int getDay() {
return day;
}
public void setDay(int day) {
this.day = day;
}
public int getWd() {
return wd;
}
public void setWd(int wd) {
this.wd = wd;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(year);
out.writeInt(month);
out.writeInt(day);
out.writeInt(wd);
}
@Override
public void readFields(DataInput in) throws IOException {
this.year=in.readInt();
this.month=in.readInt();
this.day=in.readInt();
this.wd=in.readInt();
}
@Override
public int compareTo(TQ that) {
// 约定俗称:日期正序
int c1 = Integer.compare(this.year,that.getYear());
if(c1 == 0) {
int c2 = Integer.compare(this.month,that.getMonth());
if (c2 == 0) {
return Integer.compare(this.day, that.getDay());
}
return c2;
}
return c1;
}
}
TsortComparator .java :自定义排序
package com.SL.tq;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
public class TsortComparator extends WritableComparator{
public TsortComparator() {
super(TQ.class,true);
}
@Override
public int compare(WritableComparable a, WritableComparable b){
TQ t1 = (TQ)a;
TQ t2 = (TQ)b;
int c1 = Integer.compare(t1.getYear(), t2.getYear());
if (c1 == 0) {
int c2 = Integer.compare(t1.getMonth(),t2.getMonth());
if(c2 == 0) {
return Integer.compare(t1.getWd(),t2.getWd());
}
return c2;
}
return c1;
}
}