Hadoop之电信日志数据处理（二）------mapreducer端处理
实体类实现：
public class HttpAPPHost implements Writable{
//发送报文时间
private String reportTime;
private String cellid;
private int appType;
private int appSubType;
//用户ip
private String userIP;
//用户端口
private int userPort;
//服务器ip
private String appServerIP;
//服务器端口
private int appServerPort;
private String host;
//连接请求
private int attempts;
//响应请求
private int accepts;
//上行
private long trafficUL;
//下行
private long trafficDL;
//重传上行
private long retranUL;
//重传下行
private long retranDL;
private long transDelay;
@Override
public String toString() {
return "HttpAPPHost [reportTime=" + reportTime + ", cellid=" + cellid + ", appType=" + appType + ", appSubType="
+ appSubType + ", userIP=" + userIP + ", userPort=" + userPort + ", appServerIP=" + appServerIP
+ ", appServerPort=" + appServerPort + ", host=" + host + ", attempts=" + attempts + ", accepts="
+ accepts + ", trafficUL=" + trafficUL + ", trafficDL=" + trafficDL + ", retranUL=" + retranUL
+ ", retranDL=" + retranDL + ", transDelay=" + transDelay + "]";
}
public HttpAPPHost() {}
public HttpAPPHost(String reportTime, String cellid, int appType, int appSubType, String userIP, int userPort,
String appServerIP, int appServerPort, String host, int attempts, int accepts, long trafficUL,
long trafficDL, long retranUL, long retranDL, long transDelay) {
this.reportTime = reportTime;
this.cellid = cellid;
this.appType = appType;
this.appSubType = appSubType;
this.userIP = userIP;
this.userPort = userPort;
this.appServerIP = appServerIP;
this.appServerPort = appServerPort;
this.host = host;
this.attempts = attempts;
this.accepts = accepts;
this.trafficUL = trafficUL;
this.trafficDL = trafficDL;
this.retranUL = retranUL;
this.retranDL = retranDL;
this.transDelay = transDelay;
}
public String getReportTime() {
return reportTime;
}
public void setReportTime(String reportTime) {
this.reportTime = reportTime;
}
public String getCellid() {
return cellid;
}
public void setCellid(String cellid) {
this.cellid = cellid;
}
public int getAppType() {
return appType;
}
public void setAppType(int appType) {
this.appType = appType;
}
public int getAppSubType() {
return appSubType;
}
public void setAppSubType(int appSubType) {
this.appSubType = appSubType;
}
public String getUserIP() {
return userIP;
}
public void setUserIP(String userIP) {
this.userIP = userIP;
}
public int getUserPort() {
return userPort;
}
public void setUserPort(int userPort) {
this.userPort = userPort;
}
public String getAppServerIP() {
return appServerIP;
}
public void setAppServerIP(String appServerIP) {
this.appServerIP = appServerIP;
}
public int getAppServerPort() {
return appServerPort;
}
public void setAppServerPort(int appServerPort) {
this.appServerPort = appServerPort;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getAttempts() {
return attempts;
}
public void setAttempts(int attempts) {
this.attempts = attempts;
}
public int getAccepts() {
return accepts;
}
public void setAccepts(int accepts) {
this.accepts = accepts;
}
public long getTrafficUL() {
return trafficUL;
}
public void setTrafficUL(long trafficUL) {
this.trafficUL = trafficUL;
}
public long getTrafficDL() {
return trafficDL;
}
public void setTrafficDL(long trafficDL) {
this.trafficDL = trafficDL;
}
public long getRetranUL() {
return retranUL;
}
public void setRetranUL(long retranUL) {
this.retranUL = retranUL;
}
public long getRetranDL() {
return retranDL;
}
public void setRetranDL(long retranDL) {
this.retranDL = retranDL;
}
public long getTransDelay() {
return transDelay;
}
public void setTransDelay(long transDelay) {
this.transDelay = transDelay;
}
@Override
public void readFields(DataInput in) throws IOException {
// TODO Auto-generated method stub
this.reportTime=in.readUTF();
this.cellid=in.readUTF();
this.appType=in.readInt();
this.appSubType=in.readInt();
this.userIP=in.readUTF();
this.userPort=in.readInt();
this.appServerIP=in.readUTF();
this.appServerPort=in.readInt();
this.host=in.readUTF();
this.attempts=in.readInt();
this.accepts=in.readInt();
this.trafficUL=in.readLong();
this.trafficDL=in.readLong();
this.retranUL=in.readLong();
this.retranDL=in.readLong();
this.transDelay=in.readLong();
}
@Override
public void write(DataOutput out) throws IOException {
// TODO Auto-generated method stub
out.writeUTF(reportTime);
out.writeUTF(cellid);
out.writeInt(appType);
out.writeInt(appSubType);
out.writeUTF(userIP);
out.writeInt(userPort);
out.writeUTF(appServerIP);
out.writeInt(appServerPort);
out.writeUTF(host);
out.writeInt(attempts);
out.writeInt(accepts);
out.writeLong(trafficUL);
out.writeLong(trafficDL);
out.writeLong(retranUL);
out.writeLong(retranDL);
out.writeLong(transDelay);
}
}
map端代码：
public class HahMap extends Mapper {
public void map(LongWritable key, Text value,Mapper.Context context)
throws IOException, InterruptedException {
//获取行数据,按照|切分
String line=value.toString();
String [] data=line.split("\\|");
//获取用户名
FileSplit fileSplit=(FileSplit)context.getInputSplit();
String filename=fileSplit.getPath().getName();
//遍历,创建javabean对象
HttpAPPHost haha=new HttpAPPHost();
//创建小区id
haha.setCellid(data[16]);
//应用大类
haha.setAppType(Integer.parseInt(data[22]));
//应用小类
haha.setAppSubType(Integer.parseInt(data[23]));
//用户ip
haha.setUserIP(data[26]);
//用户端口号
haha.setUserPort(Integer.parseInt(data[28]));
//服务器ip
haha.setAppServerIP(data[30]);
//服务器端口号
haha.setAppServerPort(Integer.parseInt(data[32]));
//域名
haha.setHost(data[58]);
//日志生成时间
haha.setReportTime(filename.split("_")[1]);
//获取请求状态码----状态码是103表示请求成功
int appTypeCode=Integer.parseInt(data[18]);
//传输状态码
String transStatus=data[54];
if(haha.getCellid()==null||haha.getCellid().equals(""))
{
//如果小区id为null,或者为空时,就在当前字段补齐9个0
haha.setCellid("000000000");
}
//当请求成功 -103 尝试次数为1
if(appTypeCode == 103) {
haha.setAttempts(1);
}
//如果请求成功--103,响应成功设置次数为1
if (appTypeCode==103
&&"10,11,12,13,14,15,32,33,34,35,36,37,38,48,49,50,51,52,53,54,55,199,200,201,202,203,204,205,206,302,304,306".contains(transStatus)) {
haha.setAccepts(1);
}else {
haha.setAccepts(0);
}
//如果请求成功----设置上行流量
if(appTypeCode==103) {
haha.setTrafficUL(Long.parseLong(data[33]));
}
//如果请求成功,设置下行流量
if(appTypeCode==103) {
haha.setTrafficDL(Long.parseLong(data[34]));
}
//如果成功,设置重传上行流量(重新发起请求服务产生的流量)
if(appTypeCode==103) {
haha.setRetranUL(Long.parseLong(data[39]));
}
//如果成功设置,设置重传下行流量(重新发起下载请求服务产生的流量)
if(appTypeCode==103) {
haha.setRetranDL(Long.parseLong(data[40]));
}
//设置传输总用时
if(appTypeCode==103) {
haha.setTransDelay(Long.parseLong(data[20])-Long.parseLong(data[19]));
}
//进行字段结算,封装
String userKey=haha.getReportTime()+"|"+haha.getAppType()+"|"+haha.getAppSubType()+"|"+haha.getUserIP()+"|"
+haha.getUserPort()+"|"+haha.getAppServerIP()+"|"+haha.getAppServerPort()+"|"+haha.getHost()+"|"+
haha.getCellid();
//输出数据
context.write(new Text(userKey),haha);
}
} reduce端代码：
public class HaHaReducer extends Reducer {
public void reduce(Text key, Iterable values,
Reducer.Context context)
throws IOException, InterruptedException {
// process values
//遍历
Iterator it=values.iterator();
if(it.hasNext()) {
//获取属性
HttpAPPHost haha=it.next();
while(it.hasNext()) {
HttpAPPHost otherhaha=it.next();
//累加总的尝试请求
haha.setAttempts(otherhaha.getAttempts()+haha.getAttempts());
//累加总的接受次数
haha.setAccepts(otherhaha.getAccepts()+haha.getAccepts());
//累加总的上行流量
haha.setTrafficUL(otherhaha.getTrafficUL()+haha.getTrafficUL());
//累加总的下行流量
haha.setTrafficDL(otherhaha.getTrafficDL()+haha.getTrafficDL());
//累加总的重传上行流量
haha.setRetranUL(otherhaha.getRetranUL()+haha.getRetranUL());
//累加总的重传下行流量
haha.setRetranDL(otherhaha.getRetranDL()+haha.getRetranDL());
//累加总的请求时间
haha.setTransDelay(otherhaha.getTransDelay()+haha.getTransDelay());
}
//还需要一些基本的字段进行封装
String time=haha.getReportTime();
time=time.substring(0,4)+"-"+time.substring(4,6)+"-"+time.substring(6,8)+" "+
time.substring(8,10)+":"+time.substring(10,12)+":"+time.substring(12,14);
String result=time+"|"+haha.getCellid()+"|"+haha.getAppType()+"|"+haha.getAppSubType()+"|"
+haha.getUserIP()+"|"+haha.getUserPort()+"|"+haha.getAppServerIP()+"|"+haha.getAppServerPort()+
"|"+haha.getHost()+"|"+haha.getAttempts()+"|"+haha.getAccepts()+"|"+haha.getTrafficUL()+
"|"+haha.getTrafficDL()+"|"+haha.getRetranUL()+"|"+haha.getRetranDL()+"|"+haha.getReportTime();
//输出数据
context.write(new Text(result),NullWritable.get());
}
}
} driver端实现：
public class HaHaDriver {
public static void main(String[] args) throws Exception {
//声明作业对象
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "JobName");
//声明入口
job.setJarByClass(pdsu.edu.cn.HaHaDriver.class);
//声明mapper类
job.setMapperClass(pdsu.edu.cn.HahMap.class);
// 声明reducer类
job.setReducerClass(pdsu.edu.cn.HaHaReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(HttpAPPHost.class);
//设置文件的处理路径及存储路径
FileInputFormat.setInputPaths(job, new Path("hdfs://rjxy/haha"));
FileOutputFormat.setOutputPath(job, new Path("hdfs://rjxy/haha/result"));
if (!job.waitForCompletion(true))
return;
}
}
Hadoop之电信日志数据处理（一）------业务简介
Hadoop之电信日志数据处理（三）------创建数据库及数据收集
Hadoop之电信日志数据处理（四）-------web界面实现及可视化效果展示