import java.text.SimpleDateFormat;
import java.util.*;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.*;
import scala.Tuple2;
public class SparkDemo {
static final String USER = "huanglong";
public static void main(String[] args) throws Exception {
SparkConf conf =new SparkConf().setAppName("Spark Log").setMaster("local[4]");
JavaSparkContext sc = new JavaSparkContext(conf); //其底层就是scala的sparkcontext
//JavaSparkContext sc = new JavaSparkContext("local[4]", "Spark Log", "/user/huanglong/spark", new String[0], envs);
String file="file:///Users/huanglong/access_05_30.log";
JavaRDD data = sc.textFile(file, 4).cache();
//日志格式
//27.19.74.143 - - [30/May/2013:17:38:20 +0800] "GET /static/image/common/faq.gif HTTP/1.1" 200 1127;
JavaRDD filter = data.filter(new org.apache.spark.api.java.function.Function() {
@Override
public Boolean call(String s) throws Exception {
//过滤合法和非静态的资源的日志
return (s.contains("POST")||s.contains("GET"))&& !s.contains("/static/");
}
});
final SimpleDateFormat tf = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z",
Locale.ENGLISH);
//只有一天的记录，不需要年月日了,时间精度到分钟
final SimpleDateFormat tf1 = new SimpleDateFormat("HH:mm");
JavaRDD> lines = filter.flatMap(new FlatMapFunction>(){
@Override
public Iterator> call(String s) throws Exception {
List> list = new ArrayList>(1);
Map map = new HashMap();
list.add(map);
String ip = s.substring(0,s.indexOf(" - - "));
map.put("ip",ip);
String t = s.substring(s.indexOf("[")+1,s.indexOf("]"));
Date date = tf.parse(t);
map.put("time",tf1.format(date));
String http = s.substring(s.indexOf("\"")+1,s.indexOf("\" "));
String[] h= http.split(" ");
if(h.length>1){
if(h[1]!=null&&h[1].indexOf("?")!=-1)
{
map.put("url",h[1].substring(0,h[1].indexOf("?")));
}else {
map.put("url",h[1]);
}
}
if(h.length>2)
{
map.put("version",h[2]);
}
if(map.get("version")!=null&&map.get("version").contains("1.1"))
{
String state = s.substring(s.indexOf("1.1\" ")+5,s.length());
String[] strings = state.split(" ");
if(strings.length>0) {
map.put("state", strings[0]);
}
}else if(map.get("version")!=null&&map.get("version").contains("1.0"))
{
String state = s.substring(s.indexOf("1.0\" ")+5,s.length());
String[] strings = state.split(" ");
if(strings.length>0) {
map.put("state", strings[0]);
}
}else {
//System.out.println(s);
}
return list.iterator();
}
});
System.out.println("总记录数:"+lines.count());
// 统计ip
ipsHandle(lines);
System.out.println("--------------------------------------------------");
// 统计分时页面访问量
timeHandle(lines);
System.out.println("--------------------------------------------------");
// 统计分时ip访问量
timeHandle2(lines);
System.out.println("--------------------------------------------------");
// 统计状态 400，300 200 各有多少,计算失败率
stateHandle(lines);
System.out.println("--------------------------------------------------");
// 统计http版本 1.1、1.0
versionHandle(lines);
// 页面热度
System.out.println("--------------------------------------------------");
pageHandle(lines);
sc.close();
}
private static void pageHandle(JavaRDD> lines) {
JavaRDD> filterData = lines.filter(new Function, Boolean>() {
@Override
public Boolean call(Map map) throws Exception {
String url = map.get("url");
return "200".equals(map.get("state"))&&url.endsWith(".php");
}
});
JavaPairRDD versionPairs = filterData.mapToPair(new PairFunction, String,Integer>() {
@Override
public Tuple2 call(Map stringStringMap) throws Exception {
return new Tuple2(stringStringMap.get("url"),1);
}
});
JavaPairRDD versionCount = versionPairs.reduceByKey(new Function2() {
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
return v1+v2;
}
});
versionCount.foreach(new VoidFunction>() {
@Override
public void call(Tuple2 pairs) throws Exception {
System.out.println("URL->PV："+pairs._1+" : " +pairs._2);
}
});
}
private static void versionHandle(JavaRDD> lines) {
JavaPairRDD versionPairs = lines.mapToPair(new PairFunction, String,Integer>() {
@Override
public Tuple2 call(Map stringStringMap) throws Exception {
return new Tuple2(stringStringMap.get("version"),1);
}
});
JavaPairRDD versionCount = versionPairs.reduceByKey(new Function2() {
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
return v1+v2;
}
});
versionCount.foreach(new VoidFunction>() {
@Override
public void call(Tuple2 pairs) throws Exception {
System.out.println("VERSION->PV："+pairs._1+" : " +pairs._2);
}
});
}
private static void stateHandle(JavaRDD> lines) {
JavaPairRDD statePairs = lines.mapToPair(new PairFunction, String,Integer>() {
@Override
public Tuple2 call(Map stringStringMap) throws Exception {
return new Tuple2(stringStringMap.get("state"),1);
}
});
JavaPairRDD stateCount = statePairs.reduceByKey(new Function2() {
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
return v1+v2;
}
});
stateCount.foreach(new VoidFunction>() {
@Override
public void call(Tuple2 pairs) throws Exception {
System.out.println("STATE->PV："+pairs._1+" : " +pairs._2);
}
});
}
private static void ipsHandle(JavaRDD> lines) {
JavaPairRDD ipPairs = lines.mapToPair(new PairFunction, String,Integer>() {
@Override
public Tuple2 call(Map stringStringMap) throws Exception {
return new Tuple2(stringStringMap.get("ip"),1);
}
});
JavaPairRDD ipCount = ipPairs.reduceByKey(new Function2() {
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
return v1+v2;
}
});
System.out.println("总ip数:"+ipCount.count());
ipCount.foreach(new VoidFunction>() {
@Override
public void call(Tuple2 pairs) throws Exception {
System.out.println("IP->PV："+pairs._1+" : " +pairs._2);
}
});
}
private static void timeHandle2(JavaRDD> lines) {
JavaPairRDD> timePairs = lines.mapToPair(new PairFunction, String,Map>() {
@Override
public Tuple2> call(Map stringStringMap) throws Exception {
Map m = new HashMap();
m.put(stringStringMap.get("ip"),Boolean.TRUE);
return new Tuple2>(stringStringMap.get("time"),m);
}
});
JavaPairRDD> timeCount = timePairs.reduceByKey(new Function2, Map, Map>() {
@Override
public Map call(Map v1, Map v2) throws Exception {
HashMap m = new HashMap();
m.putAll(v1);
m.putAll(v2);
return m;
}
});
timeCount.foreach(new VoidFunction>>() {
@Override
public void call(Tuple2> pairs) throws Exception {
System.out.println("TIME->IP："+pairs._1+" : " +pairs._2.size());
}
});
}
private static void timeHandle(JavaRDD> lines) {
JavaPairRDD timePairs = lines.mapToPair(new PairFunction, String,Integer>() {
@Override
public Tuple2 call(Map stringStringMap) throws Exception {
return new Tuple2(stringStringMap.get("time"),1);
}
});
JavaPairRDD timeCount = timePairs.reduceByKey(new Function2() {
@Override
public Integer call(Integer v1, Integer v2) throws Exception {
return v1+v2;
}
});
timeCount.foreach(new VoidFunction>() {
@Override
public void call(Tuple2 pairs) throws Exception {
System.out.println("TIME->PV："+pairs._1+" : " +pairs._2);
}
});
}
}
