JAVA 根据URL获取JSON数据，并解析后存入数据库
1、主函数：设置定时器根据一定时间获取JSON数据并存储
package Main;
import getJson.GET;//自定义的获取JSON函数
import java.io.PrintWriter;//外部函数
import java.io.StringWriter;
import java.util.Date;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.Logger;//日志
import org.apache.log4j.PropertyConfigurator;
import city.PublicName;//自定义函数
public class USE {
public static String getTrace(Throwable t){
StringWriter stringWriter =new StringWriter();
PrintWriter writer = new PrintWriter(stringWriter);
t.printStackTrace(writer);
StringBuffer buffer=stringWriter.getBuffer();
return buffer.toString();
}
public static void main(String[] args) throws Exception {
PropertyConfigurator.configure("log4j.properties");//定义日志
Logger logger =Logger.getLogger(USE.class);
logger.debug("debug");
String cityName = "";
int cityNUM=0;
cityName = PublicName.getCityName(cityNUM);
while(cityName != " "){
PublicName.setCityValue(cityName,0);
cityNUM++;
cityName = PublicName.getCityName(cityNUM);
}
for(int val=0;val<3;val++){
PublicName.setVal(val, 0);
}
ScheduledExecutorService execService = Executors.newScheduledThreadPool(3);
execService.scheduleAtFixedRate(new Runnable() { //按照固定频率定时，本次设置时间间隔位3分钟，第一个参数为程序运行多久后开始执行
public void run() {
Logger logger =Logger.getLogger(USE.class);
logger.debug("debug");
int result = 0;
result = GET.getWeatherInform("七台河"); //1表示开始获取数据
if (result == 6) {
System.out.println("===========================\n"
+ "=== 数据获取成功 ===\n"
+ "======= =======\n");
logger.info("===============================\n"
+ "=== 数据获取成功 ======\n"
+ "==== ======\n");
System.out.print("当前时间===" + new Date());
} else {
System.out.println("========数据获取失败，失败代码======" + result);
logger.info("========数据获取失败，失败代码======" + result);
System.out.print("当前时间===" + new Date());
// logger.info("当前时间===" + new Date());
return;
}
}
}, 1, 3*60, TimeUnit.SECONDS);
}
}package getJson;
import internet.Test;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.sql.SQLException;
import Main.USE;//主函数
import jxJson.JX;//自定义解析函数
import org.apache.log4j.Logger;
public class GET {
public static int getWeatherInform(String cityName){
Logger logger =Logger.getLogger(USE.class);
int state = -1;
String mytext = null;
try {
mytext = java.net.URLEncoder.encode(cityName, "utf-8");
} catch (UnsupportedEncodingException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
String PM2_5Url = "http://www.pm25.in/api/querys/aqi_details.json?city="+mytext+"&token=5j1znBVAsnSf5xQyNQyq"; // 公共key
System.out.println("=== " + cityName + " start! ===");
logger.info("=== " + cityName + " start! ===");
int checkNum;
try {
checkNum=Test.check();//测试网络
if(checkNum==0)
return 0;
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
logger.info(USE.getTrace(e1));
}
StringBuffer strBuf;
strBuf = new StringBuffer();
try {
URL url = new URL(PM2_5Url);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();//设置网络获取时间间隔，超出时间后跳出
conn.setConnectTimeout(30000);
conn.setReadTimeout(30000);
state = conn.getResponseCode();
if (state == 200) {
BufferedReader reader = new BufferedReader(
new InputStreamReader(conn.getInputStream(),
"utf-8"));// 转码。
String line = null;
while ((line = reader.readLine()) != null)
strBuf.append(line + " ");
System.out.println("connect success");
logger.info("connect success");
reader.close();
} else {
System.out.println("connect false" + cityName);
logger.info("connect false" + cityName);
return 0;
}
} catch (MalformedURLException e) {
e.printStackTrace();
logger.info(USE.getTrace(e));
} catch (ConnectException ce) {
ce.printStackTrace();
logger.info(USE.getTrace(ce));
} catch (IOException e) {
e.printStackTrace();
logger.info(USE.getTrace(e));
}
// }
String json = strBuf.toString();//判断获取的数据
if (json == null || json.isEmpty()) {
System.out.println("++++++++字符串为空+++++++");
logger.info("++++++++字符串为空+++++++");
return 2; // 返回2表示字符串为空
} else if (json.charAt(0) == '{') {
System.out.println(json);
logger.info(json);
return 3; // 返回3表示字符串以{开头
}else if (json.charAt(0) == '['){
try {
JX.jxJson(json,cityName);//调用解析函数
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
else {
System.out.println("+++++++未知问题++++++++");
logger.info("++++++++未知问题+++++++");
return 4; //返回4表示未知问题
}
System.out.println(cityName + ", The end!!!");
logger.info(cityName + ", The end!!!\n");
return 6;
}
}
package internet;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import Main.USE;
public class Test {
static BufferedReader bufferedReader;
public static int check () throws IOException{
PropertyConfigurator.configure("log4j.properties");
Logger logger =Logger.getLogger(USE.class);
logger.debug("debug");
String address="www.baidu.com";
int state=0;
try {
Process process=Runtime.getRuntime().exec("ping "+address+" -n 1");
bufferedReader=new BufferedReader(new InputStreamReader(process.getInputStream()));
String line="";
String connectionStr=null;
while ((line = bufferedReader.readLine())!=null) {
connectionStr += line;
}
System.out.println(connectionStr);
logger.info(connectionStr);
if(connectionStr.indexOf("0% 丢失")!=-1){
System.out.println("===the internet working===");
logger.info("===the internet working===");
state= 1;
}
else
{
System.out.println("else===the internet dont working ===");
logger.info("else===the internet dont working===");
state= 0;
}
} catch(IOException e){
System.out.println("Exception===the internet dont working ===");
logger.info("Exception===the internet dont working===");
state= 0;
e.printStackTrace();
}finally{
System.out.println("===finally ===");
logger.info("===finally===");
bufferedReader.close();
}
return state;
}
}
package jxJson;
import getJson.GET;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import org.apache.log4j.Logger;
import Main.USE;
import city.PublicName;
import net.sf.json.JSONArray;
public class JX {
public static int jxJson(String json,String cityname) throws SQLException {
Logger logger =Logger.getLogger(USE.class);
// 连接数据库
// int state = 0;
for (int k = 0; k < 5; ++k) {
try {
Class.forName("org.postgresql.Driver");
} catch (ClassNotFoundException e) {
System.out.println("Where is your PostgreSQL JDBC Driver? "
+ "Include in your library path!");
e.printStackTrace();
logger.info("Where is your PostgreSQL JDBC Driver? "
+ "Include in your library path!");
logger.info(USE.getTrace(e));
continue;
}
System.out.println("PostgreSQL JDBC Driver Registered!");
logger.info("PostgreSQL JDBC Driver Registered!");
Connection connection = null;
try {
connection = DriverManager.getConnection(
"jdbc:postgresql://localhost:5432/postgres",
"postgres", "shmily");
} catch (SQLException e) {
System.out.println("Connection Failed! Check output console");
e.printStackTrace();
logger.info("Connection Failed! Check output console");
logger.info(USE.getTrace(e));
continue;
/*
* try { Thread.sleep(1000*10); } catch (InterruptedException
* e1) { // TODO Auto-generated catch block
* e1.printStackTrace(); }
*/
}
if (connection != null) {
System.out
.println("You made it, take control your database now!");
logger.info("You made it, take control your database now!");
} else {
System.out.println("Failed to make connection!");
logger.info("Failed to make connection!");
}
JSONArray jsonArr = JSONArray.fromObject(json);
String aqi[] = new String[jsonArr.size()];
String area[] = new String[jsonArr.size()];
String co[] = new String[jsonArr.size()];
String co_24h[] = new String[jsonArr.size()];
String no2[] = new String[jsonArr.size()];
String no2_24h[] = new String[jsonArr.size()];
String o3[] = new String[jsonArr.size()];
String o3_24h[] = new String[jsonArr.size()];
String o3_8h[] = new String[jsonArr.size()];
String o3_8h_24h[] = new String[jsonArr.size()];
String pm10[] = new String[jsonArr.size()];
String pm10_24h[] = new String[jsonArr.size()];
String pm2_5[] = new String[jsonArr.size()];
String pm2_5_24h[] = new String[jsonArr.size()];
String position_name[] = new String[jsonArr.size()];
String primary_pollutant[] = new String[jsonArr.size()];
String quality[] = new String[jsonArr.size()];
String so2[] = new String[jsonArr.size()];
String so2_24h[] = new String[jsonArr.size()];
String station_code[] = new String[jsonArr.size()];
String time_point[] = new String[jsonArr.size()];
Statement stmt = null;
String strSQL = "";
String timePoint = "";
connection.setAutoCommit(false);
stmt=connection.createStatement();
for (int i = 0; i < jsonArr.size(); i++) {
aqi[i] = jsonArr.getJSONObject(i).getString("aqi");
area[i] = jsonArr.getJSONObject(i).getString("area");
co[i] = jsonArr.getJSONObject(i).getString("co");
co_24h[i] = jsonArr.getJSONObject(i).getString("co_24h");
no2[i] = jsonArr.getJSONObject(i).getString("no2");
no2_24h[i] = jsonArr.getJSONObject(i).getString("no2_24h");
o3[i] = jsonArr.getJSONObject(i).getString("o3");
o3_24h[i] = jsonArr.getJSONObject(i).getString("o3_24h");
o3_8h[i] = jsonArr.getJSONObject(i).getString("o3_8h");
o3_8h_24h[i] = jsonArr.getJSONObject(i).getString(
"o3_8h_24h");
pm10[i] = jsonArr.getJSONObject(i).getString("pm10");
pm10_24h[i] = jsonArr.getJSONObject(i)
.getString("pm10_24h");
pm2_5[i] = jsonArr.getJSONObject(i).getString("pm2_5");
pm2_5_24h[i] = jsonArr.getJSONObject(i).getString(
"pm2_5_24h");
position_name[i] = jsonArr.getJSONObject(i).getString(
"position_name");
primary_pollutant[i] = jsonArr.getJSONObject(i).getString(
"primary_pollutant");
quality[i] = jsonArr.getJSONObject(i).getString("quality");
so2[i] = jsonArr.getJSONObject(i).getString("so2");
so2_24h[i] = jsonArr.getJSONObject(i).getString("so2_24h");
station_code[i] = jsonArr.getJSONObject(i).getString(
"station_code");
time_point[i] = jsonArr.getJSONObject(i).getString(
"time_point");
timePoint = time_point[i];
strSQL = "INSERT into pm2_5_1 (aqi, area, co, co_24h, no2, no2_24h, o3, o3_24h, o3_8h, o3_8h_24h, pm10, pm10_24h, pm2_5, pm2_5_24h, position_name, primary_pollutant,quality, so2, so2_24h, station_code, time_point,created_time_with_tz,created_time_without_tz,id) values("
+ aqi[i]
+ ",'"
+ area[i]
+ "',"
+ co[i]
+ ","
+ co_24h[i]
+ ","
+ no2[i]
+ ","
+ no2_24h[i]
+ ","
+ o3[i]
+ ","
+ o3_24h[i]
+ ","
+ o3_8h[i]
+ ","
+ o3_8h_24h[i]
+ ","
+ pm10[i]
+ ","
+ pm10_24h[i]
+ ","
+ pm2_5[i]
+ ","
+ pm2_5_24h[i]
+ ",'"
+ position_name[i]
+ "','"
+ primary_pollutant[i]
+ "','"
+ quality[i]
+ "',"
+ so2[i]
+ ","
+ so2_24h[i]
+ ",'"
+ station_code[i]
+ "','"
+ time_point[i]
+ "',"
+ "now(),now()"
+ ",'"
+ area[i]
+ position_name[i]
+ time_point[i]
+ "');";
// if(x==0){
try {
/*
stmt = connection.createStatement(
ResultSet.TYPE_SCROLL_INSENSITIVE,
ResultSet.CONCUR_READ_ONLY);
*/
String sql = "select area from pm2_5_1 where area =" + "'"
+ area[i] + "' and position_name ='"+ position_name[i] +"' and time_point ='"+ time_point[i]
+ "';";
System.out.println("当前数据的时间====="+time_point[i]);
logger.info("当前数据的时间====="+time_point[i]);
ResultSet rs1 = stmt.executeQuery(sql); // 查询数据库看数据是否已经存在
if (rs1.next()) { // 该条数据已经存在
// PublicName.getValue(0,0);
System.out.println("dont need insert " + strSQL);
logger.info("dont need insert " + strSQL);
} else {
stmt.executeUpdate(strSQL);
System.out.println(strSQL);
}
rs1.close();
} catch (Exception e) {
e.printStackTrace();
logger.info(USE.getTrace(e));
}
}
//探测数据库
connection.commit();
PublicName.setCityValue(cityname, 1);
try {
int values=PublicName.getVal(1);
if (values==0) { // 判断是否检索过数据库
int numble = 0;
ArrayList cityName = new ArrayList();
ResultSet rs2 = stmt
.executeQuery("select distinct area from pm2_5_1 where time_point="
+ "'" + timePoint + "';");
while (rs2.next()) {
String str = rs2.getString("area");
cityName.add(str);
numble++;
// 探测数据库，修改 value
}
rs2.close();
for (int j = 0; j < numble; ++j) {
PublicName.setCityValue(cityName.get(j), 1);
}
stmt.close();
connection.close();
PublicName.setVal(1,1);
String cityNames = "";
int cityNUM=0;
cityNames = PublicName.getCityName(cityNUM);
while(cityNames != " "){
int temp=PublicName.getCityValue(cityNames);
if(temp==0){
GET.getWeatherInform(cityNames);
}
cityNUM++;
cityNames = PublicName.getCityName(cityNUM);
}
stmt.close();
connection.close();
String cityName1 = "";
int cityNUM1=0;
cityName1 = PublicName.getCityName(cityNUM1);
while(cityName1 != " "){
PublicName.setCityValue(cityName1,0);
cityNUM1++;
cityName1 = PublicName.getCityName(cityNUM1);
}
for(int val=0;val<3;val++){
PublicName.setVal(val, 0);
}
return 6;
} else {
stmt.close();
connection.close();
}
} catch (Exception e) {
e.printStackTrace();
logger.info(USE.getTrace(e));
}
k=5;
} //for结束
System.out.print("当前时间===" + new Date());
return 6;
}
}
所需的外部jar文件如附图。
