Java爬虫（一）利用GET和POST发送请求，获取服务器返回信息
本人所使用软件
- eclipse
- fiddle
- UC浏览器
分析请求信息
以知乎(https://www.zhihu.com)为例，模拟登陆请求，获取登陆后首页，首先就是分析请求信息。
用UC浏览器F12，点击Network，按F5刷新。使用自己账号登陆知乎后，点www.zhihu.com网址后，出现以下界面
在General中，看到请求方式是GET，在fiddle里请求构造中，方法选定GET。
下拉后，看到Request Header，将里面所有的内容复制下来，粘贴到fiddle的请求构造里
点击Execute，在fiddle中点击访问的网址，点击嗅探，点击下方的TextView，发现并没有显示内容
删除部分无用的Request Header，点击Execute，发现返回数据成功！
发送请求信息，获取数据
从以上的分析可以知道，访问知乎需要的Request Header，只需要有COOKIE就足够了，因此，我们将请求构造里帮我们格式化的内容，复制到txt文件内
接下来就是使用Java发送请求信息了。发送请求信息很简单，就直接贴出代码共同讨论。
package Main;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.util.HashMap;
import java.util.Map;
import Utils.HttpUtils;
import Utils.HttpUtils.OnVisitingListener;
import Utils.StreamUtils;
import Utils.StreamUtils.OnGetStringListener;
public class Main {
public static void main(String[] args) {
// 获取网页数据
getWebData();
// 设置参数
// 得到返回数据
}
private static void getWebData() {
HttpUtils httpUtils = HttpUtils.newInstance();
httpUtils.setOnVisitingListener(new OnVisitingListener() {
@Override
public void onSuccess(HttpURLConnection conn) {
try {
InputStream inputStream = conn.getInputStream();
String string = StreamUtils.getString(inputStream);
System.out.println(string);
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void onSetDetails(HttpURLConnection conn, HttpUtils httpUtils) {
Map<String, String> map = new HashMap<String, String>();
StreamUtils.getString("requestheader.txt", new OnGetStringListener() {
@Override
public void onGeted() {
}
@Override
public void onGetString(String line) {
System.out.println(line);
String[] strings = line.split(":");
map.put(strings[0], strings[1]);
}
});
httpUtils.setRequestProperties(map);
}
@Override
public void onFail(IOException e) {
}
}).startConnenction("https://www.zhihu.com/", "GET");
}}
- Utils封装工具类
package Utils;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
/**
* Created by admin on 2016/3/2.
*/
public class HttpUtils {
private HttpURLConnection conn;
public void setConnection(String fileUrl, String method) throws IOException {
URL url = new URL(fileUrl);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod(method);
conn.setConnectTimeout(5000);
conn.setReadTimeout(5000);
listener.onSetDetails(conn, this);
conn.connect();
}
OnVisitingListener listener;
public interface OnVisitingListener {
void onSuccess(HttpURLConnection conn);
void onSetDetails(HttpURLConnection conn, HttpUtils httpUtils);
void onFail(IOException e);
}
public HttpUtils setOnVisitingListener(OnVisitingListener listener) {
this.listener = listener;
return this;
}
public void startConnenction(String url, String method) {
try {
setConnection(url, method);
if (conn.getResponseCode() == 200) {
listener.onSuccess(conn);
} else {
throw new IOException();
}
} catch (IOException e) {
listener.onFail(e);
}
// if (conn != null) {
// conn.disconnect();
// }
}
public void setRequestProperties(Map<String, String> map) {
String key;
String value;
Set<String> set = map.keySet();
Iterator<String> it = set.iterator();
while (it.hasNext()) {
key = it.next();
value = map.get(key);
conn.setRequestProperty(key, value);
}
}
public void setRequestBody(String body) {
try {
PrintWriter writer = new PrintWriter(conn.getOutputStream());
writer.write(body);
writer.flush();
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public void setRequestProperty(String type, String value) {
conn.setRequestProperty(type, value);
}
public static HttpUtils newInstance() {
return new HttpUtils();
}
}
package Utils;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class RegexUtils {
public static String RegexGroup(String targetStr, String patternStr, int which) {
Pattern pattern = Pattern.compile(patternStr);
Matcher matcher = pattern.matcher(targetStr);
if (matcher.find()) {
return matcher.group(which);
}
return “Nothing!”;
}
public static List<String> RegexGroups(String targetStr, String patternStr, int which) {
Pattern pattern = Pattern.compile(patternStr);
Matcher matcher = pattern.matcher(targetStr);
List<String> list = new ArrayList<String>();
while (matcher.find()) {
list.add(matcher.group(which));
}
return list;
}
public static String RegexString(String targetStr, String patternStr) {
Pattern pattern = Pattern.compile(patternStr);
Matcher matcher = pattern.matcher(targetStr);
if (matcher.find()) {
return matcher.group();
}
return "Nothing!";
}}
package Utils;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
/**
* Created by admin on 2016/2/18.
*/
public class StreamUtils {
public static String readFromStream(InputStream inputStream) throws IOException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
int len = 0;
byte[] buffer = new byte[1024];
while ((len = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, len);
}
String result = outputStream.toString();
inputStream.close();
outputStream.close();
return result;
}
private static String line;
public static FileReader createFileReader(File file) throws FileNotFoundException {
return new FileReader(file);
}
public static FileWriter createFileWriter(File file) throws IOException {
return new FileWriter(file);
}
public static InputStreamReader createInputStreamReader(Object obj) {
if (obj instanceof File)
if (!((File) obj).exists())
((File) obj).getParentFile().mkdirs();
try {
return new InputStreamReader(new FileInputStream((File) obj), "utf-8");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
}
return null;
}
public static OutputStreamWriter createOutputStreamWriter(Object obj) {
if (obj instanceof File)
if (!((File) obj).exists())
((File) obj).getParentFile().mkdirs();
try {
return new OutputStreamWriter(new FileOutputStream((File) obj, true), "utf-8");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
}
return null;
}
public static BufferedReader createBufferedReader(Object obj, String cd) throws IOException {
if (obj instanceof String)
return new BufferedReader(createInputStreamReader(new File((String) obj)));
if (obj instanceof InputStream) {
if (cd == null)
return new BufferedReader(new InputStreamReader((InputStream) obj));
else
return new BufferedReader(new InputStreamReader((InputStream) obj, cd));
}
if (obj instanceof File) {
if (!((File) obj).exists())
((File) obj).createNewFile();
return new BufferedReader(createFileReader((File) obj));
}
if (obj instanceof Reader)
return new BufferedReader((Reader) obj);
if (obj instanceof BufferedReader)
return (BufferedReader) obj;
return null;
}
public static BufferedWriter createBufferedWriter(Object obj) throws IOException {
if (obj instanceof String)
return new BufferedWriter(createOutputStreamWriter(new File((String) obj)));
if (obj instanceof OutputStream)
return new BufferedWriter(new OutputStreamWriter((OutputStream) obj, "utf-8"));
if (obj instanceof File)
return new BufferedWriter(createOutputStreamWriter(obj));
if (obj instanceof Writer)
return new BufferedWriter((Writer) obj);
if (obj instanceof BufferedWriter)
return (BufferedWriter) obj;
return null;
}
public interface OnGetStringListener {
void onGetString(String line);
void onGeted();
}
public static void getString(Object obj, OnGetStringListener listener) {
BufferedReader br;
try {
br = createBufferedReader(obj, null);
if (br != null) {
while ((line = br.readLine()) != null) {
listener.onGetString(line);
}
listener.onGeted();
br.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static String getString(Object obj) {
BufferedReader br;
String str = "";
try {
br = createBufferedReader(obj, "utf-8");
if (br != null) {
while ((line = br.readLine()) != null) {
str += line + "\n";
}
}
} catch (IOException e) {
e.printStackTrace();
}
return str;
}
public static void writeString(Object obj, String str) {
BufferedWriter bw;
try {
bw = createBufferedWriter(obj);
if (bw != null) {
bw.write(str);
bw.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}