java下载网络文件WGet代码

阅读更多

写道

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;

public class WGet {
public static void main(String args[]) throws IOException{
String str=getText("http://bbs.jinghua.cn/viewthread.php?tid=52723&extra=page%3D1","GBK");
System.out.println(str);
}
private static void setConn(HttpURLConnection httpUrl) {
httpUrl.setRequestProperty(
"User-agent",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11");
httpUrl.setRequestProperty("User-agent", "Mozilla/4.0");
httpUrl.setRequestProperty(
"Accept",
"text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
httpUrl.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");
httpUrl.setRequestProperty("Accept-Charset",
"gb2312,utf-8;q=0.7,*;q=0.7");
httpUrl.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");
httpUrl.setRequestProperty("Keep-Alive", "300");
httpUrl.setRequestProperty("Connection", "keep-alive");
}

public static void writeFile(String txt, String path, String code) {
java.io.File f = new java.io.File(path);

f.getParentFile().mkdirs();
OutputStreamWriter out = null;
try {
out = new OutputStreamWriter(new FileOutputStream(path), code);
out.write(txt);
out.flush();

} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
out.close();
} catch (IOException e) {

e.printStackTrace();
}
}

}

public static boolean testUrl(String destUrl) throws IOException {
System.out.println("testUrlText@=" + destUrl);
boolean flag = true;
HttpURLConnection httpUrl = null;
try {

java.io.ByteArrayOutputStream bout = new java.io.ByteArrayOutputStream();
BufferedInputStream bis = null;
URL url = null;
byte[] buf = new byte[4096];
int size = 0;
url = new URL(destUrl);
httpUrl = (HttpURLConnection) url.openConnection();
setConn(httpUrl);
httpUrl.connect();
bis = new BufferedInputStream(httpUrl.getInputStream());
while ((size = bis.read(buf)) != -1)
bout.write(buf, 0, size);
bout.flush();
bout.close();
bis.close();
} catch (Exception e) {
return false;
} finally {
httpUrl.disconnect();
}
return flag;
}


public static void main2(String args[]) throws Exception {
File f1 = new File(
"D:/workDir/2009_0619/epaperAd/src/cn/jh/core/util/JHPage.java");
File f2 = new File(
"Y:/wwwroot/ftp/site1/jhsb/html/2009-08/29/content_456074.htm");
fileCopy(f1, f2);
}

public static long fileCopy(File f1, File f2) throws Exception {
long time = new Date().getTime();
f2.getParentFile().mkdirs();
int length = 2097152;
FileInputStream in = new FileInputStream(f1);
FileOutputStream out = new FileOutputStream(f2);
byte[] buffer = new byte[length];
while (true) {
int ins = in.read(buffer);
if (ins == -1) {
in.close();
out.flush();
out.close();
return new Date().getTime() - time;
} else
out.write(buffer, 0, ins);
}
}

public static String getText(String destUrl, String code)
throws IOException {

String str = null;
HttpURLConnection httpUrl = null;

try {
java.io.ByteArrayOutputStream bout = new java.io.ByteArrayOutputStream();
BufferedInputStream bis = null;

URL url = null;
byte[] buf = new byte[4096];
int size = 0;
url = new URL(destUrl);
httpUrl = (HttpURLConnection) url.openConnection();
setConn(httpUrl);
httpUrl.connect();
bis = new BufferedInputStream(httpUrl.getInputStream());
while ((size = bis.read(buf)) != -1)
bout.write(buf, 0, size);
bout.flush();
bout.close();
str = new String(bout.toByteArray(), code);
bis.close();

} catch (Exception e) {

return "";
} finally {
httpUrl.disconnect();
}
return str;
}

static public void wget(String destUrl, String baseStr,
String basePath) throws Exception {
FileOutputStream fos = null;
BufferedInputStream bis = null;
HttpURLConnection httpUrl = null;
URL url = null;
int BUFFER_SIZE = 4096;

byte[] buf = new byte[BUFFER_SIZE];
int size = 0;
try {
System.out.println("baseStr=" + baseStr);
if (destUrl.indexOf(baseStr) < 0)
return;
String file = basePath
+ destUrl.substring(destUrl.indexOf("jinghua.cn")
+ "jinghua.cn".length());
String dir = file.substring(0, file.lastIndexOf("/"));
File f = new File(dir);
f.mkdirs();

url = new URL(destUrl.replace(baseStr,
"http://www.yesky.com/imagelist/"));
httpUrl = (HttpURLConnection) url.openConnection();
httpUrl.connect();
bis = new BufferedInputStream(httpUrl.getInputStream());

fos = new FileOutputStream(file);
while ((size = bis.read(buf)) != -1) {
fos.write(buf, 0, size);
}
fos.flush();
System.out.println("save " + file);
} finally {
try {
fos.close();
bis.close();
httpUrl.disconnect();
} catch (IOException e) {
} catch (NullPointerException e) {
}
}
}

static public void saveToFile(String destUrl, String baseStr,
String basePath) throws Exception {
FileOutputStream fos = null;
BufferedInputStream bis = null;
HttpURLConnection httpUrl = null;
URL url = null;
int BUFFER_SIZE = 4096;

byte[] buf = new byte[BUFFER_SIZE];
int size = 0;
try {
System.out.println("baseStr=" + baseStr);
if (destUrl.indexOf(baseStr) < 0)
return;
String file = basePath
+ destUrl.substring(destUrl.indexOf("jinghua.cn")
+ "jinghua.cn".length());
String dir = file.substring(0, file.lastIndexOf("/"));
File f = new File(dir);
f.mkdirs();

url = new URL(destUrl.replace(baseStr,
"http://himg1.huanqiu.com/attachment/"));
httpUrl = (HttpURLConnection) url.openConnection();
httpUrl.connect();
bis = new BufferedInputStream(httpUrl.getInputStream());

fos = new FileOutputStream(file);
while ((size = bis.read(buf)) != -1) {
fos.write(buf, 0, size);
}
fos.flush();
System.out.println("save " + file);
} finally {
try {
fos.close();
bis.close();
httpUrl.disconnect();
} catch (IOException e) {
} catch (NullPointerException e) {
}
}
}
/**
* 将HTTP资源另存为文件
*
* @param destUrl
* String
* @param fileName
* String
* @throws Exception
*/
static public void wget(String destUrl, String fileName) throws IOException {
FileOutputStream fos = null;
BufferedInputStream bis = null;
HttpURLConnection httpUrl = null;
URL url = null;
int BUFFER_SIZE=4096;
byte[] buf = new byte[BUFFER_SIZE];
int size = 0;
File f=new File(fileName);
f.getParentFile().mkdirs();
// 建立链接
url = new URL(destUrl);
httpUrl = (HttpURLConnection) url.openConnection();
// 连接指定的资源
httpUrl.connect();
// 获取网络输入流
bis = new BufferedInputStream(httpUrl.getInputStream());
// 建立文件
fos = new FileOutputStream(fileName);

// 保存文件
while ((size = bis.read(buf)) != -1)
fos.write(buf, 0, size);

fos.close();
bis.close();
httpUrl.disconnect();
}
static public void click(String destUrl, String path) throws Exception {

BufferedInputStream bis = null;
HttpURLConnection httpUrl = null;
URL url = null;
int BUFFER_SIZE = 4096;

byte[] buf = new byte[BUFFER_SIZE];

try {

System.out.println("wget=" + destUrl);
url = new URL(destUrl);
httpUrl = (HttpURLConnection) url.openConnection();
httpUrl.connect();
bis = new BufferedInputStream(httpUrl.getInputStream());

while ((bis.read(buf)) != -1) {

}

} finally {
try {

bis.close();
httpUrl.disconnect();
} catch (IOException e) {
} catch (NullPointerException e) {
}
}
}
}