android之访问网络获取网页数据并存入数据库
工具：android studio
APK:http://fir.im/ksal
使用URLConnection 连接网络获取网页数据，把联网放在异步线程里执行 使用AsyncTask
extends AsyncTask<String, Integer, StringBuffer> 在 doInBackground 方法里执行联网的方法，
onPostExecute(StringBuffer result) 方法用于在执行完后台任务后更新UI,显示结果
连接网络有get 和 post 方法，
public void doGetURL(String url) throws Exception {
URL localURL = new URL(url);
URLConnection connection = localURL.openConnection();
HttpURLConnection httpURLConnection = (HttpURLConnection)connection;
httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");
httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
if (httpURLConnection.getResponseCode() >= 300) {
throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());
}
inputStream = httpURLConnection.getInputStream();
inputStreamReader = new InputStreamReader(inputStream);
reader = new BufferedReader(inputStreamReader);
while ((tempLine = reader.readLine()) != null) {
tempHTML.append(tempLine);
}
}
private void doPostURL(String url) throws Exception{
URL localURL = new URL(url);
URLConnection connection = localURL.openConnection();
HttpURLConnection httpURLConnection = (HttpURLConnection)connection;
httpURLConnection.setDoOutput(true);
httpURLConnection.setRequestMethod("POST");
httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");
httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
httpURLConnection.setRequestProperty("Content-Length", String.valueOf(url.length()));
outputStream = httpURLConnection.getOutputStream();
outputStreamWriter = new OutputStreamWriter(outputStream);
outputStreamWriter.write(url.toString());
outputStreamWriter.flush();
if (httpURLConnection.getResponseCode() >= 300) {
throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());
}
inputStream = httpURLConnection.getInputStream();
inputStreamReader = new InputStreamReader(inputStream);
reader = new BufferedReader(inputStreamReader);
while ((tempLine = reader.readLine()) != null) {
tempHTML.append(tempLine);
}
}
很多时候我们看的URL 地址都是有重定向的，这个好解决，自己递归或者使用 URLConnection提供的方法，解决重定向问题，获取真实的地址
/**
* 直接将重定向交给HttpURLConnection完成，调用它的setInstanceFollowRedirects(true)方法，
* 这样一来重定向对于外部来说是透明的，我们完全感知不到重定向的发生，
* 但是我们没有办法截获重定向的过程，并且对于重定向次数有限制，如果超过4次的重定向，后续的重定向将会被忽略。
* */
public String redirectGetPath(final String str)
throws MalformedURLException {
URL url = new URL(str);
String realURL = null;
HttpURLConnection conn = null;
try {
conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(30000);
conn.setReadTimeout(30000);
conn.setRequestProperty("Accept-Charset", "utf-8");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setInstanceFollowRedirects(true);
conn.getResponseCode();// trigger server redirect
realURL = conn.getURL().toString();
// Log.d("TAG", str + "\r\n" + "redirect to \r\n" + realURL);
}catch (Exception e) {
e.printStackTrace();
} finally {
if (conn != null) {
conn.disconnect();
}
}
return realURL;
}
/**
* 方法完全自己接管重定向过程，直接调用HttpURLConnection的setInstanceFollowRedirects(false)，
* 传入参数为false，然后递归的方式进行http请求，然后从header参数中取出location字段。
* 看看转了多少次
*
* */
public String recursiveTraceGetPath(String path) {
String realURL = null;
if (mStackDeep.getAndDecrement() > 0) {// 避免异常递归导致StackOverflowError
URL url = null;
HttpURLConnection conn = null;
try {
url = new URL(path);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Charset", "utf-8");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setConnectTimeout(30000);
conn.setReadTimeout(30000);
conn.setInstanceFollowRedirects(false);
int code = conn.getResponseCode();// Network block 301,302
if (needRedirect(code)) {
//临时重定向和永久重定向location的大小写有区分
String location = conn.getHeaderField("Location");
if (location == null) {
location = conn.getHeaderField("location");
}
if (!(location.startsWith("http://") || location
.startsWith("https://"))) {
//某些时候会省略host，只返回后面的path，所以需要补全url
URL origionUrl = new URL(path);
location = origionUrl.getProtocol() + "://"
+ origionUrl.getHost() + location;
}
// Log.d("TAG", "redirect to \r\n" + location);
return recursiveTraceGetPath(location);
} else {
// redirect finish.
realURL = path;
}
} catch (MalformedURLException e) {
Log.w("TAG", "recursiveTracePath MalformedURLException");
} catch (IOException e) {
Log.w("TAG", "recursiveTracePath IOException");
} catch (Exception e) {
Log.w("TAG", "unknow exception");
} finally {
if (conn != null) {
conn.disconnect();
}
}
}
return realURL;
}
private boolean needRedirect(int code) {
return (code == HttpStatus.SC_MOVED_PERMANENTLY
|| code == HttpStatus.SC_MOVED_TEMPORARILY
|| code == HttpStatus.SC_SEE_OTHER || code == HttpStatus.SC_TEMPORARY_REDIRECT);
}
自己写的方法可以自定义重定向的次数，但是需要一个jar文件的支持 import org.apache.http.HttpStatus;
但是如果网页不是重定向，而是使用js 跳转，我写网页也用过这个，用来用户登录后获取首页显示的数据，比如一个简单的
<body>
<span style="white-space:pre"> </span><script> var jump_url="http://active.cliim.cn/AJ8nLm?e1baa1462641130bb5295595b173123b2237fe5268";
window.location.href=jump_url;
// setTimeout(function(){
// window.location.href=jump_url;
// }, 500 );
</script>
</body>那我们就真的需要读取网页数据了。使用StringBuffer tempHTML 存储起来，百度一个html 在线格式化的，格式一下获取的数据，方便阅读
public void doGetURL(String url) throws Exception {
url = redirectGetPath(url);
// url = recursiveTraceGetPath(url);
URL localURL = new URL(url);
URLConnection connection = localURL.openConnection();
HttpURLConnection httpURLConnection = (HttpURLConnection)connection;
httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");
httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
httpURLConnection.setConnectTimeout(30000);
httpURLConnection.setReadTimeout(30000);
if (httpURLConnection.getResponseCode() >= 300) {
throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());
}
String realURL = httpURLConnection.getURL().toString();
Log.d("TAG", url + "\r\n" + "redirect to \r\n" + realURL);
inputStream = httpURLConnection.getInputStream();
inputStreamReader = new InputStreamReader(inputStream);
reader = new BufferedReader(inputStreamReader);
while ((tempLine = reader.readLine()) != null) {
tempHTML.append(tempLine);
}
}
剩下的就是对获取的网页源码的提取了，常用的就是字符串的一些操作，比如字符串截取，字符定位，
<span style="white-space:pre"> </span>tempHTML.substring(tempHTML.indexOf("***"),tempHTML.lastIndexOf("***") + 4);获取相要的结果在listview 里显示
protected void onPostExecute(StringBuffer result) {
if(result != null){
// ArrayList ls = new ArrayList<String>(){ { add("str01"); add("str02"); } };
ArrayList ls = new ArrayList<String>();
ls.add("ClientVersion: android 1.1.1");
ls.add("获取url网页数据");
doHtmlGetRealUrl(result, ls);
for(int i = 0; i<ls.size();i++) {
helper.getReadableDatabase().execSQL("insert into url_table values(null, ?)",
new String[]{ls.get(i).toString()});
}
listview.setAdapter(new ArrayAdapter<String>(context, android.R.layout.simple_list_item_1, ls));
}
super.onPostExecute(result);
}上面的循环里，除了把数据添加在list里，还循环插入了数据库，先新建一个类 UrlInfoDatabaseHelper extends SQLiteOpenHelper
我只是简单存储一下，就只有一个id，一个content
public class UrlInfoDatabaseHelper extends SQLiteOpenHelper {
final String SQL_CREATE_TABLE = "create table url_table (" +
"_id integer primary key autoincrement, " +
"url_content varchar(1000))";
/*
* 构造方法 :
* 参数介绍 :
* 参数① : 上下文对象
* 参数② : 数据库名称
* 参数③ : 数据库版本号
*/
public UrlInfoDatabaseHelper(Context context, String name, int version) {
super(context, name, null, version);
}
@Override
public void onCreate(SQLiteDatabase db) {
db.execSQL(SQL_CREATE_TABLE);
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
System.out.println("call update");
}
}初始化时
//数据库帮助类
private UrlInfoDatabaseHelper helper;
helper = new UrlInfoDatabaseHelper(context, "url", 1);然后怎么从数据库里查出来呢
private void showSQlDate(){
Cursor cursor = helper.getReadableDatabase().rawQuery("select * from url_table", null);
//遍历Cursor
// while(cursor.moveToNext()){
// Log.e("TAG",cursor.getString(1));
// }
inflateListView(cursor);
}
/*
* 刷新数据库列表显示
* 1. 关联SimpleCursorAdapter与数据库表, 获取数据库表中的最新数据
* 2. 将最新的SimpleCursorAdapter设置给ListView
*/
private void inflateListView(Cursor cursor) {
SimpleCursorAdapter cursorAdapter = new SimpleCursorAdapter(
getApplicationContext(),//上下文对象
R.layout.item, //List显示布局
cursor,//数据源
new String[]{"url_content"}, //游标数据的名称，实际是Table列名字
new int[]{R.id.textView});//填充到的布局文件
listView.setAdapter(cursorAdapter);
}这里是把数据库的数据在listview 里显示 ，直接使用 SimpleCursorAdapter
在数据库显示页面，我做了一个新增和删除
新增是显示一个dialog
private void add(){
//步骤2.1：通过LayoutInflater从Android的XML文件中生成View
LayoutInflater inflater = LayoutInflater.from(this);
final View addView = inflater.inflate(R.layout.add_dialgo,null);
//步骤2.2：通过AlertDialog弹出对话框，并且在第一个button，即PositiveButton监听事件，触发操作
new AlertDialog.Builder(this)
.setTitle("添加框")
.setView(addView)
.setPositiveButton("确定", new DialogInterface.OnClickListener() {
//我们希望得到addView中的数据，但是这个inner class，只能获取final的值，所以之前将addView设置为final，也就是所有addView的地址是固定的，而不是动态生成。
public void onClick(DialogInterface dialog, int which) {
EditText nameView = (EditText)addView.findViewById(R.id.show_name);
// addData是下面步骤三，实现SQLite的数据更新和ListView的显示同步add(name,weight);
addData(nameView.getText().toString());
}
})
.setNegativeButton("取消",null)
.show();
}
// 更新数据库和同步ListView，具体如下：
private void addData(String name){
/* 略去数据的判断，例如如果name一样，采用update的方式等等*/
//步骤3.1 在数据库表格中添加数据
helper.getReadableDatabase().execSQL("insert into url_table values(null, ?)",
new String[]{name});
//步骤3.2 同步ListView，更新游标的信息
showSQlDate();
}删除时，是用户长按屏幕上的某条数据，使用ContextMenu，即用户手指长按某个View触发的菜单
private static final int DELETE_ID = 2;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.content_sql);
helper = new UrlInfoDatabaseHelper(this, "url", 1);
listView = (ListView) findViewById(R.id.listViewSql);
/**
* ContextMenu用户手指长按某个View触发的菜单
* 实现场景：用户长按某个List元素，则弹出ContextMenu，选择菜单“Delete”，按下后，弹出AlertDialog，
* 请用户再去确定是否删除，确定后将数据从SQLite中删除，并更新ListView的显示。
* */
//向ListView注册Context Menu，当系统检测到用户长按某单元是，触发Context Menu弹出
registerForContextMenu(listView);
showSQlDate();
findViewById(R.id.btn_sqlAdd).setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View arg0) {
add();
}
});
}
// 步骤2：创建ContextMenu同OptionMenu，用户长按元素后，会弹出菜单
public void onCreateContextMenu(ContextMenu menu, View v, ContextMenu.ContextMenuInfo menuInfo) {
menu.add(Menu.NONE, DELETE_ID , Menu.NONE, "Delete");
super.onCreateContextMenu(menu, v, menuInfo);
}
//步骤 3: ContextMenu的触发操作，例子将触发delete() 还可以做编辑等待
public boolean onContextItemSelected(MenuItem item) {
switch(item.getItemId()){
case DELETE_ID:
/* 在此处，我们关键引入 AdapterView.AdapterContextMenuInfo来获取单元的信息。
在有三个重要的信息。
1、id：The row id of the item for which the context menu is being displayed ，
在cursorAdaptor中，实际就是表格的_id序号；
2、position 是list的元素的顺序；
3、view就可以获得list中点击元素的View， 通过view可以获取里面的显示的信息
*/
AdapterView.AdapterContextMenuInfo info = (AdapterView.AdapterContextMenuInfo)item.getMenuInfo();
delete(info.id);
return true;
default:
break;
}
return super.onContextItemSelected(item);
}
//步骤4: 对触发弹框，和Add的相似，确定后，更新数据库和更新ListView的显示，上次学习已有相类的例子，不再重复。其中getNameById是通过id查名字的方法。值得注意的是，为了内部类中使用，delete的参数采用来final的形式。
private void delete(final long rowId){
if(rowId>0){
new AlertDialog.Builder(this)
.setTitle("确定要删除?")
.setPositiveButton("确定", new DialogInterface.OnClickListener() {
public void onClick(DialogInterface dialog, int which) {
deleteData(rowId);
}
})
.setNegativeButton("取消", null)
.show();
}
}
private void deleteData(long rowId){
String[] str = {String.valueOf(rowId)};
helper.getReadableDatabase().delete("url_table", "_id=?", str);
showSQlDate();
}附件：
源码下载：http://download.csdn.net/detail/i_do_can/9409977