使用jsoup爬取网页信息，保存到txt中

首先建立maven项目，导入相关的jar包

pom

  4.0.0
  com.text.jsoup
  com.text.jsoup
  0.0.1-SNAPSHOT
  
  
  			
			    org.apache.httpcomponents
			    httpclient
			    4.5.2
			
  			
			
			    org.jsoup
			    jsoup
			    1.11.3
			
  			
  
  
//以下是打包所使用的
  
                import_tool
         
            
                org.apache.maven.plugins
                maven-compiler-plugin
                
                    1.8
                    1.8
                    UTF-8
                
            
            
                org.apache.maven.plugins
                maven-dependency-plugin
                
                    
                        copy
                        install
                        
                            copy-dependencies
                        
                        
                            ${project.build.directory}/lib
                        
                    
                
            
            
             
            org.apache.maven.plugins  
            maven-assembly-plugin  
            2.5.5  
              
                  
                      
                        com.text.jsoup.ThreadTest  
                      
                  
                  
                
                    jar-with-dependencies  
                  
              
          
        
        

爬取过程

这个业务是在某一个网站上爬取到某个class的所有的值，然后保存到date.txt 中，这里我使用的是多线程

package com.text.jsoup;


import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class ThreadTest implements Runnable{
	 
	 
    int i = FileUtil.getI();
 
    public static void main(String[] args) {
    	ThreadTest t = new ThreadTest();
       for (int j = 0; j < 100; j++) {
    	   Thread t1 = new Thread(t);
    	   t1.setName("线程"+j);
    	   t1.start();
       }
    }
    
    
	
 
    public void run(){
        while(true){
            synchronized(this){
                notify();
                try{
                    Thread.sleep(10);
                }catch (Exception e){
                    e.printStackTrace();
                }
                if(i <= 30000000){
                    
                    CloseableHttpClient httpClient = HttpClients.createDefault();
					HttpGet httpGet = new HttpGet("http://www.okooo.com/member/"+i);
					CloseableHttpResponse response = null;
					try {
						response = httpClient.execute(httpGet);
					} catch (Exception e1) {
						e1.printStackTrace();
					}//执行get请求
					HttpEntity httpEntity = response.getEntity();
						String string = null;
						try {
							string = EntityUtils.toString(httpEntity, "utf-8");
						} catch (Exception e1) {
							e1.printStackTrace();
						} 
						Document parse = Jsoup.parse(string);
						Elements elementsByTag = parse.getElementsByClass("xxx");
						Element element = null;
						try {
							 element = elementsByTag.get(0);
						} catch (Exception e) {
						}
						String text = element.text();
						String fileName = "/home/date.txt";
						FileUtil.saveTxt(fileName, i+":"+text);
//						System.out.println(i+":"+text);
						try {
							response.close();
							httpClient.close();
						} catch (Exception e1) {
							e1.printStackTrace();
						}
                    
                    i++;
                    try{
                        wait();
                    }catch (InterruptedException e ){
                        e.printStackTrace();
                    }
                }
            }
        }
    }
    
 
}

部分工具类

package com.text.jsoup;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;

public class FileUtil {
/**
读取最后一行的值
*/

	public static String readLastLine(File file, String charset) throws IOException {
        if (!file.exists() || file.isDirectory() || !file.canRead()) {
            return null;
        }
        RandomAccessFile raf = null;
        try {
            raf = new RandomAccessFile(file, "r");
            long len = raf.length();
            if (len == 0L) {
                return "";
            } else {
                long pos = len - 1;
                while (pos > 0) {
                    pos--;
                    raf.seek(pos);
                    if (raf.readByte() == '\n') {
                        break;
                    }
                }
                if (pos == 0) {
                    raf.seek(0);
                }
                byte[] bytes = new byte[(int) (len - pos)];
                raf.read(bytes);
                if (charset == null) {
                    return new String(bytes);
                } else {
                    return new String(bytes, charset);
                }
            }
        } catch (FileNotFoundException e) {
        } finally {
            if (raf != null) {
                try {
                    raf.close();
                } catch (Exception e2) {
                }
            }
        }
        return null;
    }
	
/**
保存文件并换行

*/
	 public static void saveTxt(String fileName, String content) {
			try {
				// 打开一个随机访问文件流，按读写方式
				RandomAccessFile randomFile = new RandomAccessFile(fileName, "rw");
				// 文件长度，字节数
				long fileLength = randomFile.length();
				// 将写文件指针移到文件尾。
				randomFile.seek(fileLength);
				randomFile.write((content + "\r\n").getBytes());
				randomFile.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
	    }
	 
/**
这个是为了如果网络断掉，可以读取到上次保存的最后一行继续往下写入
*/	 

public static int getI(){
		 String readLastLine = null;
		try {
			readLastLine = FileUtil.readLastLine(new File("/home/date.txt"), "utf-8");
		} catch (IOException e) {
			e.printStackTrace();
		}
			if (readLastLine != null) {
				String[] split = readLastLine.split(":");
				int parseInt = Integer.parseInt(split[0]);
//				System.out.println(parseInt);
				return parseInt + 1;
			}
		return 0;
		 
	 }
	 
}

该项目可以直接打包成可执行jar，然后运行即可，效率方面一般，还望路过的大神指点