爬取数据，但是写入数据库时，提示Exception in thread "main" java.lang.NullPointerException，同时发现很多问题。

---

主函数

package com.company.spriderjd;

import com.company.dao.ProductDao;
import com.company.domain.Product;
import com.google.gson.Gson;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.List;
import java.util.Map;

/**
* @Author: Ori
* @Date: 2018/8/21
* @Description: version-01
*/
public class JDSprider01 {

private static  ProductDao productDao =new ProductDao();
public static void main(String[] args) throws Exception {
    /*//1.确定url
    String urlStr="https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8" +
                     "&suggest=1.def.0.V01&wq=sh&pvid=4e17067564564bc398d0ffdaba5be084";
    //2.获取httpClient对象
    CloseableHttpClient httpClient = HttpClients.createDefault();
    //3.指定请求方式
    HttpGet httpGet = new HttpGet(urlStr);
    //4.执行请求，获取响应对象
    CloseableHttpResponse response = httpClient.execute(httpGet);
    //5.获取HTML页面（获取数据）
    String html = EntityUtils.toString(response.getEntity(), "utf-8");

    //获取商品数据
    parsePid(html);*/
   page();
}
//执行分页，共100页
public static void page() throws Exception {
    for (int i = 1; i <100; i++) {
        //定义分页的url
        String pageUrl="https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&page="+(2*i-1);
        //2.获取httpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();
        //3.指定请求方式
        HttpGet httpGet = new HttpGet(pageUrl);
        //4.执行请求，获取响应对象
        CloseableHttpResponse response = httpClient.execute(httpGet);
        //5.获取HTML页面（获取数据）
        String html = EntityUtils.toString(response.getEntity(), "utf-8");

        //获取商品数据
        parsePid(html);
    }
}
//定义方法获取商品的pid
private static void parsePid(String html) throws Exception {
    //1.获取document对象
    Document document = Jsoup.parse(html);
    //2.解析document
    Elements lis = document.select("ul[class=gl-warp clearfix] li[data-pid]");
    for (Element li : lis) {
        String pid = li.attr("data-pid");
       // System.out.println(pid);
        parseProduct(pid);
    }
}
//定义方法根据获得pid，获取商品数据
private static void parseProduct(String pid) throws Exception {
    if(pid!=null){
        //1.获取商品所在的url
        String productUrl="https://item.jd.com/"+pid+".html";
        //2.获取httpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();
        //3.指定请求方式
        HttpGet httpGet = new HttpGet(productUrl);
        //4.执行请求获取请求对象
        CloseableHttpResponse response = httpClient.execute(httpGet);
        //5.获取html对象
        String html = EntityUtils.toString(response.getEntity(), "utf-8");
        //6.受用Jsoup解析html
        Document document = Jsoup.parse(html);
        //7.创建Product类准备封装数据
        Product product = new Product();
        //7.1获取pid
        product.setPid(pid);
        //7.2获取name
        Elements nameEL = document.select(".sku-name");
        String name = nameEL.text();
        product.setName(name);
        //7.3获取brand
        Elements liEl = document.select("#parameter-brand li");
        String brand = liEl.attr("title");
        product.setBrand(brand);
        //7.4获取title
        Elements titleEl = document.select("ul[class=parameter2 p-parameter-list] li:nth-child(1)");
        String title = titleEl.attr("title");
        product.setTitle(title);
        //获取商品价格
        String priceUrl="https://p.3.cn/prices/mgets?pduid=2008552323&skuIds=J_"+pid;
        //3.指定请求方式
         httpGet = new HttpGet(priceUrl);
        //4.执行请求获取请求对象
         response = httpClient.execute(httpGet);
        //5.获取html对象
        String priceJson = EntityUtils.toString(response.getEntity(), "utf-8");
        //System.out.println(priceJson);
        //使用GSON将json转换成字符串
        Gson gson=new Gson();
        List> list = gson.fromJson(priceJson, List.class);
        String price = list.get(0).get("p");
        product.setPrice(price);
        System.out.println(product);
       // System.out.println(product.getPid()+product.getName()+product.getPrice()+product.getBrand()+product.getTitle());
        productDao.addProduct(product);
    }
}

}

执行层

package com.company.dao;

import com.company.domain.Product;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import org.springframework.jdbc.core.JdbcTemplate;

import java.beans.PropertyVetoException;

/**
* @Author: Ori
* @Date: 2018/8/21
* @Description: 使用spring-jdbc完成写入数据库.
*/
public class ProductDao extends JdbcTemplate {
// private JdbcTemplate jdbcTemplate;

/* public ProductDao(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}*/

public ProductDao() {
    ComboPooledDataSource dataSource = new ComboPooledDataSource();
    try {
        dataSource.setDriverClass("com.mysql.jdbc.Driver");
        dataSource.setJdbcUrl("jdbc:mysql://127.0.0.1:3306/jdsprider");
        dataSource.setUser("root");
        dataSource.setPassword("root");
        // JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
        super.setDataSource(dataSource);
    } catch (PropertyVetoException e) {
        e.printStackTrace();
    }
}
public void addProduct(Product product) {
    String sql = "insert into product values(?,?,?,?,?)";
    String[] params = {product.getPid(), product.getName(), product.getPrice(), product.getBrand(), product.getTitle()};
    this.update(sql, params);
}

}

问题总结：

1.为什么提示空指针异常？

在主函数中没有new ProductDao类
private static ProductDao productDao =new ProductDao();

2.在ProductDao 为什么会这样写？

public class ProductDao extends JdbcTemplate {
// private JdbcTemplate jdbcTemplate;

/* public ProductDao(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}*/
public ProductDao() {
ComboPooledDataSource dataSource = new ComboPooledDataSource();
try {
dataSource.setDriverClass(“com.mysql.jdbc.Driver”);
dataSource.setJdbcUrl(“jdbc:mysql://127.0.0.1:3306/jdsprider”);
dataSource.setUser(“root”);
dataSource.setPassword(“root”);
// JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
super.setDataSource(dataSource);
} catch (PropertyVetoException e) {
e.printStackTrace();
}
}

public void addProduct(Product product) {
    String sql = "insert into product values(?,?,?,?,?)";
    String[] params = {product.getPid(), product.getName(), product.getPrice(), product.getBrand(), product.getTitle()};
    this.update(sql, params);
}

}
ProductDao extends JdbcTemplate，后通过super.setDataSource(dataSource);完成JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
同时定义方法addProduct(Product product)进行数据的操作，在该方法中update(sql, params)其实就是this.update(sql, params)；相当于jdbcTemplate.update(sql, params)。