爬美股吧 最终修改

先上代码

# -*- coding:utf-8 -*-
import requests
from lxml import etree
import csv

import sys

reload(sys)
sys.setdefaultencoding('utf-8')

start_url = "http://guba.eastmoney.com/list,meigu_1.html"
headers = {
    "User-Agent": "Mozilla / 5.0(Windows NT 6.1;Win64;x64)"
                  "AppleWebKit / 537.36(KHTML, likeGecko)"
                  "Chrome / 58.0.3029.110"
                  "Safari / 537.36"
}


# def get_total_page(start_url):
#    html = requests.get(url=start_url, headers=headers).content
#    selector = etree.HTML(html)
#    sum_page = selector.xpath("//span[@class='sumpage']/text()")
#    return sum_page


def parse_title():
    # sum_page = get_total_page(start_url)
    rows = []
    for num in range(1, 23):
        url = "http://guba.eastmoney.com/list,meigu_" + str(num) + ".html"
        html = requests.get(url=url, headers=headers).content
        selector = etree.HTML(html)
        items = selector.xpath("//div[@id='articlelistnew']/div[position()>1 and position()

结果

之前不知道为何会乱码，这个星期这个作业就一直在我心里纠缠着我，今天冷静的分析了一下原因，终于发现了问题，别提有多开心了。

问题就出在这里
html = requests.get(url=url, headers=headers).text
html = requests.get(url=url, headers=headers).content

看源码

    @property
    def text(self):
        """Content of the response, in unicode.

        If Response.encoding is None, encoding will be guessed using
        ``chardet``.

        The encoding of the response content is determined based solely on HTTP
        headers, following RFC 2616 to the letter. If you can take advantage of
        non-HTTP knowledge to make a better guess at the encoding, you should
        set ``r.encoding`` appropriately before accessing this property.
        """

    #content的完整代码就不贴了。
    @property
    def content(self):
        """Content of the response, in bytes."""

.text返回的是Unicode型的数据。
.content返回的是bytes型也就是二进制的数据
之前一直用.content来解析所以一直出错。
心头只恨总算除掉了！又涨经验了！