python抓取所有航空公司新闻

#coding=utf-8

import urllib
import time
import re
import threading
import string
import urllib2
import chardet
import MySQLdb
import base64
import os
import sys
from pyquery import PyQuery
reload(sys)
sys.setdefaultencoding("utf-8")

#新闻最早时间
earliestTime = "2010-06-01"
mutex = threading.Lock()

def printf(obj):
    """多线程的输出"""
    global mutex
    mutex.acquire()
    print obj
    mutex.release()
    

def mysqlDataBase_init(database="jipiao"):
    """连接mysql
返回conn,cursor"""
    conn = MySQLdb.connect(host="localhost", user="root", passwd="", db=database, unix_socket="/opt/lampp/var/mysql/mysql.sock")
    cursor = conn.cursor()
    cursor.execute("SET NAMES utf8")
    cursor.execute("SET CHARACTER_SET_CLIENT=utf8")
    cursor.execute("SET CHARACTER_SET_RESULTS=utf8")
    conn.commit()
    return (conn, cursor)       

def countNewsUrl(cursor, newsUrl):
    "count  newsUrl 个数"
    selectSql = "select count(*) from jipiaonews where newsUrl='%s'" % newsUrl
    cursor.execute(selectSql)
    result = cursor.fetchone()
    return result[0]
    pass

def checkNewsUrl(cursor, newsUrl):
    """检查newsUrl
返回是否已经存在"""
    #已经有的url，不需要抓取了
    if countNewsUrl(cursor, newsUrl) > 0:
        return 1
    
    #多线程输出
    global mutex
    mutex.acquire()
    print newsUrl
    mutex.release()
    return 0

def executeSql(cursor, title, postDate, content, airport, source, newsUrl):
    """执行sql语句"""
    #替换不友好字符
    content = content.replace("'", "")
    content = content.replace("\"", "")
    
    insertsql = ("insert into jipiaonews (title,postDate,content,airport,source,newsUrl)" 
            " values('%s','%s','%s','%s','%s','%s')"
            % (title, postDate, content, airport, source, newsUrl))    
    insertsql = insertsql.encode("utf-8")
    cursor.execute(insertsql)    
    pass

def parseDate(postDate):
    """转换日期"""
    return postDate.replace("年", "-").replace("月", "-").replace("日", "")
    pass

def xiamenhangkong():
    """厦门航空"""
    airport = "厦门航空"
    matchPostDate=re.compile(r"[\d]+-[\d]+-[\d]")
    
    conn,cursor=mysqlDataBase_init()
    page=1
    while page!=-1:
        newsPageUrl="http://www.xiamenair.com.cn/newslist/news-%s.shtml"%page
        page+=1
        
        try:
            pq=PyQuery(newsPageUrl)
        except:
            break
        i=0
        while True:
            a= pq("table.artlist a:eq(%s)"%i)
            i+=1
            
            newsUrl=a.attr("href")
            if not newsUrl:
                #第一个新闻都没有，则退出
                if i==1:
                    page=-1
                break
            newsUrl="http://www.xiamenair.com.cn"+newsUrl
            if checkNewsUrl(cursor, newsUrl):
                page=-1
                break
            title=a.text()
            tr=pq("table.artlist tr:eq(%s)"%i)
            postDate = matchPostDate.search(tr.text()).group()
            newpq=PyQuery(newsUrl)
            content=newpq("div.artcon").text()

            if content.find("）")!=-1:
                content=content[content.find("）")+1:]
            content=content.strip() 
            source="" 
            
            executeSql(cursor, title, postDate, content, airport, source, newsUrl)
            pass
        pass
    conn.commit()
    conn.close()
#    newsBasePage = "http://www.xiamenair.com.cn/about_news.aspx?Page=%s"
#    #由于新闻少，所以时间向前推了
#    ownEarliestTime = "2010-01-01"
#    #页码
#    pages = 1
#    while pages != -1:
#        newsBaseUrl = newsBasePage % pages    
#        pages += 1
#        
#        pq = PyQuery(newsBaseUrl)
#        table = pq("div#header table:eq(25)")
#        conn, cursor = mysqlDataBase_init()
#        index = 0
#        while True:
#            a = table.find("a:eq(%s)" % index)
#            index += 1
#
#            newsUrl = a.attr("href")
#            if not newsUrl:
#                break
#            newsUrl = "http://www.xiamenair.com.cn/" + newsUrl    
#            title = a.text()
#            if checkNewsUrl(cursor, newsUrl):
#                pages=-1#结束
#                break
#            newspq = PyQuery(newsUrl)
#            newsTable = newspq("table:eq(3)")    
#            postDate = newsTable("div:eq(1)").html()  
#            #函数终止条件
#            if postDate < ownEarliestTime:
#                pages = -1#结束
#                break  
#            content = newsTable("table:eq(1)").text()
#            source = "" 
#            postDate = postDate.encode("utf8")
#            
#            executeSql(cursor, title, postDate, content, airport, source, newsUrl)           
#            pass
#        conn.commit()
#        conn.close()
#        pass    
#    pass

def chongqinghangkong():
    """重庆航空"""
    airport = "重庆航空"
    
    newsBaseUrl = "http://www.flycq.com/info/news/index.html"
    pq = PyQuery(newsBaseUrl) 
    conn, cursor = mysqlDataBase_init()
    #使用正则表达式，查找日期
    patternDate = re.compile(r"[\d]{6}")
    i = 0
    while i!=-1:
        a = pq("div#newsContent div ul li:eq(%s)" % i).find("a")
        newsUrl = a.attr("href")
        i += 1
        if not newsUrl:
            break
        if newsUrl.startswith("http"):
            continue
        #使用正则表达式，查找日期
        postDate = "20" + patternDate.search(newsUrl).group()
        #需要修改个别的网址
        newsUrl = newsUrl[newsUrl.rfind('/') + 1:]       
        try:         
            newsUrl = "http://www.flycq.com/info/news/" + newsUrl
            if checkNewsUrl(cursor, newsUrl):
                i=-1#结束
                break
            newspq = PyQuery(newsUrl)
        except urllib2.HTTPError:
            printf(newsUrl + "   illegal url!")
            continue            
        content = newspq("div#newsContent").text()
        if content.find("InstanceBeginEditable") != -1:
            content = content[len("InstanceBeginEditable name=\"content\" "):]
        if content.find("InstanceEndEditable") != -1:
            content = content[:-len("InstanceEndEditable")]

        title = a.text()
        source = ""

        executeSql(cursor, title, postDate, content, airport, source, newsUrl)   
        pass   
    conn.commit()
    conn.close()    
    pass

def hannanhangkong():
    """海南航空"""
    airport = "海南航空"
    
    matchUrl = re.compile(r"/.*\.html")
    matchPostDate = re.compile(r"[\d]{4}-[\d]+-[\d]+")
    
    page = 0
    while page != -1:
        if page == 0:
            newsPageUrl = "http://www.hnair.com/gyhh/hhxw/index.html" 
        else:
            newsPageUrl = "http://www.hnair.com/gyhh/hhxw/index_%s.html" % page
        page += 1
        
        conn, cursor = mysqlDataBase_init()
        pq = PyQuery(newsPageUrl)
        i = 0
        while True:
            a = pq("div.body a:eq(%s)" % i)
            newsUrl = a.attr("onclick")
            if not newsUrl:
                break
            newsUrl = matchUrl.search(newsUrl).group()
            newsUrl = "http://www.hnair.com/gyhh/hhxw" + newsUrl
            li = pq("div.body li:eq(%s)" % i)
            i += 1
            
            postDate = matchPostDate.search(li.text()).group()
            if postDate < earliestTime:
                page = -1#结束标记
                break
            if checkNewsUrl(cursor, newsUrl):
                page = -1#结束标记
                break
            title = li.text()[:-(len(postDate) + 2)]                    
            newspq = PyQuery(newsUrl)
            content = newspq("div.body").text()
            if content.find("本报讯") != -1 and content.find("）") != -1:
                content = content[content.find("）") + len("）"):]
            source = ""

            executeSql(cursor, title, postDate, content, airport, source, newsUrl)
            pass
        conn.commit()
        conn.close()
        pass

def xingfuhangkong():
    """幸福航空"""
    airport = "幸福航空"
    newsBasePage = "http://www.joy-air.com/AboutUs/NewsCenter/List.aspx?id=1"
    pq = PyQuery(newsBasePage)
    
    url = pq("div#newslist").find("a").attr("href") 
    id = int(re.search(r"[\d]+", url).group())
    newsBaseUrl = "http://www.joy-air.com/AboutUs/NewsCenter/Detail.aspx?id="
    
    conn, cursor = mysqlDataBase_init()
    while True:
        newsUrl = newsBaseUrl + str(id)
        id -= 1
        
        if checkNewsUrl(cursor, newsUrl):
            break                   
        try:
            newspq = PyQuery(newsUrl)
        except urllib2.HTTPError:
            continue
        postDate = newspq("div.dynamicmessagesbox span:eq(1)").text()
        #函数终止条件
        if postDate < earliestTime:        
            break 
        content = newspq("div.dynamicmessagesbox").text()
        title = newspq("div.dynamicmessagesbox span:eq(0)").text()
        source = ""
            
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)
        pass
    conn.commit()
    conn.close()
    pass

def shenzhenhangkong():
    """深圳航空"""
    airport = "深圳航空"
    newsPageUrl = "http://www.shenzhenair.com/more.jsp?category=423afc988259499faa94bbaf8b6983b5&title=%E6%B7%B1%E8%88%AA%E6%96%B0%E9%97%BB&lan=1" 
    
    sourceFile = urllib.urlopen(newsPageUrl).read()
    pq = PyQuery(sourceFile.decode("gbk"))
    
    conn, cursor = mysqlDataBase_init()
    newspqs = pq("table:eq(17)")
    match = re.compile(r"</script>.*")
    i = 0
    while True:
        a = newspqs("a:eq(%s)" % i)
        i += 1
        
        newsUrl = a.attr("href")
        title = a.text()
        if not newsUrl:
            break
        newsUrl = newsUrl[:newsUrl.rfind("&")]
        newsUrl = "http://www.shenzhenair.com/" + newsUrl   
        title = match.search(title).group()[len(r"</script>"):].strip()
        if checkNewsUrl(cursor, newsUrl):
            break        
        sourceFile = urllib.urlopen(newsUrl).read()
        newspq = PyQuery(sourceFile.decode("gbk"))
        content = newspq("table:eq(2)").text()
        #没有找到时间
        postDate = time.strftime("%Y%m%d")
        source = ""
        begin = content.find("】")
        begin = content.find("】", begin + 1)
        end = content.rfind("【")
        if begin == -1:
            begin = 0
        content = content[begin + 2:end]
  
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)
        if i % 20 == 0:
            time.sleep(5)
        pass
    conn.commit()
    conn.close()
    pass

def chunqiuhangkong():
    """春秋航空
采用的是新闻页的，比首页的新闻还少"""
    airport = "春秋航空"
    newsPageUrl = "http://www.china-sss.com/Static/New_information"

    conn, cursor = mysqlDataBase_init()
    sourceFile = urllib.urlopen(newsPageUrl).read()
    pq = PyQuery(unicode(sourceFile, "utf-8"))
    i = 1
    while True:
        span = pq("div.New_information span:eq(%s)" % i)
        a = pq("div.New_information a:eq(%s)" % i)
        i += 1

        newsUrl = a.attr("href")
        if not newsUrl:
            break 
        title = a.text()
        postDate = span.text()
        newsUrl = "http://www.china-sss.com" + newsUrl
        if checkNewsUrl(cursor, newsUrl):
            break            
        sourceFile = urllib.urlopen(newsUrl).read()
        newspq = PyQuery(unicode(sourceFile, "utf-8"))
        content = newspq("dl#content_top dd").text()
        source = ""
          
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)
        pass
    conn.commit()
    conn.close()
    pass

def dongfanghangkong():
    """东方航空"""
    airport = "东方航空"
    newsPageUrl = "http://www.ce-air.com/mu/main/gydh/xwgg/index.html"
    index = 0
    while index != -1:  
        if index != 0:
            newsPageUrl = ("http://www.ce-air.com/"
                         "mu/main/gydh/xwgg/index_%s.html" % index)
             
        index += 1#下一个页面    
        pq = PyQuery(newsPageUrl)
        conn, cursor = mysqlDataBase_init()
        i = 0
        while True:
            a = pq("div.s_xw_list ul li:eq(%s)" % i).find("a")
            i += 1

            href = a.attr("href")
            if not href:
                break
            title = a.text()
            newsUrl = "http://www.ce-air.com/mu/main/gydh/xwgg/" + href[2:]    
            postDate = re.search(r"[\d]{8}", newsUrl).group()
            if postDate < earliestTime.replace("-", ""):
                index = -1#外层循环结束标记
                break 
            if checkNewsUrl(cursor, newsUrl):
                index = -1#外层循环结束标记
                break                      
            newspq = PyQuery(newsUrl)
            content = newspq("div.body p:eq(1)").text()
            if content == None:
                content = newspq("div.body p:eq(0)").text()
            if len(content) == 0:
                    content = newspq("div.body").text()
                    content = content[content.rfind("}") + 1:]
                
            table = newspq("div.body table").text()           
            source = re.search("'.*'", table).group()[1:-1]
            
            executeSql(cursor, title, postDate, content, airport, source, newsUrl)
            pass
        conn.commit()
        conn.close()
        pass
    pass

def shanghaihangkong():
    """上海航空
没有新闻，只有快讯"""
    airport = "上海航空"
    newsPageUrl = "http://www.shanghai-air.com/cmsweb/sale.aspx?pageIndex=0"
    sourceFile = urllib.urlopen(newsPageUrl).read()
    pq = PyQuery(unicode(sourceFile, "utf-8"))
    
    conn, cursor = mysqlDataBase_init()
    i = 0
    while True:
        a = pq("div.items li:eq(%s)" % i).find("a")
        i += 1

        newsUrl = a.attr("href")
        title = a.attr("title")
        if not newsUrl:
            break
        newsUrl = "http://www.shanghai-air.com/cmsweb/" + newsUrl
        if checkNewsUrl(cursor, newsUrl):
            break            
        sourceFile = urllib.urlopen(newsUrl).read()
        newspq = PyQuery(unicode(sourceFile, "utf-8"))
        content = newspq("div.nowtext").text()
        if not content:
            content = ""
        postDate = time.strftime("%Y%m%d")
        source=""
        
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)
        pass
    conn.commit()
    conn.close()
    pass

    """上海航空旧版的新闻"""
    airport = "上海航空"
    
    newsPageUrl = "http://ww1.shanghai-air.com/news/saldt.asp?ttt="
    matchPostDate = re.compile(r"[\d]+-[\d]+-[\d]+")
    
    conn, cursor = mysqlDataBase_init()
    pq = PyQuery(newsPageUrl)
    i = 0
    while True:
        a = pq("a:eq(%s)" % i)
        i += 1
        newsUrl = a.attr("href")
        if not newsUrl:
            break
        if not newsUrl.find("PublicInfo") != -1:
            continue
        newsUrl = "http://ww1.shanghai-air.com" + newsUrl
        if checkNewsUrl(cursor, newsUrl):
            break
        title = a.text()[1:]
        postDate = parseDate(matchPostDate.search(a.text()).group())
        title = title[:-(len(postDate) + 2)]
        newspq = PyQuery(newsUrl)
        content = newspq("div.texttext").text()
        source = ""
        if not content:
            continue
        
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)
        pass
    conn.commit()
    conn.close()
    pass
def zhongguohangkong():
    """中国航空
有时会出现urlopen error"""
    airport = "中国航空"
    newsPageUrl = "http://ffp.airchina.com.cn/cms/ffp/jszx/xwxx/default.html"
    
    conn, cursor = mysqlDataBase_init()
    count=0
    while count!=10:
        try:    
            pq = PyQuery(newsPageUrl)
        except urllib2.URLError:
            count+=1
            time.sleep(5)
        else:
            break   
    i = 0
    while True:
        newsUrl = pq("a:eq(%s)" % i).attr("href")   
        if not newsUrl:            
            break
        span = pq("span:eq(%s)" % i).text()
        i += 1
        postDate = re.search(r"[\d]{4}-[\d]{2}-[\d]{2}", span).group()
        if postDate < earliestTime:
            break
        title = span[:-14]
        
        newsUrl = "http://ffp.airchina.com.cn" + newsUrl   
        if checkNewsUrl(cursor, newsUrl):
            break   
        count=0   
        while count!=10:
            try:    
                newspq = PyQuery(newsUrl)
            except urllib2.URLError:
                count+=1
                time.sleep(5)
            else:
                break          
        content = newspq("body").text()
        source = ""   
             
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)        
        pass
    conn.commit()
    conn.close()
    pass

def sichuanhangkong():
    """四川航空"""
    airport = "四川航空"
    newsPageUrl = "http://www.scal.com.cn/ScalB2CWeb/News/More_News.aspx?code=NC00391"
    
    pq = PyQuery(urllib.urlopen(newsPageUrl).read())
    ul = pq("ul.news_n")
    
    conn, cursor = mysqlDataBase_init()
    match = re.compile(r"[\d]{4}")
    i = 0
    while True:
        a = pq("ul.news_n a:eq(%s)" % i)
        i += 1
        id = a.attr("onclick")
        if not id:
            break
        
        id = match.search(id).group()
        newsUrl = "http://www.scal.com.cn/Scal.WebMaster/FileUpLoad/htmlpage/%s.html"
        newsUrl = newsUrl % id  
        if checkNewsUrl(cursor, newsUrl):
            break           
        newspq = PyQuery(newsUrl)
        title = newspq("span.NewsTitle").text()
        postDate = newspq("span.NewsInfo_Time").text()
        postDate = parseDate(postDate)[5:]
        source = newspq("span.NewsInfo_Publisher").text()
        content = newspq("div.NewsContent").text()
        
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)
        pass
    conn.commit()
    conn.close()
    pass
    
def tianjinhangkong():
    """天津航空"""
    airport = "天津航空"
    
    #天津航空官网上的的新闻
    url = "http://www.tianjin-air.com/index/ann!doAjaxQuery.action"
    pq = PyQuery(urllib.urlopen(url).read())
    match = re.compile("'.*'")
    conn, cursor = mysqlDataBase_init()
    i = 0
    while True:
        a = pq("a:eq(%s)" % i)
        i += 1
        title = a.attr("title")
        newsUrl = a.attr("href")
        if not newsUrl:
            break
        
        newsUrl = match.search(newsUrl).group()[1:-1]
        newsUrl = "http://www.tianjin-air.com" + newsUrl
        if checkNewsUrl(cursor, newsUrl):
            break             
        content = urllib.urlopen(newsUrl).read()
        begin = content.find("Date:")
        postDate = content[begin:content.find("\n", begin)][6:-6].strip()
        try:
            postDate = time.strftime("%Y-%m-%d", time.strptime(postDate, "%a, %m %b %Y %H:%M:%S"))
        except:
            matchp=re.search(r"[\d]+-[\d]+-[\d]+", postDate)
            if matchp:
                postDate=matchp.group()
            else:
                postDate=time.strftime("%Y%m%d")
        
        content = content[content.find("\n", content.find("X-MimeOLE")):].strip() 
        #若是base64需要解码
        content = base64.decodestring(content)  
        try:
            newspq = PyQuery(content)
        except:
            continue
        content = newspq("body").text()
        if not content:
            continue
        source=""
        
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)
        pass
    conn.commit()
    conn.close()
    pass
def shandonghangkong():
    """山东航空"""
    airport = "山东航空"
    
    headerString = """Accept:*/*
Accept-Charset:GBK,utf-8;q=0.7,*;q=0.3
Accept-Encoding:gzip,deflate,sdch
Accept-Language:zh-CN,zh;q=0.8
Connection:keep-alive
Content-Length:176
Content-Type:application/x-www-form-urlencoded
Cookie:JSESSIONID=0000Dva3t-EvL-J6jQ5uEa8YppU:-1; GUEST_LANGUAGE_ID=zh_CN; COOKIE_SUPPORT=true; __ozlvd671=1302764071; __ozlvd=1302764071
Host:www.shandongair.com
Method:POST /c/portal/render_portlet HTTP/1.1
Origin:http://www.shandongair.com
Referer:http://www.shandongair.com/web/shair_zh/news
User-Agent:Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16"""
    headerString = headerString.split("\n")

    dataString = """p_l_id:89978
p_p_id:journal_news_INSTANCE_1h76
p_p_action:0
p_p_state:normal
p_p_mode:view
p_p_col_id:column-3
p_p_col_pos:2
p_p_col_count:3
currentURL:/web/shair_zh/news"""  
    dataString = dataString.split("\n")

    dataDict = {}
    for line in dataString:
        line = line.split(":")
        dataDict[line[0]] = line[1]
    
    
    url = "http://www.shandongair.com/c/portal/render_portlet"
    request = urllib2.Request(url)
    for line in headerString:
        line = line.strip().split(":")
        request.add_header(line[0], line[1])  
        
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) 
    
    page = 1
    while page != -1:
        dataDict["cur"] = "%s" % page
        data = urllib.urlencode(dataDict)
        page += 1
        
        sourceFile = opener.open(request, data).read()    
        conn, cursor = mysqlDataBase_init()
        pq = PyQuery(unicode(sourceFile, "utf8"))
        i = 0
        while True:
            a = pq("table.taglib-search-iterator a:eq(%s)" % i)
            span = pq("table.taglib-search-iterator span:eq(%s)" % i)
            i += 1
            
            newsUrl = a.attr("href")
            if not newsUrl:
                break
            newsUrl = "http://www.shandongair.com" + newsUrl
            if checkNewsUrl(cursor, newsUrl):
                page = -1#终止条件
                break
            title = a.text()
            postDate = span.text()
            if postDate < earliestTime:
                page = -1#终止条件
                break
            sourceFile = urllib.urlopen(newsUrl).read()
            newspq = PyQuery(unicode(sourceFile, "utf8"))
            content = newspq("td.main").text()
            source = ""
            
            executeSql(cursor, title, postDate, content, airport, source, newsUrl)
            pass
        conn.commit()
        conn.close()        
        pass 


    #山东航空的首页新闻
    newsPageUrl = "http://www.shandongair.com/"
    conn, cursor = mysqlDataBase_init()
    pq = PyQuery(newsPageUrl)
    i = 0
    while True:
        a = pq("div.ind_news_box_center a:eq(%s)" % i)
        i += 1
        newsUrl = a.attr("href")
        if not newsUrl:
            break
        if checkNewsUrl(cursor, newsUrl):
            break 
        sourceFile = urllib.urlopen(newsUrl).read()
        sourceFile = unicode(sourceFile, "utf-8")
        newspq = PyQuery(sourceFile)
        content = newspq("body table:eq(1)").text()[:-6]
        title = newspq("title").text()
        
        m = re.search(r"[\d]{4}年[\d]+月[\d]+日", content)
        if m:
            postDate = m.group()
            postDate = parseDate(postDate)
        else:
            postDate = time.strftime("%Y%m%d")
        source = ""
    
        executeSql(cursor, title, postDate, content, airport, source, newsUrl)
        pass 
    conn.commit()
    conn.close()
    pass

def jixianghangkong():
    """吉祥航空
没有看见新闻，就找了促销信息"""

    airport = "吉祥航空" 
    page = 1
    while page != -1:
        newsPageUrl = "http://www.juneyaoair.com/JY_Airlines_dtlist.jsp?page=%s&menu_no=203" % page
        page += 1
        
        try:
            pq = PyQuery(newsPageUrl) 
        except:
            break
        conn, cursor = mysqlDataBase_init()
        table = pq("table:eq(18)")
        trindex = 1
        while True:
            tr = table("tr:eq(%s)" % trindex)
            trindex += 1
            if not tr:
                break          
            a = tr("td a")
            newsUrl = "http://www.juneyaoair.com/" + a.attr("href")
            
            title = a.text()
            source = tr("td:eq(3)").text()
            postDate = tr("td:eq(4)").text()
            if not postDate:
                break
            if postDate < earliestTime:
                page = -1#终止条件
                break
            if checkNewsUrl(cursor, newsUrl):
                page = -1#终止条件
                break           
            newspq = PyQuery(newsUrl)
            content = newspq("table:eq(6)").text()

            executeSql(cursor, title, postDate, content, airport, source, newsUrl)
            pass
        conn.commit()
        conn.close()
        pass
    pass

def nanfanghangkong():
    """南方航空
还有一定的问题"""
    airport = "南方航空"
    
    year = int(time.strftime("%Y"))
    match = re.compile(r"[\d]{4}-[\d]{2}-[\d]{2}")
    
    conn, cursor = mysqlDataBase_init()
    while str(year) >= earliestTime[:4]:
        newsBasePage = "http://www.csair.com/cn/aboutcsn/04/newsCompany/%s/list_news_%s.asp" % (year, year)
        sourceFile = urllib.urlopen(newsBasePage).read()
        pq = PyQuery(unicode(sourceFile, "utf8"))
        liindex = 0
        while True:
            li = pq("li:eq(%s)" % liindex)
            liindex += 1
            a = li("a")
            newsUrl = a.attr("href")
            if not newsUrl:
                break
            newsUrl = "http://www.csair.com" + newsUrl    
            postDate = match.search(li.text()).group()
            if postDate < earliestTime:
                return
            if checkNewsUrl(cursor, newsUrl):
                return             
            newspq = PyQuery(newsUrl) 
            content = newspq("#content").text()       
            title = content.split()[2]
            source = ""    
            if content.find("InstanceBeginEditable") != -1:
                content = content[len("InstanceBeginEditable name=\"con\" "):]
            if content.find("InstanceEndEditable") != -1:
                content = content[:-len("InstanceEndEditable")]

            executeSql(cursor, title, postDate, content, airport, source, newsUrl)
            pass
        year -= 1
        pass
    conn.commit()
    conn.close()
    pass

def chunqiuhangkongwang():
    """春秋航空网"""
    airport = "春秋航空"
    source = "春秋航空网"
    newsBasePageUrl = "http://info.china-sss.com/article_ps.do?xsl=topic_first_ilist_e&pt=1&ps=30&pi=%s&sf=seq_num&sfdt=number&st=ascending&cid=8069761218&bid=1&qtopic=3&qtit=&qct1=&qct2=#"

    conn, cursor = mysqlDataBase_init()
    sourceMatch = re.compile(r"来源：.*\[")
    page = 1
    while page != -1:
        newsPageUrl = newsBasePageUrl % page       
        page += 1
        
        sourceFile = urllib.urlopen(newsPageUrl).read() 
        pq = PyQuery(unicode(sourceFile, "gbk"))
        i = 0
        while True:
            a = pq("a:eq(%s)" % i)
            tr = pq("td#container1 tr:eq(%s)" % i)
            i += 1
            
            newsUrl = a.attr("href")
            if not newsUrl:
                break
            if len(newsUrl) == 1:
                continue    
            title = a.text()       
            newsUrl = "http://info.china-sss.com" + newsUrl
            postDate = "20" + tr("td:eq(2)").text()
            if postDate < earliestTime:
                page = -1
                break    
            if checkNewsUrl(cursor, newsUrl):
                page = -1
                break  
            sourceFile = urllib.urlopen(newsUrl).read()
            newspq = PyQuery(unicode(sourceFile, "gbk"))
            content = newspq("td.fontmain1").text()
            if content.find("function showContent()") != -1:
                content = content[:content.find("function showContent()")]
            
            sourceInContent = sourceMatch.search(content)
            if sourceInContent:
                source = sourceInContent.group()[len("来源："):-1]
            if source.find("】") != -1:
                source = source[:-len("】")]
            content = content.replace("[/链接]", "")
            if content.find("[链接 地址=") != -1:
                content = content[:-content.find("[链接 地址=")]        
            
            executeSql(cursor, title, postDate, content, airport, source, newsUrl)
            #恢复默认
            source = "春秋航空网"
            pass 
        pass
    conn.commit()
    conn.close()
    pass


def minhangziyuanwang():
    source = "民航资源网"
    matchPostDate = re.compile(r"[\d]+-[\d]+-[\d]+")
    conn, cursor = mysqlDataBase_init()
    
    au = ["天津航空", "吉祥航空", "厦门航空", "四川航空", "中国航空"]

    for airport in au:
        url = "http://news.carnoc.com/search.jsp?key=%s&querytype=0&page=1"    
        url = url % (urllib.quote(airport.decode("utf8").encode("gb2312")))

        sourceFile = urllib.urlopen(url).read()
        pq = PyQuery(sourceFile)
        index = 0
        while True:
            li = pq("div.text li:eq(%s)" % index)
            index += 1
            if not li:
                break
            newsUrl = li("a").attr("href")

            title = li("a").text()
            match = matchPostDate.search(li.text())
            if not match:
                continue
            postDate = match.group()
            if newsUrl.find("2000") != -1:
                continue
            if checkNewsUrl(cursor, newsUrl):
                break
            
            sourceFile = urllib.urlopen(newsUrl).read()
            newspq = PyQuery(sourceFile)
            content = newspq("div#newstext").text()
            
            if not content:
                continue
            if len(content) < 100:
                continue
            executeSql(cursor, title, postDate, content, airport, source, newsUrl)
            pass 
    
    #恢复默认   
    source = ""
    conn.commit()
    conn.close()
    pass

def crawl_news():
    startTime = time.time()
    #最总版本使用的
    import jipiao
    threads = []
    #添加线程
    threads.append(threading.Thread(target=minhangziyuanwang))
    for f in dir(jipiao):
        if f.find("hangkong") != -1:
            exec("threads.append(threading.Thread(target=%s))" % f)
    #启动线程
    for t in threads:
        t.start()
    #等待线程结束
    for t in threads:
        t.join()
        
    printf((time.time() - startTime))    
    pass

if __name__ == "__main__":
    startTime = time.time()
    crawl_news()
    
    #写日志
    if not os.path.exists("logs"):
        os.mkdir("logs")
    logs = file("./logs/logs.txt", "a")
    now = time.strftime("%Y-%m-%d %H:%M:%S")
    logs.write("%s\t耗时:%sms\n" % (now, time.time() - startTime))
    logs.close()
    
    printf(time.time() - startTime)

#    chongqinghangkong()
#    chunqiuhangkong()
#    chunqiuhangkongwang()#春秋航空网
#    dongfanghangkong()
#    hannanhangkong()
#    jixianghangkong()
#    nanfanghangkong()
#    shandonghangkong()#ajax不易处理,使用post
#    shanghaihangkong()
#    shanghaihangkong_old()#上海航空旧版
#    shenzhenhangkong()
#    sichuanhangkong()
#    tianjinhangkong()
#    xiamenhangkong()
#    xingfuhangkong()
#    zhongguohangkong()
#    minhangziyuanwang()

创建数据的SQL

CREATE TABLE IF NOT EXISTS `jipiaonews` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `title` text NOT NULL,
  `postDate` datetime NOT NULL,
  `content` text NOT NULL,
  `airport` varchar(255) NOT NULL,
  `source` varchar(255) NOT NULL,
  `newsUrl` varchar(255) NOT NULL DEFAULT '',
  PRIMARY KEY (`id`)
) ENGINE=MyISAM  DEFAULT CHARSET=utf8;