��Python������绗�����娆� xpath�寸�������冲���靛奖缃��靛奖璧�婧�

# 涓���瑙ｆ���靛奖url
# 璇锋�15棰������虹��姣�涓��靛奖��������椤�url锛�瑙ｆ���烘��涓��靛奖url
# 浜���xpath瑙ｆ���靛奖璧�婧�
# 瀵圭��涓�棰�瑙ｆ���烘�ョ���靛奖url杩�琛�璇锋�,瑙ｆ���烘��缁����靛奖��绉� �靛奖涓�杞藉�板��
# 涓���瀵逛唬��杩�琛���瑁����芥��

#�翠釜缃�绔���灞�娆★� main_menu => page => item => source
import requests
from lxml import etree
import os
def get_html(url):
    headers=''
    req=requests.get(url)
    # print(type(req))
    ret=req.status_code
    if ret!=200 :
        print('�峰��涓昏����澶辫触锛�杩�����璇�锛�'+str(ret))
        return ('')
    req.encoding='gb2312'
    html=req.text
    return(html)

def get_main_menus(url):
    html=get_html(url)
    # print(html)
    selector=etree.HTML(html)
    main_menus=selector.xpath('//div[@class="contain"]/ul/li[position()<=9]/a')
    main_menus_dict={}
    for main_menu in main_menus :
        text=main_menu.xpath('text()')[0]
        if text == '�藉���靛奖' or text == '���板奖��' or text =='��璇��佃�' or text=='娆х��靛奖' or text=='娆х��佃�' or text=='�ラ�╃�靛奖': #缃�缁�涓��������ц�涓���锛���瑕��ュ�讳���瀹�������瀹�
            pass
        else:
            if len(text)>=1 :
                href = url + main_menu.xpath('@href')[0]
                # print(text,href)
                main_menus_dict.update({text:href})
            else:
                pass
    return(main_menus_dict)

def get_pages(menus):
    url_list=[]
    # main_menus_dict={}

    menu_name=menu[0]
    menu_url=menu[1]
    html_one_menu = get_html(menu_url)
    # print(html_one_menu)
    selector_one_menu=etree.HTML(html_one_menu)
    # // *[ @ id = "header"] / div / div[3] / div[3] / div[2] / div[2] / div[2] / div / select / option[2]
    pages=selector_one_menu.xpath('//select/option[starts-with(@value,"list")]')
    print(menu_name,'灏�璁�'+str(len(pages))+'椤�:')
    if len(pages)>=1 :
        for page in pages :
            page_url=menu_url.split('index.html')[0] + page.xpath('@value')[0]
            # print(page_url)
            url_list.append(page_url)
    return(url_list)
def get_items(menu_name,page_url):
    html_one_page=get_html(page_url)
    selector_one_page = etree.HTML(html_one_page)
    #// *[ @ id = "header"] / div / div[3] / div[3] / div[2] / div[2] / div[2] / ul / table[23] / tbody / tr[2] / td[2] / b / a[2]
    items=selector_one_page.xpath('//a[@class="ulink" and not(contains(@href,"index.html")) ]')
    i=0
    for item in items :
        i = i+1
        if len(item.xpath('text()')) >=1 :
            item_name=item.xpath('text()')[0]
            item_url=url+item.xpath('@href')[0]
            # print(menu_name,item_name)
            get_source(menu_name, item_name, item_url)
    return(i)

def get_source(menu_name,item_name,item_url):
    print(item_name,item_url)
    html_one_item=get_html(item_url)
    selector_one_item = etree.HTML(html_one_item)
    sources=selector_one_item.xpath('//td[@style="WORD-WRAP: break-word"]/a')
    if len(sources)>=1 :
        dirname = 'G:\\�冲���靛奖缃�\\' + menu_name
        if not (os.path.exists(dirname)):
            os.mkdir(dirname)
        filename = dirname + '\\' + item_name.replace('/', '-').replace('?','锛�').replace(':', '锛�').replace('|','_').replace('"', '_') + '.txt'
        fp=open(filename,'w',encoding='utf-8')
        for source in sources:
            print(menu_name,item_name)
            if len(source.xpath('text()'))>=1 :
                source_text=source.xpath('text()')[0]
                source_url=source.xpath('@href')[0]
                print(source_text,source_url)
                fp.write(source_url+'\n')
                fp.write(source_text+'\n')
        fp.close()




if __name__ == '__main__' :
    url = 'http://www.ygdy8.com'
    main_menu_dict=get_main_menus(url)
    # print(main_menu_dict)
    total_page=0
    item_count = 0
    for menu in main_menu_dict.items():
        print('����',menu)
        menu_name = menu[0]
        page_url_list=[] #褰�������涓����ㄩ�ㄧ�靛奖椤甸�㈢����琛�
        page_url_list=get_pages(menu)
        total_page = total_page + len(page_url_list)

        for page_url in page_url_list :
            item_count=item_count+ get_items(menu_name,page_url)
        print('绱�璁�' + str(total_page) + '椤�'+str(item_count)+'��')