爬虫寻图记

偶有兴致想找点图片来作视频设计的素材，然而百度图片质量不高，像素太渣，网上找了几个提供妹纸摄影照片的网站，嫌手动太慢，花了1个小时写了两个爬虫脚本，亲测效率还不错，爬了几千张照片，可以慢慢找需要的美图了(@_@)

爬取蜂鸟网

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2018-01-16 18:34:12
# @Author  : bb ([email protected])
# @Word    : python can change world!
# @Version : python3.6

import requests
from bs4 import BeautifulSoup 
import urllib
import time
import os

url="http://bbs.fengniao.com/forum/10344999_1.html"

agents = {'User-Agent':"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0; Baiduspider-ads) Gecko/17.0 Firefox/17.0"} 
def get_url(url,agents):
    res=requests.get(url,headers=agents)
    html=res.text
    soup=BeautifulSoup(html,"html.parser")
    return soup
#print(soup)

def return_html(soup):
    html_list=[]
    for link in soup.find_all('a',attrs={"target":"_blank"}):
        if str(link['href'])[1:6]=='forum' and '_' not in str(link['href']):
            html='http://bbs.fengniao.com'+(link['href'])        
            html_list.append(html)
            #print("获取帖子",html)
    print("----------------------------")
    print("总共获取帖子数------->>>>>>>",len(html_list))
    print("----------------------------")
    time.sleep(3)
    print('倒计时......'+'\n'*3)
    return html_list

def guolv(html_list):
    new_list=[]
    nab=set()
    for i in html_list:    
        if str(str(i).split('_')[0].split('/')[-1]) not in nab:
            new_list.append(i)
            nab.add(str(i).split('_')[0].split('/')[-1])
    print('过滤完成！')
    for i in new_list:
        print("获取帖子",i)
    return new_list

    

def img_save(soup):
    for link in soup.find_all('img'):
        tj=str(link['src'])[-2]
        if tj =='1':
            img_src=link['src']
            print('获取图片地址',link['src'])
            nn=img_src.split('.jpg')
            nn=str(nn[0]).split('/')[-1]
            filename=nn+'.jpg'
            img = requests.get(img_src)   
            with open(filename,'ab') as f: #存储图片，多媒体文件需要参数b（二进制文件）  
                f.write(img.content) #多媒体存储content  
def main():
    page=1 ##设置起始页
    while page<2:  ##设置最大页数
        print("这是第%s页!"%page)   
        url='http://bbs.fengniao.com/forum/forum_101_'+str(page)+'_execpost.html' ##精选地址
        soup=get_url(url,agents)
        html_list=return_html(soup) ##帖子地址
        #new_list=guolv(html_list)   
        for i in html_list:
            soup=get_url(i,agents)
            img_save(soup)
        with open('output.txt','a') as f:
            print("第%s页完成!"%page,file=f)
        print("第%s页完成!"%page)
        print('\n'*5)
        page=page+1
        time==time.strftime("%Y-%m-%d %H:%M:%S",time.localtime()[0:9])
        # ostime='2018-01-17 17:00:00'  
        # if time < ostime:
        #     os.system("shutdown -h now")  ##定时关机

def test():
    url='http://bbs.fengniao.com/forum/forum_101_2_execpost.html'
    soup=get_url(url,agents)
    html_list=return_html(soup)
    print(html_list)
if __name__ == '__main__':
    main()

爬取煎蛋网

import threading
import time
import requests
import urllib.request
import re
from queue import Queue
import random
from bs4 import BeautifulSoup
agents = [  
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0; Baiduspider-ads) Gecko/17.0 Firefox/17.0",  
    "Mozilla/5.0 (Linux; U; Android 2.3.6; en-us; Nexus S Build/GRK39F) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",  
    "Avant Browser/1.2.789rel1 (http://www.avantbrowser.com)",  
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5"]


class jandanspridy(threading.Thread):
    
    def __init__(self,queue):
        threading.Thread.__init__(self)
        self._queue=queue
    
    def run(self):
        while not self._queue.empty():
            each=self._queue.get_nowait()
            filename = each.split('/')[-1]
            req=urllib.request.urlopen(each)
            img=req.read()
            with open(filename, 'wb') as f:
                f.write(img)

def getProxyIp():
 proxy = []
 header = {'User-Agent':random.choice(agents)}
 for i in range(1,3):
    
    url = 'http://www.xicidaili.com/nn/'+str(i)
    req = requests.get(url,headers=header)
    res = req.text
    soup = BeautifulSoup(res,"html.parser")
    ips = soup.findAll('tr')
    for x in range(1,len(ips)):
        ip = ips[x]
        tds = ip.findAll("td")
        ip_temp = tds[1].contents[0]+":"+tds[2].contents[0]
        proxy.append(ip_temp)
 return proxy

def get_proxies():
    proxy=getProxyIp()
    a=1
    print('正在核对ip代理。。。。。。')
    while a>0:
        try:
            ipchoice=random.choice(proxy)
            proxies={'http':ipchoice}
            res=requests.get("https://www.baidu.com/",proxies=proxies,timeout=2)
            if res.status_code==200:
                
                print(proxies,'is up')
                return proxies
                break
            else:
                print(ipchoice+'连接不上哦')
                proxy.remove(ipchoice)
        except :
            #print('有错误，让我重启下')
            pass
            a=1

def get_page():
    url="http://jandan.net/ooxx"
    req = urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0')
    response = urllib.request.urlopen(url)
    html= response.read().decode('utf-8')
    page_list=re.findall(r'\[(.*?)\]',html)

    page=page_list[0]
    return page


def eachpageurl():
    global queue
    #proxies=get_proxies()
    #print(proxies,'可以使用')
    page=int(get_page())
    f=open('output1.txt','w+')
    html_list=[]
    queue=Queue()
    
    while page>200:

        print("这是第%s页哦！"% page)
        page_list=str("这是第%s页哦！"% page)
        f.write(page_list+'\n')
        url='http://jandan.net/ooxx/page-'+str(page)+'#comments'
        page=page-1
        req=requests.get(url) ##如果代理不稳定可以去掉代理参数proxies再运行
        try:
            html=req.text
            html2=re.findall('