Python 测试redis中的代理

最近抓取的免费代理很多,却有很多是无效的,于是写了一个测试代理的脚本

import redis
import threading
import requests
import random

Lock=threading.Lock()
Key='proxies'
threads_number=5
count=0
db=redis.StrictRedis(host='127.0.0.1',port=6379,password=None,decode_responses=True)

def request(proxy):
    global count
    try:#对代理进行测试
        proxies={
        'http':'http://'+proxy,
        'https':'https://'+proxy
            }
        list_url=['https://www.baidu.com/','https://www.cnblogs.com/alan-babyblog/p/5335035.html','https://hao.360.cn/','https://www.hao123.com/']
        headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36'}
        requests.get(random.choice(list_url),proxies=proxies,headers=headers)

        db.zadd(Key,100,proxy)
        print(proxy+'可用')
        Lock.acquire() #count代表可用代理数目,用Lock进行保护
        count+=1
        Lock.release()
    except Exception as e: #对于请求失败的代理定义规则进行处理
        score=db.zscore(Key,proxy)
        if score:
            if score>97:  
                db.zincrby(Key,proxy,-1)        
            else: 
                db.zrem(Key,proxy)

class MyThread(threading.Thread):#继承多线程类
    def __init__(self,proxy):
        threading.Thread.__init__(self)
        self.proxy=proxy
    def run(self):
        request(self.proxy)
def main():

    proxy_all=db.zrangebyscore(Key,0,110)
    threads=[]
    while proxy_all:
        for thread in threads:
            if not thread.is_alive():
                threads.remove(thread)
        while len(threads)

以上皆为个人理解,如有错误之处,欢迎留言指正

你可能感兴趣的:(Python 测试redis中的代理)