python爬虫采集66代理（66IP）并验证可用性

1. 获取代理IP

import re
import requests

proxies = []
base_url = "http://www.66ip.cn/areaindex_35/1.html"
head = {
     
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/81.0.4044.138 Safari/537.36',
    'Cookie': 'Hm_lvt_1761fabf3c988e7f04bec51acd4073f4=1595389409; '
              'Hm_lpvt_1761fabf3c988e7f04bec51acd4073f4=1595397559 '
}
req1 = requests.get(url=base_url, headers=head)
patten = re.findall("(.*?)(.*?)(.*?)(.*?)(.*?)", req1.text)
for index in range(1, len(patten)):
    proxy = patten[index]
    if proxy[0]+':'+proxy[1] not in proxies:
        proxies.append(proxy[0]+':'+proxy[1])
print(proxies)
print("共获取到{}个代理".format(len(proxies)))

此处使用正则表达式对网页表格数据进行提取

2. 验证代理是否可用

import telnetlib

def test_ip(ip, port):
    try:
        telnetlib.Telnet(ip, port, timeout=1)
        print("代理ip:{}:{}有效！".format(ip, port))
    except:
        pass

利用 Telnetlib 模块进行代理可用性测试

3. 汇总代码

import re
import telnetlib

import requests


def test_ip(ip, port):
    try:
        telnetlib.Telnet(ip, port, timeout=1)
        print("代理ip:{}:{}有效！".format(ip, port))
        return True
    except:
        pass
    return False


proxies = []
base_url = "http://www.66ip.cn/areaindex_35/1.html"
head = {
     
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/81.0.4044.138 Safari/537.36',
    'Cookie': 'Hm_lvt_1761fabf3c988e7f04bec51acd4073f4=1595389409; '
              'Hm_lpvt_1761fabf3c988e7f04bec51acd4073f4=1595397559 '
}
req1 = requests.get(url=base_url, headers=head)
patten = re.findall("(.*?)(.*?)(.*?)(.*?)(.*?)", req1.text)
for index in range(1, len(patten)):
    proxy = patten[index]
    if proxy[0] + ':' + proxy[1] not in proxies:
        proxies.append(proxy[0] + ':' + proxy[1])

# 可用代理
usable_proxies = []

for proxy in proxies:
    p = proxy.split(':')
    if test_ip(p[0], p[1]):
        usable_proxies.append(proxy)

print("共获取到 {} 个代理，其中可用代理为：{} 个，详情如下:\n{}".format(len(proxies), len(usable_proxies),usable_proxies))

上述为获取代理并验证可用性的全部代码，请根据需求进行修改。

版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/xu_cxiang/article/details/112221737