1.在爬虫中,频繁的访问目标服务器,可能会使服务器奔溃.网站为了防止非正常访问,造成服务器奔溃,一般会检测用户ip,如果某个ip频繁的访问目标网址,可能会被对象进行封ip.所以我们需要用到代理ip来隐藏自己,从而躲过对方的检测.
2.我在一些收费的代理ip网站中爬取了一些免费使用的高匿代理ip,并封装成模块方便使用
3.某收费代理ip网站的爬虫代码1(建议不要用从某个代理网站爬取下来的代理ip,取爬它本网站的东西,分分钟被检查出来被封ip,我以实践.)
import requests
from lxml import etree
from user_proxy.user_agent import user
from user_proxy.proxys import proxys
def spider_proxy(url):
# 设置用户代理, 隐藏爬虫身份
user_agent = user()
headers = {
'User-Agent': user_agent
}
# 设置代理ip
proxies = proxys()
# 发送请求,并返回响应
response = requests.get(url=url, headers=headers, proxies=proxies, verify=False)
# 接收页面数据
html = response.text
# 穿件xpath对象
tree = etree.HTML(html)
# 获取全部免费proxy的li
proxy_li = tree.xpath('//*[@id="list"]/table/tbody/tr')
# 保存为HTTPS
with open('proxy.txt', 'a', encoding='utf-8') as fp:
for proxy in proxy_li:
# 获取代理ip
proxy_ip = proxy.xpath('./td[@data-title="IP"]/text()')[0]
# 获取代理端口
proxy_port = proxy.xpath('./td[@data-title="PORT"]/text()')[0]
string = '{"HTTPS": "%s:%s"},' % ( proxy_ip, proxy_port)
fp.write(string + '\n')
fp.flush()
print(string)
# 保存为HTTP
# with open('porxy.txt', 'a', encoding='utf-8') as fp:
# for proxy in proxy_li:
# # 获取代理ip
# proxy_ip = proxy.xpath('./td[@data-title="IP"]/text()')[0]
# # 获取代理端口
# proxy_port = proxy.xpath('./td[@data-title="PORT"]/text()')[0]
# string = '{"HTTP": "%s:%s"},' % ( proxy_ip, proxy_port)
# fp.write(string + '\n')
# fp.flush()
if __name__ == '__main__':
for num in range(1, 2933):
url = 'https://www.kuaidaili.com/free/inha/{}/'.format(num)
spider_proxy(url)
4.某收费代理ip网站的爬虫代码2
import requests
from lxml import etree
from user_proxy.user_agent import user
from user_proxy.proxys import proxys
def spider_proxy(url):
# 设置用户代理, 隐藏爬虫身份
user_agent = user()
headers = {
'User-Agent': user_agent
}
# 设置代理ip
proxies = proxys()
# 发送请求,并返回响应
response = requests.get(url=url, headers=headers, proxies=proxies, verify=False)
# 接收页面数据
html = response.text
# 穿件xpath对象
tree = etree.HTML(html)
# 获取全部免费proxy的li
proxy_li = tree.xpath('//div[3]/div[1]/div/div[1]/table/tbody/tr')
# 保存为HTTPS
with open('98proxy.txt', 'a', encoding='utf-8') as fp:
for proxy in proxy_li:
# 获取代理ip
proxy_ip = proxy.xpath('./td/text()')[0]
proxy_ip = proxy_ip.strip()
# 获取代理端口
proxy_port = proxy.xpath('./td[2]/text()')[0]
proxy_port = proxy_port.strip()
string = '{"HTTPS": "%s:%s"},' % ( proxy_ip, proxy_port)
fp.write(string + '\n')
fp.flush()
print(string)
if __name__ == '__main__':
for num in range(1, 21):
url = 'http://www.89ip.cn/index_{}.html'.format(num)
spider_proxy(url)
- 将爬取下来的代理ip,处理成模块方便使用
"""
1.自定义 ip代理模块
2.使用:
2.1, 导入模块
2.2, 创建user_agent对象:proxies = proxys()
"""
import random
def proxys():
proxy_list = [
{"HTTPS": "114.113.222.154:80"},
{"HTTPS": "117.90.252.164:9999"},
{"HTTPS": "60.13.42.182:9999"},
{"HTTPS": "180.119.68.6:9999"},
{"HTTPS": "182.34.27.0:9999"},
{"HTTPS": "140.143.48.49:1080"},
{"HTTPS": "123.207.43.128:1080"},
{"HTTPS": "58.253.158.194:9999"},
{"HTTPS": "163.204.247.107:9999"},
{"HTTPS": "183.129.207.89:28144"},
{"HTTPS": "183.129.207.80:12889"},
{"HTTPS": "121.225.67.109:3128"},
{"HTTPS": "222.189.191.64:9999"},
{"HTTPS": "59.62.26.93:9000"},
{"HTTPS": "58.253.152.43:9999"},
{"HTTPS": "113.124.95.138:9999"},
{"HTTPS": "220.143.107.178:3128"},
{"HTTPS": "117.90.6.32:9000"},
{"HTTPS": "180.119.68.167:9999"},
{"HTTPS": "218.95.50.228:9000"},
{"HTTPS": "58.253.158.57:9999"},
{"HTTPS": "117.90.137.150:9000"},
{"HTTPS": "139.196.208.220:80"},
{"HTTPS": "119.101.118.26:9999"},
{"HTTPS": "121.204.150.159:8118"},
{"HTTPS": "180.119.141.104:9999"},
{"HTTPS": "125.123.120.233:9000"},
{"HTTPS": "49.51.70.42:1080"},
{"HTTPS": "117.191.11.105:80"},
{"HTTPS": "117.90.252.244:9000"},
{"HTTPS": "183.129.207.89:27985"},
{"HTTPS": "118.122.114.238:9000"},
{"HTTPS": "222.189.144.239:9999"},
{"HTTPS": "47.93.28.220:8118"},
{"HTTPS": "112.27.167.74:33641"},
{"HTTPS": "183.129.207.89:27794"},
{"HTTPS": "60.182.162.250:61234"},
{"HTTPS": "117.191.11.111:80"},
{"HTTPS": "111.230.254.195:8118"},
{"HTTPS": "27.43.189.148:9999"},
{"HTTPS": "47.107.160.99:8118"},
{"HTTPS": "49.86.183.99:9999"},
{"HTTPS": "1.197.203.27:9999"},
{"HTTPS": "49.86.178.78:9999"},
{"HTTPS": "27.43.191.118:9999"},
{"HTTPS": "125.115.180.190:3128"},
{"HTTPS": "122.136.212.132:53281"},
{"HTTPS": "113.65.5.149:8118"},
{"HTTPS": "175.42.158.140:9999"},
{"HTTPS": "113.247.252.114:9090"},
{"HTTPS": "42.238.90.109:9999"},
{"HTTPS": "36.255.86.232:82"},
{"HTTPS": "49.70.64.182:9999"},
{"HTTPS": "58.22.222.170:9000"},
{"HTTPS": "180.160.62.196:8118"},
{"HTTPS": "117.191.11.103:80"},
{"HTTPS": "36.248.133.126:9999"},
{"HTTPS": "222.128.9.235:33428"},
{"HTTPS": "42.238.85.40:9999"},
{"HTTPS": "183.62.37.164:8080"},
{"HTTPS": "114.113.220.187:80"},
{"HTTPS": "101.231.104.82:80"},
{"HTTPS": "118.24.89.122:1080"},
{"HTTPS": "119.145.2.98:44129"},
{"HTTPS": "47.94.221.41:8118"},
{"HTTPS": "183.129.207.89:11056"},
{"HTTPS": "222.162.239.92:80"},
{"HTTPS": "36.248.132.82:9999"},
{"HTTPS": "182.34.33.154:9999"},
{"HTTPS": "182.107.13.45:9000"},
{"HTTPS": "210.26.49.88:3128"},
{"HTTPS": "60.13.42.8:9999"},
{"HTTPS": "117.191.11.110:80"},
{"HTTPS": "140.143.152.93:1080"},
{"HTTPS": "111.230.167.207:8118"},
{"HTTPS": "163.125.233.93:8118"},
{"HTTPS": "117.127.16.206:8080"},
{"HTTPS": "115.29.214.175:8118"},
{"HTTPS": "163.204.244.171:9999"},
{"HTTPS": "123.206.200.63:1080"},
{"HTTPS": "121.232.148.37:9000"},
{"HTTPS": "119.3.253.33:8080"},
{"HTTPS": "125.123.142.228:9999"},
{"HTTPS": "125.110.114.212:8118"},
{"HTTPS": "119.28.203.242:8000"},
{"HTTPS": "222.189.144.88:9999"},
{"HTTPS": "121.79.131.58:8080"},
{"HTTPS": "114.113.222.202:80"},
{"HTTPS": "222.189.191.170:9999"},
{"HTTPS": "39.137.168.230:80"},
{"HTTPS": "36.69.136.130:8080"},
{"HTTPS": "118.190.95.43:9001"},
{"HTTPS": "103.59.213.29:8080"},
{"HTTPS": "124.160.56.76:31484"},
{"HTTPS": "125.110.82.142:9000"},
{"HTTPS": "183.158.207.111:9000"},
{"HTTPS": "125.123.141.70:9999"},
{"HTTPS": "183.146.213.56:80"},
{"HTTPS": "60.191.57.78:14051"},
{"HTTPS": "112.85.164.215:9999"},
{"HTTPS": "163.204.246.116:9999"},
{"HTTPS": "123.206.30.254:8118"},
{"HTTPS": "27.43.185.96:9999"},
{"HTTPS": "222.89.32.187:9999"},
{"HTTPS": "59.120.231.207:3128"},
{"HTTPS": "36.248.132.57:9999"},
{"HTTPS": "113.105.201.183:3128"},
{"HTTPS": "61.164.39.68:53281"},
{"HTTPS": "218.22.7.62:53281"},
{"HTTPS": "42.238.91.164:9999"},
{"HTTPS": "101.132.66.66:80"},
{"HTTPS": "222.189.190.188:9999"},
{"HTTPS": "183.63.114.204:54321"},
{"HTTPS": "124.16.71.141:1080"},
{"HTTPS": "121.233.207.124:9999"},
{"HTTPS": "45.115.174.29:30086"},
{"HTTPS": "222.189.190.30:9999"},
{"HTTPS": "218.91.112.168:9999"},
{"HTTPS": "114.113.222.136:80"},
{"HTTPS": "118.190.145.138:9001"},
{"HTTPS": "117.90.1.248:9000"},
{"HTTPS": "39.106.127.234:80"},
{"HTTPS": "222.189.191.121:9999"},
{"HTTPS": "182.34.32.252:9999"},
{"HTTPS": "120.77.45.130:80"},
{"HTTPS": "222.189.144.228:9999"},
{"HTTPS": "182.34.37.125:9999"},
{"HTTPS": "113.140.1.82:53281"},
{"HTTPS": "42.238.81.147:9999"},
{"HTTPS": "221.2.175.238:8060"},
{"HTTPS": "27.43.185.45:9999"},
{"HTTPS": "114.113.220.159:80"},
{"HTTPS": "180.119.141.20:9999"},
{"HTTPS": "58.22.207.137:9000"},
{"HTTPS": "140.143.142.14:1080"},
{"HTTPS": "139.199.38.177:8118"},
{"HTTPS": "222.89.32.139:9999"},
{"HTTPS": "222.135.92.68:38094"},
{"HTTPS": "117.191.11.102:80"},
{"HTTPS": "49.51.155.45:8081"},
{"HTTPS": "182.34.33.216:9999"},
{"HTTPS": "140.143.246.163:8080"},
{"HTTPS": "222.189.191.134:9999"},
{"HTTPS": "124.237.83.14:53281"},
{"HTTPS": "119.255.248.54:808"},
{"HTTPS": "182.34.34.17:9999"},
{"HTTPS": "220.133.57.116:49242"},
{"HTTPS": "61.164.39.66:53281"},
{"HTTPS": "125.123.136.226:9999"},
{"HTTPS": "58.219.126.221:9999"},
{"HTTPS": "47.93.36.195:8118"},
{"HTTPS": "163.125.221.24:8118"},
{"HTTPS": "59.62.26.172:9000"},
{"HTTPS": "122.234.204.214:9000"},
{"HTTPS": "153.35.213.79:8118"},
{"HTTPS": "121.15.254.156:888"},
{"HTTPS": "60.13.42.31:9999"},
{"HTTPS": "210.5.10.87:53281"},
{"HTTPS": "117.91.232.55:9999"},
{"HTTPS": "175.18.13.45:8080"},
{"HTTPS": "222.189.190.33:9999"},
{"HTTPS": "113.110.225.11:3128"},
{"HTTPS": "14.204.211.133:8060"},
{"HTTPS": "27.43.185.62:9999"},
{"HTTPS": "163.204.246.231:9999"},
{"HTTPS": "36.255.86.231:83"},
{"HTTPS": "116.113.27.170:61852"},
{"HTTPS": "180.119.68.128:9999"},
{"HTTPS": "113.109.28.105:53281"},
{"HTTPS": "49.86.180.22:9999"},
{"HTTPS": "49.89.143.213:9999"},
{"HTTPS": "122.243.13.249:9000"},
{"HTTPS": "183.209.23.154:8118"},
{"HTTPS": "118.24.157.49:8118"},
{"HTTPS": "122.4.45.101:9999"},
{"HTTPS": "120.27.208.131:80"},
{"HTTPS": "180.118.247.183:9999"},
{"HTTPS": "180.119.141.237:9999"},
{"HTTPS": "121.232.194.14:9000"},
{"HTTPS": "180.119.141.186:9999"},
{"HTTPS": "221.176.206.29:8060"},
{"HTTPS": "119.101.112.219:9999"},
{"HTTPS": "222.89.32.188:9999"},
{"HTTPS": "222.223.182.66:8000"},
{"HTTPS": "175.44.109.52:9999"},
{"HTTPS": "113.121.40.95:9999"},
{"HTTPS": "218.91.112.200:9999"},
{"HTTPS": "180.107.55.119:8118"},
{"HTTPS": "117.90.5.109:9000"},
{"HTTPS": "120.234.138.102:53779"},
{"HTTPS": "36.255.87.227:82"},
{"HTTPS": "220.249.149.147:9999"},
{"HTTPS": "114.113.222.149:80"},
{"HTTPS": "116.30.122.155:9000"},
{"HTTPS": "115.216.36.133:9000"},
{"HTTPS": "47.94.89.87:3128"},
{"HTTPS": "1.175.133.55:3128"},
{"HTTPS": "114.119.116.92:61066"},
{"HTTPS": "223.241.78.26:8010"},
{"HTTPS": "123.163.96.145:9999"},
{"HTTPS": "182.38.189.17:8118"},
{"HTTPS": "183.129.207.89:28057"},
{"HTTPS": "59.62.6.9:9000"},
{"HTTPS": "211.159.171.58:80"},
{"HTTPS": "140.143.137.69:1080"},
{"HTTPS": "222.189.191.51:9999"},
{"HTTPS": "182.34.33.95:9999"},
{"HTTPS": "120.83.101.213:9999"},
{"HTTPS": "140.143.246.163:9999"},
{"HTTPS": "219.85.252.143:80"},
{"HTTPS": "49.86.178.37:9999"},
{"HTTPS": "218.91.112.219:9999"},
{"HTTPS": "27.43.186.234:9999"},
{"HTTPS": "163.204.245.234:9999"},
{"HTTPS": "183.129.207.93:14153"},
{"HTTPS": "58.54.227.111:61234"},
{"HTTPS": "119.145.2.99:44129"},
{"HTTPS": "111.75.193.24:48449"},
{"HTTPS": "211.141.111.114:61395"},
{"HTTPS": "136.228.128.232:8080"},
{"HTTPS": "124.193.37.5:8888"},
{"HTTPS": "163.204.244.210:9999"},
{"HTTPS": "222.189.191.242:9999"},
{"HTTPS": "180.119.68.71:9999"},
{"HTTPS": "36.99.16.241:82"},
{"HTTPS": "60.13.42.63:9999"},
{"HTTPS": "42.238.88.113:9999"},
{"HTTPS": "222.189.191.204:9999"},
{"HTTPS": "115.171.85.202:9000"},
{"HTTPS": "222.189.144.92:9999"},
{"HTTPS": "120.83.104.46:9999"},
{"HTTPS": "222.189.190.80:9999"},
{"HTTPS": "59.172.27.6:38380"},
{"HTTPS": "114.113.222.211:80"},
{"HTTPS": "118.89.44.224:8118"},
{"HTTPS": "42.228.3.158:8080"},
{"HTTPS": "27.43.190.124:9999"},
{"HTTPS": "47.101.156.232:8080"},
{"HTTPS": "222.189.190.254:9999"},
{"HTTPS": "180.122.145.43:8888"},
{"HTTPS": "175.44.109.192:9999"},
{"HTTPS": "1.192.242.22:9999"},
{"HTTPS": "1.192.244.225:9999"},
{"HTTPS": "221.227.74.141:9999"},
{"HTTPS": "103.235.199.38:56975"},
{"HTTPS": "115.61.60.228:9999"},
{"HTTPS": "125.110.120.214:9000"},
{"HTTPS": "144.123.69.201:9999"},
{"HTTPS": "49.86.179.133:9999"},
{"HTTPS": "163.204.246.133:9999"},
{"HTTPS": "42.238.90.196:9999"},
{"HTTPS": "60.217.73.238:8060"},
{"HTTPS": "175.6.5.234:8888"},
{"HTTPS": "124.205.155.155:9090"},
{"HTTPS": "101.132.164.113:8118"},
{"HTTPS": "45.125.32.181:3128"},
{"HTTPS": "58.87.68.189:1080"},
{"HTTPS": "58.253.158.182:9999"},
{"HTTPS": "180.119.141.247:9999"},
{"HTTPS": "36.255.87.252:83"},
{"HTTPS": "183.158.201.100:9000"},
{"HTTPS": "163.204.247.75:9999"},
{"HTTPS": "47.106.205.193:8888"},
{"HTTPS": "42.238.82.46:9999"},
{"HTTPS": "221.239.86.26:32228"},
{"HTTPS": "180.119.141.140:9999"},
{"HTTPS": "59.32.37.166:61234"},
{"HTTPS": "221.122.91.65:80"},
{"HTTPS": "27.208.91.251:8060"},
{"HTTPS": "171.11.33.209:9999"},
{"HTTPS": "61.183.233.6:54896"},
{"HTTPS": "219.159.38.200:56210"},
{"HTTPS": "121.238.76.53:8118"},
{"HTTPS": "222.189.191.138:9999"},
{"HTTPS": "42.238.88.59:9999"},
{"HTTPS": "113.108.242.36:47713"},
{"HTTPS": "123.57.84.116:8118"},
{"HTTPS": "183.147.29.213:9000"},
{"HTTPS": "119.145.2.100:44129"},
{"HTTPS": "1.199.30.227:9999"},
{"HTTPS": "115.196.48.108:9000"},
{"HTTPS": "120.25.203.182:7777"},
{"HTTPS": "120.79.184.148:8118"},
{"HTTPS": "120.79.62.89:3128"},
{"HTTPS": "61.130.235.241:8010"},
{"HTTPS": "114.239.144.227:9999"},
{"HTTPS": "47.107.160.233:808"},
{"HTTPS": "116.196.81.58:3128"},
{"HTTPS": "180.118.86.10:9000"},
{"HTTPS": "45.115.175.137:30944"},
{"HTTPS": "27.43.190.157:9999"},
{"HTTPS": "144.123.69.78:9999"},
{"HTTPS": "182.34.34.139:9999"},
{"HTTPS": "218.91.112.235:9999"},
{"HTTPS": "182.34.37.145:9999"},
{"HTTPS": "49.86.182.6:9999"},
{"HTTPS": "42.238.81.160:9999"},
{"HTTPS": "1.175.133.9:3128"},
{"HTTPS": "111.177.161.65:9999"},
{"HTTPS": "36.248.129.213:9999"},
{"HTTPS": "58.219.126.189:9999"},
{"HTTPS": "45.115.176.176:8080"},
{"HTTPS": "27.40.146.233:9999"},
{"HTTPS": "180.119.141.61:9999"},
{"HTTPS": "58.253.159.121:9999"},
{"HTTPS": "59.48.237.6:8060"},
{"HTTPS": "121.69.37.6:9797"},
{"HTTPS": "180.107.248.4:9999"},
{"HTTPS": "106.14.11.65:8118"},
{"HTTPS": "182.34.21.118:9999"},
{"HTTPS": "120.194.109.179:47812"},
{"HTTPS": "117.141.215.3:1080"},
{"HTTPS": "163.204.243.136:9999"},
{"HTTPS": "115.53.34.200:9999"},
{"HTTPS": "117.127.16.207:8080"},
{"HTTPS": "118.24.246.249:80"},
{"HTTPS": "182.92.113.183:8118"},
{"HTTPS": "27.189.128.205:53281"},
{"HTTPS": "60.205.57.4:3128"},
{"HTTPS": "27.43.184.231:9999"},
{"HTTPS": "125.123.69.45:9000"},
{"HTTPS": "49.86.183.69:9999"},
{"HTTPS": "222.189.144.119:9999"},
{"HTTPS": "218.73.134.50:9999"},
{"HTTPS": "221.122.91.60:80"},
{"HTTPS": "220.168.52.245:50657"},
{"HTTPS": "125.40.109.154:61875"},
{"HTTPS": "45.221.77.246:30496"},
{"HTTPS": "58.253.152.57:9999"},
{"HTTPS": "36.248.132.184:9999"},
{"HTTPS": "193.112.128.212:8118"},
{"HTTPS": "49.86.182.228:9999"},
{"HTTPS": "120.234.138.99:53779"},
{"HTTPS": "124.205.143.212:39904"},
{"HTTPS": "121.17.174.121:9999"},
{"HTTPS": "42.238.89.232:9999"},
{"HTTPS": "125.123.121.247:9000"},
{"HTTPS": "59.52.158.127:8060"},
{"HTTPS": "58.211.183.46:40225"},
{"HTTPS": "218.91.50.234:1080"},
{"HTTPS": "42.238.84.235:9999"},
{"HTTPS": "121.232.148.166:9000"},
{"HTTPS": "210.22.176.146:47578"},
{"HTTPS": "47.106.244.250:8118"},
{"HTTPS": "123.206.204.51:8118"},
{"HTTPS": "113.124.84.217:9999"},
{"HTTPS": "47.103.197.197:8080"},
{"HTTPS": "117.191.11.109:80"},
{"HTTPS": "58.254.220.116:53579"},
{"HTTPS": "182.34.35.127:9999"},
{"HTTPS": "117.90.137.30:9000"},
{"HTTPS": "163.125.114.63:8118"},
{"HTTPS": "124.207.82.166:8008"},
{"HTTPS": "219.159.38.207:56210"},
{"HTTPS": "120.83.104.79:9999"},
{"HTTPS": "223.85.196.75:9797"},
{"HTTPS": "47.104.172.108:8118"},
{"HTTPS": "220.130.205.58:8080"},
{"HTTPS": "115.193.99.104:9000"},
{"HTTPS": "183.146.213.157:80"},
{"HTTPS": "39.137.69.7:80"},
{"HTTPS": "125.123.142.133:9999"},
{"HTTPS": "171.11.179.29:9999"},
{"HTTPS": "222.189.191.164:9999"},
{"HTTPS": "182.122.180.226:9999"},
{"HTTPS": "182.92.242.11:80"},
{"HTTPS": "58.87.98.112:1080"},
{"HTTPS": "58.253.154.211:9999"},
{"HTTPS": "122.4.44.86:9999"},
{"HTTPS": "163.125.115.84:8118"},
{"HTTPS": "117.93.83.55:53281"},
{"HTTPS": "115.159.155.83:8118"},
{"HTTPS": "117.131.99.210:53281"},
{"HTTPS": "47.92.83.106:3128"},
{"HTTPS": "182.34.34.45:9999"},
{"HTTPS": "60.9.1.81:80"},
{"HTTPS": "183.129.244.16:12471"},
{"HTTPS": "49.72.134.34:9999"},
{"HTTPS": "222.189.144.164:9999"},
{"HTTPS": "218.59.139.238:80"},
{"HTTPS": "36.255.86.226:82"},
{"HTTPS": "222.189.191.6:9999"},
{"HTTPS": "163.125.30.31:8118"},
{"HTTPS": "42.238.88.105:9999"},
{"HTTPS": "36.255.87.235:84"},
{"HTTPS": "122.243.15.140:9000"},
{"HTTPS": "117.87.177.253:9000"},
{"HTTPS": "61.128.208.94:3128"},
{"HTTPS": "125.110.125.12:9000"},
{"HTTPS": "113.78.255.55:9000"},
{"HTTPS": "119.176.96.134:9999"},
{"HTTPS": "222.189.191.100:9999"},
{"HTTPS": "111.40.84.73:9999"},
{"HTTPS": "59.32.37.17:3128"},
{"HTTPS": "49.70.94.249:9999"},
{"HTTPS": "136.228.128.14:61158"},
{"HTTPS": "222.189.144.236:9999"},
{"HTTPS": "60.13.42.135:9999"},
{"HTTPS": "39.100.37.58:3128"},
{"HTTPS": "122.234.206.94:9000"},
{"HTTPS": "115.239.24.218:9000"},
{"HTTPS": "124.47.7.38:80"},
{"HTTPS": "117.91.133.201:9999"},
{"HTTPS": "113.116.245.211:9797"},
{"HTTPS": "119.101.116.93:9999"},
{"HTTPS": "125.73.220.18:49128"},
{"HTTPS": "163.125.248.39:8088"},
{"HTTPS": "222.89.32.189:9999"},
{"HTTPS": "103.205.15.97:8080"},
{"HTTPS": "1.192.240.115:9999"},
{"HTTPS": "112.14.47.6:52024"},
{"HTTPS": "125.40.109.154:36761"},
{"HTTPS": "123.206.201.35:8118"},
{"HTTPS": "125.123.127.70:9000"},
{"HTTPS": "124.205.155.154:9090"},
{"HTTPS": "118.190.94.224:9001"},
{"HTTPS": "110.84.208.95:8118"},
]
proxy = random.choice(proxy_list)
return proxy
6.爬下来的ip可能有重复,我简单的去了下重
# 待去重的代理ip 列表
proxy_list = [
# 快代理免费ip
{"HTTPS": "183.129.207.89:11056"},
{"HTTPS": "119.145.2.100:44129"},
{"HTTPS": "121.225.67.109:3128"},
{"HTTPS": "163.204.245.234:9999"},
{"HTTPS": "193.112.128.212:8118"},
{"HTTPS": "59.62.26.172:9000"},
{"HTTPS": "120.194.109.179:47812"},
{"HTTPS": "1.192.244.225:9999"},
{"HTTPS": "183.129.207.89:28144"},
{"HTTPS": "114.113.222.202:80"},
{"HTTPS": "211.141.111.114:61395"},
{"HTTPS": "119.145.2.99:44129"},
{"HTTPS": "125.123.142.133:9999"},
{"HTTPS": "180.118.247.183:9999"},
{"HTTPS": "59.52.158.127:8060"},
{"HTTPS": "114.113.220.159:80"},
{"HTTPS": "136.228.128.232:8080"},
{"HTTPS": "119.101.118.26:9999"},
{"HTTPS": "113.105.201.183:3128"},
{"HTTPS": "112.27.167.74:33641"},
{"HTTPS": "121.204.150.159:8118"},
{"HTTPS": "183.147.29.213:9000"},
{"HTTPS": "163.204.243.136:9999"},
{"HTTPS": "163.204.247.107:9999"},
{"HTTPS": "1.197.203.27:9999"},
{"HTTPS": "60.191.57.78:14051"},
{"HTTPS": "113.121.40.95:9999"},
{"HTTPS": "171.11.33.209:9999"},
{"HTTPS": "221.239.86.26:32228"},
{"HTTPS": "106.14.11.65:8118"},
{"HTTPS": "113.124.84.217:9999"},
{"HTTPS": "118.89.44.224:8118"},
{"HTTPS": "163.125.115.84:8118"},
{"HTTPS": "211.159.171.58:80"},
{"HTTPS": "118.122.114.238:9000"},
{"HTTPS": "117.90.137.30:9000"},
{"HTTPS": "117.90.252.164:9999"},
{"HTTPS": "47.106.244.250:8118"},
{"HTTPS": "121.232.194.14:9000"},
{"HTTPS": "115.193.99.104:9000"},
{"HTTPS": "120.234.138.99:53779"},
{"HTTPS": "115.159.155.83:8118"},
{"HTTPS": "220.249.149.147:9999"},
{"HTTPS": "45.125.32.181:3128"},
{"HTTPS": "125.123.136.226:9999"},
{"HTTPS": "180.118.86.10:9000"},
{"HTTPS": "163.125.221.24:8118"},
{"HTTPS": "122.234.206.94:9000"},
{"HTTPS": "58.22.207.137:9000"},
{"HTTPS": "222.189.191.121:9999"},
{"HTTPS": "183.129.207.89:27985"},
{"HTTPS": "59.62.26.93:9000"},
{"HTTPS": "218.91.50.234:1080"},
{"HTTPS": "218.95.50.228:9000"},
{"HTTPS": "125.110.82.142:9000"},
{"HTTPS": "60.13.42.31:9999"},
{"HTTPS": "117.90.1.248:9000"},
{"HTTPS": "222.128.9.235:33428"},
{"HTTPS": "125.115.180.190:3128"},
{"HTTPS": "27.189.128.205:53281"},
{"HTTPS": "124.47.7.38:80"},
{"HTTPS": "119.101.116.93:9999"},
{"HTTPS": "182.107.13.45:9000"},
{"HTTPS": "117.87.177.253:9000"},
{"HTTPS": "47.94.89.87:3128"},
{"HTTPS": "136.228.128.14:61158"},
{"HTTPS": "123.206.201.35:8118"},
{"HTTPS": "175.42.158.140:9999"},
{"HTTPS": "125.123.141.70:9999"},
{"HTTPS": "120.234.138.102:53779"},
{"HTTPS": "125.110.120.214:9000"},
{"HTTPS": "111.75.193.24:48449"},
{"HTTPS": "163.125.233.93:8118"},
{"HTTPS": "114.113.220.187:80"},
{"HTTPS": "61.183.233.6:54896"},
{"HTTPS": "163.125.30.31:8118"},
{"HTTPS": "60.13.42.8:9999"},
{"HTTPS": "122.243.15.140:9000"},
{"HTTPS": "121.232.148.166:9000"},
{"HTTPS": "120.77.45.130:80"},
{"HTTPS": "183.129.207.80:12889"},
{"HTTPS": "183.129.207.89:27794"},
{"HTTPS": "183.129.207.89:28057"},
{"HTTPS": "119.145.2.98:44129"},
{"HTTPS": "115.216.36.133:9000"},
{"HTTPS": "121.232.148.37:9000"},
{"HTTPS": "139.199.38.177:8118"},
{"HTTPS": "115.196.48.108:9000"},
{"HTTPS": "219.159.38.207:56210"},
{"HTTPS": "222.135.92.68:38094"},
{"HTTPS": "111.230.167.207:8118"},
{"HTTPS": "124.160.56.76:31484"},
{"HTTPS": "175.18.13.45:8080"},
{"HTTPS": "39.100.37.58:3128"},
{"HTTPS": "39.106.127.234:80"},
{"HTTPS": "59.62.6.9:9000"},
{"HTTPS": "163.204.246.231:9999"},
{"HTTPS": "1.192.240.115:9999"},
{"HTTPS": "117.90.6.32:9000"},
{"HTTPS": "125.110.125.12:9000"},
{"HTTPS": "140.143.246.163:9999"},
{"HTTPS": "120.27.208.131:80"},
{"HTTPS": "114.239.144.227:9999"},
{"HTTPS": "115.29.214.175:8118"},
{"HTTPS": "175.44.109.192:9999"},
{"HTTPS": "163.204.246.133:9999"},
{"HTTPS": "122.243.13.249:9000"},
{"HTTPS": "163.125.114.63:8118"},
{"HTTPS": "117.90.252.244:9000"},
{"HTTPS": "114.113.222.149:80"},
{"HTTPS": "183.158.201.100:9000"},
{"HTTPS": "222.189.144.228:9999"},
{"HTTPS": "114.113.222.136:80"},
{"HTTPS": "183.129.244.16:12471"},
{"HTTPS": "110.84.208.95:8118"},
{"HTTPS": "111.177.161.65:9999"},
{"HTTPS": "124.205.143.212:39904"},
{"HTTPS": "58.22.222.170:9000"},
{"HTTPS": "125.123.142.228:9999"},
{"HTTPS": "49.86.180.22:9999"},
{"HTTPS": "114.113.222.211:80"},
{"HTTPS": "61.164.39.68:53281"},
{"HTTPS": "122.234.204.214:9000"},
{"HTTPS": "183.158.207.111:9000"},
{"HTTPS": "117.90.137.150:9000"},
{"HTTPS": "119.101.112.219:9999"},
{"HTTPS": "121.238.76.53:8118"},
{"HTTPS": "118.24.157.49:8118"},
{"HTTPS": "114.113.222.154:80"},
{"HTTPS": "117.90.5.109:9000"},
{"HTTPS": "114.119.116.92:61066"},
{"HTTPS": "125.40.109.154:61875"},
{"HTTPS": "163.204.247.75:9999"},
{"HTTPS": "175.44.109.52:9999"},
# 98免费ip
{"HTTPS": "119.255.248.54:808"},
{"HTTPS": "42.238.90.196:9999"},
{"HTTPS": "27.43.185.62:9999"},
{"HTTPS": "183.146.213.157:80"},
{"HTTPS": "27.43.185.45:9999"},
{"HTTPS": "120.79.184.148:8118"},
{"HTTPS": "222.162.239.92:80"},
{"HTTPS": "60.13.42.63:9999"},
{"HTTPS": "124.16.71.141:1080"},
{"HTTPS": "218.91.112.235:9999"},
{"HTTPS": "113.116.245.211:9797"},
{"HTTPS": "183.129.207.93:14153"},
{"HTTPS": "221.122.91.60:80"},
{"HTTPS": "121.17.174.121:9999"},
{"HTTPS": "124.205.155.154:9090"},
{"HTTPS": "180.119.141.247:9999"},
{"HTTPS": "58.211.183.46:40225"},
{"HTTPS": "58.253.152.43:9999"},
{"HTTPS": "49.86.183.99:9999"},
{"HTTPS": "180.119.68.71:9999"},
{"HTTPS": "59.120.231.207:3128"},
{"HTTPS": "163.204.244.210:9999"},
{"HTTPS": "115.239.24.218:9000"},
{"HTTPS": "123.57.84.116:8118"},
{"HTTPS": "58.219.126.189:9999"},
{"HTTPS": "120.79.62.89:3128"},
{"HTTPS": "60.217.73.238:8060"},
{"HTTPS": "42.238.85.40:9999"},
{"HTTPS": "180.119.68.167:9999"},
{"HTTPS": "49.86.182.6:9999"},
{"HTTPS": "42.238.89.232:9999"},
{"HTTPS": "115.53.34.200:9999"},
{"HTTPS": "222.189.191.138:9999"},
{"HTTPS": "61.164.39.66:53281"},
{"HTTPS": "27.43.189.148:9999"},
{"HTTPS": "42.238.81.147:9999"},
{"HTTPS": "220.133.57.116:49242"},
{"HTTPS": "103.235.199.38:56975"},
{"HTTPS": "115.61.60.228:9999"},
{"HTTPS": "144.123.69.201:9999"},
{"HTTPS": "116.113.27.170:61852"},
{"HTTPS": "182.34.37.145:9999"},
{"HTTPS": "182.34.37.125:9999"},
{"HTTPS": "182.38.189.17:8118"},
{"HTTPS": "36.255.86.232:82"},
{"HTTPS": "60.13.42.135:9999"},
{"HTTPS": "125.123.120.233:9000"},
{"HTTPS": "27.43.184.231:9999"},
{"HTTPS": "47.104.172.108:8118"},
{"HTTPS": "42.238.91.164:9999"},
{"HTTPS": "49.51.70.42:1080"},
{"HTTPS": "183.62.37.164:8080"},
{"HTTPS": "112.85.164.215:9999"},
{"HTTPS": "124.207.82.166:8008"},
{"HTTPS": "103.59.213.29:8080"},
{"HTTPS": "223.241.78.26:8010"},
{"HTTPS": "125.73.220.18:49128"},
{"HTTPS": "36.248.133.126:9999"},
{"HTTPS": "180.107.248.4:9999"},
{"HTTPS": "222.89.32.139:9999"},
{"HTTPS": "221.227.74.141:9999"},
{"HTTPS": "113.140.1.82:53281"},
{"HTTPS": "163.125.248.39:8088"},
{"HTTPS": "182.34.21.118:9999"},
{"HTTPS": "60.205.57.4:3128"},
{"HTTPS": "47.94.221.41:8118"},
{"HTTPS": "101.132.164.113:8118"},
{"HTTPS": "180.119.141.61:9999"},
{"HTTPS": "120.83.104.79:9999"},
{"HTTPS": "117.131.99.210:53281"},
{"HTTPS": "58.219.126.189:9999"},
{"HTTPS": "222.189.144.88:9999"},
{"HTTPS": "222.189.191.242:9999"},
{"HTTPS": "221.2.175.238:8060"},
{"HTTPS": "120.83.104.46:9999"},
{"HTTPS": "61.183.233.6:54896"},
{"HTTPS": "183.63.114.204:54321"},
{"HTTPS": "222.189.191.6:9999"},
{"HTTPS": "36.248.129.213:9999"},
{"HTTPS": "182.92.113.183:8118"},
{"HTTPS": "113.109.28.105:53281"},
{"HTTPS": "222.189.144.92:9999"},
{"HTTPS": "42.238.84.235:9999"},
{"HTTPS": "118.24.246.249:80"},
{"HTTPS": "180.119.141.237:9999"},
{"HTTPS": "36.69.136.130:8080"},
{"HTTPS": "125.110.114.212:8118"},
{"HTTPS": "182.34.34.139:9999"},
{"HTTPS": "180.119.68.6:9999"},
{"HTTPS": "123.207.43.128:1080"},
{"HTTPS": "123.163.96.145:9999"},
{"HTTPS": "221.176.206.29:8060"},
{"HTTPS": "103.205.15.97:8080"},
{"HTTPS": "49.51.155.45:8081"},
{"HTTPS": "222.189.190.188:9999"},
{"HTTPS": "59.172.27.6:38380"},
{"HTTPS": "47.101.156.232:8080"},
{"HTTPS": "117.141.215.3:1080"},
{"HTTPS": "36.255.86.231:83"},
{"HTTPS": "122.136.212.132:53281"},
{"HTTPS": "47.106.205.193:8888"},
{"HTTPS": "58.253.158.194:9999"},
{"HTTPS": "58.253.152.57:9999"},
{"HTTPS": "222.223.182.66:8000"},
{"HTTPS": "222.89.32.189:9999"},
{"HTTPS": "36.99.16.241:82"},
{"HTTPS": "125.40.109.154:36761"},
{"HTTPS": "119.3.253.33:8080"},
{"HTTPS": "117.191.11.109:80"},
{"HTTPS": "113.110.225.11:3128"},
{"HTTPS": "115.171.85.202:9000"},
{"HTTPS": "118.190.94.224:9001"},
{"HTTPS": "113.108.242.36:47713"},
{"HTTPS": "60.182.162.250:61234"},
{"HTTPS": "117.191.11.102:80"},
{"HTTPS": "222.189.191.100:9999"},
{"HTTPS": "223.85.196.75:9797"},
{"HTTPS": "125.123.127.70:9000"},
{"HTTPS": "182.34.33.95:9999"},
{"HTTPS": "222.189.144.239:9999"},
{"HTTPS": "42.238.90.109:9999"},
{"HTTPS": "47.106.244.250:8118"},
{"HTTPS": "42.238.88.105:9999"},
{"HTTPS": "58.87.68.189:1080"},
{"HTTPS": "111.40.84.73:9999"},
{"HTTPS": "222.189.190.33:9999"},
{"HTTPS": "163.204.244.171:9999"},
{"HTTPS": "210.22.176.146:47578"},
{"HTTPS": "101.231.104.82:80"},
{"HTTPS": "42.238.88.113:9999"},
{"HTTPS": "218.22.7.62:53281"},
{"HTTPS": "49.70.64.182:9999"},
{"HTTPS": "59.32.37.17:3128"},
{"HTTPS": "140.143.48.49:1080"},
{"HTTPS": "182.34.35.127:9999"},
{"HTTPS": "47.107.160.233:808"},
{"HTTPS": "42.238.88.59:9999"},
{"HTTPS": "27.40.146.233:9999"},
{"HTTPS": "222.189.191.51:9999"},
{"HTTPS": "49.70.94.249:9999"},
{"HTTPS": "117.191.11.111:80"},
{"HTTPS": "117.191.11.105:80"},
{"HTTPS": "36.248.132.184:9999"},
{"HTTPS": "121.69.37.6:9797"},
{"HTTPS": "182.34.27.0:9999"},
{"HTTPS": "116.30.122.155:9000"},
{"HTTPS": "222.189.191.134:9999"},
{"HTTPS": "120.83.101.213:9999"},
{"HTTPS": "58.254.220.116:53579"},
{"HTTPS": "163.204.246.116:9999"},
{"HTTPS": "101.132.66.66:80"},
{"HTTPS": "1.199.30.227:9999"},
{"HTTPS": "117.127.16.206:8080"},
{"HTTPS": "58.253.154.211:9999"},
{"HTTPS": "218.91.112.200:9999"},
{"HTTPS": "220.168.52.245:50657"},
{"HTTPS": "36.248.132.57:9999"},
{"HTTPS": "171.11.179.29:9999"},
{"HTTPS": "123.206.200.63:1080"},
{"HTTPS": "111.230.254.195:8118"},
{"HTTPS": "117.191.11.103:80"},
{"HTTPS": "39.137.69.7:80"},
{"HTTPS": "222.89.32.187:9999"},
{"HTTPS": "182.122.180.226:9999"},
{"HTTPS": "42.228.3.158:8080"},
{"HTTPS": "140.143.152.93:1080"},
{"HTTPS": "113.124.95.138:9999"},
{"HTTPS": "45.115.174.29:30086"},
{"HTTPS": "47.93.28.220:8118"},
{"HTTPS": "117.91.232.55:9999"},
{"HTTPS": "182.34.33.154:9999"},
{"HTTPS": "182.34.34.45:9999"},
{"HTTPS": "58.87.98.112:1080"},
{"HTTPS": "112.14.47.6:52024"},
{"HTTPS": "222.189.190.30:9999"},
{"HTTPS": "27.43.190.157:9999"},
{"HTTPS": "120.25.203.182:7777"},
{"HTTPS": "117.93.83.55:53281"},
{"HTTPS": "60.9.1.81:80"},
{"HTTPS": "219.159.38.200:56210"},
{"HTTPS": "180.119.141.237:9999"},
{"HTTPS": "36.255.86.226:82"},
{"HTTPS": "222.189.191.64:9999"},
{"HTTPS": "1.175.133.55:3128"},
{"HTTPS": "182.34.33.216:9999"},
{"HTTPS": "27.43.191.118:9999"},
{"HTTPS": "61.128.208.94:3128"},
{"HTTPS": "121.15.254.156:888"},
{"HTTPS": "118.190.145.138:9001"},
{"HTTPS": "36.255.87.227:82"},
{"HTTPS": "210.5.10.87:53281"},
{"HTTPS": "27.43.186.234:9999"},
{"HTTPS": "61.164.39.66:53281"},
{"HTTPS": "180.119.68.128:9999"},
{"HTTPS": "144.123.69.78:9999"},
{"HTTPS": "45.115.176.176:8080"},
{"HTTPS": "218.73.134.50:9999"},
{"HTTPS": "58.54.227.111:61234"},
{"HTTPS": "218.59.139.238:80"},
{"HTTPS": "175.6.5.234:8888"},
{"HTTPS": "183.209.23.154:8118"},
{"HTTPS": "222.189.190.80:9999"},
{"HTTPS": "218.91.112.219:9999"},
{"HTTPS": "182.34.34.17:9999"},
{"HTTPS": "182.34.32.252:9999"},
{"HTTPS": "47.93.36.195:8118"},
{"HTTPS": "222.189.190.254:9999"},
{"HTTPS": "59.48.237.6:8060"},
{"HTTPS": "121.233.207.124:9999"},
{"HTTPS": "119.176.96.134:9999"},
{"HTTPS": "180.160.62.196:8118"},
{"HTTPS": "58.219.126.221:9999"},
{"HTTPS": "58.253.159.121:9999"},
{"HTTPS": "27.43.185.96:9999"},
{"HTTPS": "27.43.190.124:9999"},
{"HTTPS": "180.119.141.186:9999"},
{"HTTPS": "121.79.131.58:8080"},
{"HTTPS": "42.238.81.160:9999"},
{"HTTPS": "36.248.132.82:9999"},
{"HTTPS": "122.4.44.86:9999"},
{"HTTPS": "49.86.178.37:9999"},
{"HTTPS": "222.189.191.164:9999"},
{"HTTPS": "49.70.64.182:9999"},
{"HTTPS": "222.189.144.236:9999"},
{"HTTPS": "49.86.178.78:9999"},
{"HTTPS": "218.91.112.168:9999"},
{"HTTPS": "153.35.213.79:8118"},
{"HTTPS": "125.123.121.247:9000"},
{"HTTPS": "45.115.175.137:30944"},
{"HTTPS": "47.103.197.197:8080"},
{"HTTPS": "122.4.45.101:9999"},
{"HTTPS": "125.123.69.45:9000"},
{"HTTPS": "49.86.183.69:9999"},
{"HTTPS": "42.238.82.46:9999"},
{"HTTPS": "47.107.160.99:8118"},
{"HTTPS": "180.119.68.128:9999"},
{"HTTPS": "58.253.158.182:9999"},
{"HTTPS": "49.86.182.228:9999"},
{"HTTPS": "220.130.205.58:8080"},
{"HTTPS": "123.206.30.254:8118"},
{"HTTPS": "180.107.55.119:8118"},
{"HTTPS": "113.78.255.55:9000"},
{"HTTPS": "1.192.242.22:9999"},
{"HTTPS": "117.191.11.110:80"},
{"HTTPS": "140.143.137.69:1080"},
{"HTTPS": "36.255.87.252:83"},
{"HTTPS": "45.221.77.246:30496"},
{"HTTPS": "124.193.37.5:8888"},
{"HTTPS": "222.189.191.204:9999"},
{"HTTPS": "49.72.134.34:9999"},
{"HTTPS": "180.119.141.104:9999"},
{"HTTPS": "183.146.213.56:80"},
{"HTTPS": "119.28.203.242:8000"},
{"HTTPS": "118.190.95.43:9001"},
{"HTTPS": "113.65.5.149:8118"},
{"HTTPS": "182.92.242.11:80"},
{"HTTPS": "139.196.208.220:80"},
{"HTTPS": "140.143.142.14:1080"},
{"HTTPS": "61.130.235.241:8010"},
{"HTTPS": "222.189.191.170:9999"},
{"HTTPS": "140.143.152.93:1080"},
{"HTTPS": "36.255.87.235:84"},
{"HTTPS": "180.119.141.20:9999"},
{"HTTPS": "49.86.179.133:9999"},
{"HTTPS": "14.204.211.133:8060"},
{"HTTPS": "27.208.91.251:8060"},
{"HTTPS": "220.143.107.178:3128"},
{"HTTPS": "117.127.16.207:8080"},
{"HTTPS": "27.43.185.96:9999"},
{"HTTPS": "124.205.155.155:9090"},
{"HTTPS": "58.253.158.57:9999"},
{"HTTPS": "39.137.168.230:80"},
{"HTTPS": "47.92.83.106:3128"},
{"HTTPS": "123.206.204.51:8118"},
{"HTTPS": "222.89.32.188:9999"},
{"HTTPS": "49.89.143.213:9999"},
{"HTTPS": "222.189.144.164:9999"},
{"HTTPS": "116.196.81.58:3128"},
{"HTTPS": "221.122.91.65:80"},
{"HTTPS": "118.24.89.122:1080"},
{"HTTPS": "210.26.49.88:3128"},
{"HTTPS": "124.237.83.14:53281"},
{"HTTPS": "140.143.246.163:8080"},
{"HTTPS": "180.122.145.43:8888"},
{"HTTPS": "222.189.144.119:9999"},
{"HTTPS": "117.91.133.201:9999"},
{"HTTPS": "113.247.252.114:9090"},
{"HTTPS": "219.85.252.143:80"},
{"HTTPS": "1.175.133.9:3128"},
{"HTTPS": "27.43.189.148:9999"},
{"HTTPS": "180.119.141.140:9999"},
{"HTTPS": "59.32.37.166:61234"},
{"HTTPS": "60.13.42.182:9999"},
]
print(len(proxy_list))
# 创建一个空的列表,便利待去重列表,添加到new_list
new_list = []
for i in range(len(proxy_list)):
proxy = str(proxy_list[i].get('HTTPS'))
new_list.append(proxy)
# 去重
set_proxy = set(new_list)
# 元祖转列表
list_proxy = list(set_proxy)
print(len(list_proxy))
# 显示去重后的字典格式的proxy
with open('new_proxy.txt', 'a', encoding="utf-8") as fp:
for j in range(len(list_proxy)):
string = str('{"HTTPS": "%s"},' % (list_proxy[j]))
fp.write(string + '\n')
fp.flush()