'''
作者：Valve
申明：本程序仅供技术交流，不可用于违法之事。若出现法律纠纷，本人概不负责。
'''
import requests,shutil
import os
from bs4 import BeautifulSoup
import random
import datetime
import re
today = datetime.date(datetime.datetime.now().year, datetime.datetime.now().month, datetime.datetime.now().day)
###########################################################################
StartDate = datetime.date(2000,4,23) #起始发布日期
EndDate = datetime.date(2020,4,25) #截至发布日期
#如果想搜索2019年3月1日到2020年4月2日之间发布的图集，只需设置StartDate和EndDate即可
SearchTerm = ['尹菲'] #SearchTerm可以写入任何想搜索的关键词，也可以不写入任何关键词
BlackList = ['周于希','王雨纯','孟狐狸'] #黑名单，过滤掉任何不想要的词
SavePath = 'C:/Users/Jared/Pictures/Saved Pictures/get/mm131/' #图片保存路径
###########################################################################
url = 'https://www.mm131.net/xinggan/' #起始搜索页
PicSerUrl = 'https://img1.mmmw.net/pic/'
RefererLink = 'https://www.mm131.net'
my_headers = [
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
'Opera/9.25 (Windows NT 5.1; U; en)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "
]
if not os.path.exists(SavePath):
os.mkdir(SavePath)
if url == 'https://www.mm131.net/xinggan/':
PageNo = 1
else:
PageNo = int(url[37:-5])+1
NextUrl = url
while NextUrl:
head = {'Referer':RefererLink ,'user-agent': random.choice(my_headers)}
html = requests.get( NextUrl, headers=head )
if str(html) == '':
break
else:
PageNo += 1
NextUrl = 'https://www.mm131.net/xinggan/list_6_'+str(PageNo)+'.html'
bs = BeautifulSoup(html.content, 'lxml')
MainBox = bs.find('dl',class_='list-left public-box')
AllArticles = MainBox.find_all('dd',class_='')
for article in AllArticles:
head = {'Referer':RefererLink,'user-agent': random.choice(my_headers)}
GroupLinkTemp = article.find('a')
GroupLink = GroupLinkTemp['href']
GroupNameTemp_0 = GroupLinkTemp.find('img')
if str(type(GroupNameTemp_0)) == '':
continue
GroupNameTemp = GroupNameTemp_0['alt']
RGName = ''.join( e for e in GroupNameTemp if e.isalnum() )
Ghtml = requests.get(GroupLink,headers=head)
Gbs = BeautifulSoup(Ghtml.content, 'lxml')
NumTemp = Gbs.find('span',class_='page-ch')
if str(NumTemp) == 'None':
continue
Num = int( NumTemp.text[1:-1] )
WantMark = 1
if len(SearchTerm) > 0:
for OnlyWant in SearchTerm:
pattern = re.compile('.*'+OnlyWant+'.*')
if re.match(pattern,GroupNameTemp):
WantMark = 1
break
else:
WantMark = 0
if not WantMark:
print(GroupNameTemp,'不符合搜索条件，已忽略')
continue
#filter black list
BlackMark = 0
for black in BlackList:
pattern = re.compile('.*'+black+'.*')
if re.match(pattern,GroupNameTemp):
BlackMark = 1
print(GroupNameTemp,'在黑名单中，已忽略')
break
if BlackMark :
continue
DateBox = Gbs.find('div',class_='content-msg')
PostDateTemp = DateBox.text[5:15]
year = int(PostDateTemp[0:4])
month = int(PostDateTemp[5:7])
day = int(PostDateTemp[8:])
PostDate = datetime.date(year,month,day)
RPostDate = ''.join( e for e in PostDateTemp if e.isnumeric() )
if PostDate > EndDate:
print(GroupNameTemp,'不在指定日期范围内，已忽略')
continue
elif PostDate < StartDate:
break
n = -1
while 1:
if GroupLink[n] == '.':
end = n
elif GroupLink[n] == '/':
start = n
break
n -=1
GroupNo = GroupLink[start+1:end] ##
GroupName = RPostDate+'-'+RGName
DirPath = SavePath+GroupName+'/'
pattern = re.compile('.*'+RGName+'.*')
AllDirName = os.listdir(SavePath)
jump = ex = 0
for DirName in AllDirName:
if re.match(pattern,DirName):
if len(os.listdir(SavePath+DirName)) >= Num:
print(SavePath+DirName,'：目录已存在，且图片已全部下载，跳过此图集')
jump = 1
break
else:
shutil.rmtree(SavePath+DirName)
print(SavePath+DirName,'：目录已存在，且图片已只下载了一部分，重新下载')
break
if jump:
continue
os.mkdir(DirPath)
j = 1
while j <= Num:
ImageUrl = PicSerUrl+str(GroupNo)+'/'+str(j)+'.jpg'
head = {'Referer':'https://m.mm131.net/','user-agent': random.choice(my_headers)}
pic = requests.get(ImageUrl,headers = head)
with open(DirPath+GroupName+'-'+str(j)+'.jpg','wb') as f:
f.write(pic.content)
print('【mm131】',GroupName,'【'+str(j)+'/'+str(Num)+'】')
j += 1
if PostDate < StartDate:
break
