基于python对doi号通过sci-hub进行pubmed中的文献下载
基于python对doi号进行pubmed中的文献下载
文档准备
1.构建你的检索式,以AD为例，搜索出的结果选择
2.创建文档
代码展示
import requests
import re
import os
import urllib.request
import openpyxl
# headers 保持与服务器的会话连接
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
}
headers 获取
'''
根据doi，找到文献的pdf，然后下载到本地
'''
def getPaperPdf(url):
pattern = '/.*?\.pdf'
content = requests.get(url, headers=headers)
download_url = re.findall(pattern, content.text)
# print(download_url)
download_url[1] = "https:" + download_url[1]
print(download_url[1])
path = r"papers"
if os.path.exists(path):
pass
else:
os.makedirs(path)
# 使用 urllib.request 来包装请求
req = urllib.request.Request(download_url[1], headers=headers)
# 使用 urllib.request 模块中的 urlopen方法获取页面
u = urllib.request.urlopen(req, timeout=5)
file_name = download_url[1].split('/')[-2] + '%' + download_url[1].split('/')[-1]
f = open(path + '/' + file_name, 'wb')
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
f.write(buffer)
f.close()
print("Sucessful to download" + " " + file_name)
'''
将表格放在代码保存和运行的路径内，将wb变量内的'***.xlsx'改为自己的excel文件名,
最后下载的论文在该路径下新建的papers文件夹内
'''
wb = openpyxl.load_workbook('D:\pythonProject\doi.xlsx')
# doi在sheet1中
Sheet1 = wb['Sheet1']
# 读取第*列(doi号在的那一列)
'''
修改代码内，excel中doi所在列，比如在k列，所以col_range变量后面的字符改为‘k’
'''
col_range = Sheet1['K']
# 也可以读取其中的第几行：row_range = sheet1[2:6]
fails = [] #应该在搜索查询打印到一个.txt文件中
从pubmed下载下来的CSV格式文档另存为python文件目录下的.xlsx格式，第K行就是doi号了
下面通过sci-hub进行下载
for col in col_range: # 打印dio列单元格中的值内容
doi = col.value
print(doi)
if __name__ == '__main__':
sci_Hub_Url = "https://sci-hub.ren/"
paper_url = sci_Hub_Url + doi
print(paper_url)
nmm = 0
try:
getPaperPdf(paper_url) # 通过文献的url下载pdf
continue
except Exception:
nmm = 1
print("Failed to get pdf 1")
if nmm == 1:
try:
sci_Hub_Url_2 = "https://sci-hub.se/"
paper_url_2 = sci_Hub_Url_2 + doi
getPaperPdf(paper_url_2)
continue
except Exception:
print("Failed to get pdf 2")
记得下载之前先登录一下sci-hub网站进行测试，如果打不开请更新最新的sci-hub网址
最后获取下载失败文献的doi号
# 获取下载失败的doi
print(fails)
因为是通过sci-hub进行下载的所以代码运行结束会有很多无法下载的doi号
又无法通过pubmed官网进行爬取，
所以只能手动检测一遍，输入到pubmed上进行下载获取。
最后完整代码为
import requests
import re
import os
import urllib.request
import openpyxl
# headers 保持与服务器的会话连接
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/536.36',
}
'''
根据doi，找到文献的pdf，然后下载到本地
'''
def getPaperPdf(url):
pattern = '/.*?\.pdf'
content = requests.get(url, headers=headers)
download_url = re.findall(pattern, content.text)
# print(download_url)
download_url[1] = "https:" + download_url[1]
print(download_url[1])
path = r"papers"
if os.path.exists(path):
pass
else:
os.makedirs(path)
# 使用 urllib.request 来包装请求
req = urllib.request.Request(download_url[1], headers=headers)
# 使用 urllib.request 模块中的 urlopen方法获取页面
u = urllib.request.urlopen(req, timeout=5)
file_name = download_url[1].split('/')[-2] + '%' + download_url[1].split('/')[-1]
f = open(path + '/' + file_name, 'wb')
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
f.write(buffer)
f.close()
print("Sucessful to download" + " " + file_name)
'''
将表格放在代码保存和运行的路径内，将wb变量内的'***.xlsx'改为自己的excel文件名,
最后下载的论文在该路径下新建的papers文件夹内
'''
wb = openpyxl.load_workbook('D:\pythonProject\doi.xlsx')
# doi在sheet1中
Sheet1 = wb['Sheet1']
# 读取第*列(doi号在的那一列)
'''
修改代码内，excel中doi所在列，比如在k列，所以col_range变量后面的字符改为了‘k’
'''
col_range = Sheet1['K']
# 也可以读取其中的第几行：row_range = sheet1[2:6]
fails = [] #应该在搜索查询打印到一个.txt文件中
# 加入SCI-hub网址进行下载
for col in col_range: # 打印dio列单元格中的值内容
doi = col.value
print(doi)
if __name__ == '__main__':
sci_Hub_Url = "https://sci-hub.ren/"
paper_url = sci_Hub_Url + doi
print(paper_url)
nmm = 0
try:
getPaperPdf(paper_url) # 通过文献的url下载pdf
continue
except Exception:
nmm = 1
print("Failed to get pdf 1")
if nmm == 1:
try:
sci_Hub_Url_2 = "https://sci-hub.se/"
paper_url_2 = sci_Hub_Url_2 + doi
getPaperPdf(paper_url_2)
continue
except Exception:
print("Failed to get pdf 2")
# if nmm == 1:
# try:
# sci_Hub_Url_3 = "https://sci-hub.st/"
# paper_url_3 = sci_Hub_Url_3 + doi
# getPaperPdf(paper_url_3)
# continue
# except Exception:
# print("Failed to get pdf 3")
# if nmm == 1:
# try:
# sci_Hub_Url_4 = "https://sci-hub.shop/"
# paper_url_4 = sci_Hub_Url_4 + doi
# getPaperPdf(paper_url_4)
# continue
# except Exception:
# print("Failed to get pdf 4")
# if nmm == 1:
# try:
# sci_Hub_Url_5 = "https://sci-hub.shop/"
# paper_url_5 = sci_Hub_Url_5 + doi
# getPaperPdf(paper_url_5)
# continue
# except Exception:
# print("Failed to get pdf 5")
# if nmm == 1:
# try:
# sci_Hub_Url_6 = "https://libgen.ggfwzs.net/"
# paper_url_6 = sci_Hub_Url_6 + doi
# getPaperPdf(paper_url_6)
# continue
# except Exception:
# print("Failed to get pdf 6")
# fails.append(doi)
# if nmm == 1:
# try:
# sci_Hub_Url_7 = "https://sci-hub.do/"
# paper_url_7 = sci_Hub_Url_7 + doi
# getPaperPdf(paper_url_7)
# continue
# except Exception:
# print("Failed to get pdf 7")
# 获取下载失败的doi
print(fails)