nginx upload模块+python 后端处理模仿fastdfs实现文件存取
前提条件：
1.编译安装带有nginx upload 模块的nginx，安装脚本如下。
2.配置nginx的conf文件，一个server配置upload上传，一个server配置download下载
3.使用django快速部署python后端服务，对nginx存入的文件进行处理，并返回下载的url
注释：如果不适用后端处理，nginx也可以实现将文件上传，但是由于upload模块将文件存入时会自己命名存入，无法记录相关文件类型。如果直接从nginx存储目录下载，会导致文件的后缀丢失，无法获取文件类型。
####nginx安装脚本
[root@nginx ~]# cat autonginx
#!/bin/bash
#install nginx/1.10.3
#Installation path
INSTALL_DIR=/usr/local/
SRC_DIR=/opt/package
[ ! -d ${INSTALL_DIR} ] && mkdir -p ${INSTALL_DIR}
[ ! -d ${SRC_DIR} ] && mkdir -p ${SRC_DIR}
# Check if user is root
if [ $(id -u) != "0" ]; then
echo "Error: You must be root to run this script!!"
exit 1
fi
#Install dependency package
for Package in wget gcc gcc-c++ autoconf automake zlib zlib-devel openssl openssl-devel pcre pcre-devel unzip
do
yum -y install $Package
done
function Install_Nginx()
{
#update version
NGINX="nginx-1.10.3"
PCRE="pcre-8.40"
ZLIB="zlib-1.2.11"
OPENSSL="openssl-1.1.0e"
PURGE="ngx_cache_purge-2.3"
UPLOAD="nginx-upload-module-2.2"
NGINXFEATURES="--prefix=${INSTALL_DIR}nginx \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_gzip_static_module \
--with-http_stub_status_module \
--with-http_realip_module \
--with-stream \
--with-stream_ssl_module \
--pid-path=/var/run/nginx.pid \
--with-pcre=${SRC_DIR}/${PCRE} \
--with-zlib=${SRC_DIR}/${ZLIB} \
--with-openssl=${SRC_DIR}/${OPENSSL} \
--add-module=${SRC_DIR}/${PURGE} \
--add-module=${SRC_DIR}/${UPLOAD}
"
cd ${SRC_DIR}
#download package
echo 'Downloading NGINX'
if [ ! -f ${NGINX}.tar.gz ]
then
wget -c http://nginx.org/download/${NGINX}.tar.gz
else
echo 'Skipping: NGINX already downloaded'
fi
echo 'Downloading PCRE'
if [ ! -f ${PCRE}.tar.gz ]
then
wget -c https://ftp.pcre.org/pub/pcre/${PCRE}.tar.gz
else
echo 'Skipping: PCRE already downloaded'
fi
echo 'Downloading ZLIB'
if [ ! -f ${ZLIB}.tar.gz ]
then
wget -c http://zlib.net/${ZLIB}.tar.gz
else
echo 'Skipping: ZLIB already downloaded'
fi
echo 'Downloading OPENSSL'
if [ ! -f ${OPENSSL}.tar.gz ]
then
wget -c http://www.openssl.org/source/${OPENSSL}.tar.gz
else
echo 'Skipping: OPENSSL already downloaded'
fi
echo 'Downloading PURGE'
if [ ! -f 2.3.tar.gz ]
then
wget -c https://github.com//FRiCKLE/ngx_cache_purge/archive/2.3.tar.gz
else
echo 'Skipping: PURGE already downloaded'
fi
if [ ! -f 2.2 ]
then
wget -c https://codeload.github.com/vkholodkov/nginx-upload-module/zip/2.2
else
echo 'Skipping: nginx-upload-module already downloaded'
fi
echo '----------Unpacking downloaded archives. This process may take serveral minutes---------'
echo "Extracting ${NGINX}..."
tar xzf ${NGINX}.tar.gz
echo 'Done.'
echo "Extracting ${PCRE}..."
tar xzf ${PCRE}.tar.gz
echo 'Done.'
echo "Extracting ${ZLIB}..."
tar xzf ${ZLIB}.tar.gz
echo 'Done.'
echo "Extracting ${OPENSSL}..."
tar xzf ${OPENSSL}.tar.gz
echo 'Done.'
echo "Extracting ${PURGE}..."
tar xzf 2.3.tar.gz
echo 'Done.'
echo "Extracting ${UPLOAD}..."
unzip 2.2
echo 'Done.'
#add new user
groupadd -r nginx
useradd -r -g nginx nginx
#make
echo '###################'
echo 'Compile NGINX'
echo '###################'
cd ${SRC_DIR}/${NGINX}
./configure ${NGINXFEATURES}
make
make install
echo '###################'
echo 'ok'
echo '###################'
}
Install_Nginx在此前，创建/www/web/upload/tmp目录。然后配置一下内容重启nginx。
####nginx配置文件
#####upload所需要的server
server {
listen 99;
client_max_body_size 20m;
client_body_buffer_size 512k;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header REMOTE_ADD $remote_addr;
error_page 405 =200 @405;
location / {
index index.html index.htm index.php;
}
location @405
{
root /www/web/upload;
}
location /upload {
# 转到后台处理URL,表示Nginx接收完上传的文件后，然后交给后端处理的地址
upload_pass @python;
# 临时保存路径, 可以使用散列
# 上传模块接收到的文件临时存放的路径， 1 表示方式，该方式是需要在/tmp/nginx_upload下创建以0到9为目录名称的目录，上传时候会进行一个散列处理。
upload_store /www/web/upload/tmp 1;
# 上传文件的权限，rw表示读写 r只读
upload_store_access user:rw group:rw all:rw;
set $upload_field_name "file";
# upload_resumable on;
# 这里写入http报头，pass到后台页面后能获取这里set的报头字段
upload_set_form_field "${upload_field_name}_name" $upload_file_name;
upload_set_form_field "${upload_field_name}_content_type" $upload_content_type;
upload_set_form_field "${upload_field_name}_path" $upload_tmp_path;
# Upload模块自动生成的一些信息，如文件大小与文件md5值
upload_aggregate_form_field "${upload_field_name}_md5" $upload_file_md5;
upload_aggregate_form_field "${upload_field_name}_size" $upload_file_size;
# 允许的字段，允许全部可以 "^.*$"
upload_pass_form_field "^.*$";
# upload_pass_form_field "^submit$|^description$";
# 每秒字节速度控制，0表示不受控制，默认0, 128K
upload_limit_rate 0;
# 如果pass页面是以下状态码，就删除此次上传的临时文件
upload_cleanup 400 404 499 500-505;
# 打开开关，意思就是把前端脚本请求的参数会传给后端的脚本语言，比如：http://192.168.1.251:9000/upload/?k=23,后台可以通过POST['k']来访问。
upload_pass_args on;
}
location @python {
proxy_pass http://localhost:9999;
# return 200; # 如果不需要后端程序处理，直接返回200即可
}
}
#######download所需要的server#############
server {
listen 88;
root /tmp/nginx_upload/;
location /nginx_upload {
root /tmp;
}
}此刻，你已经完成了nginx的配置，可以使用小程序进行一下上传测试，看看nginx是否可以上传文件。
import requests
file_dir = 'images/logo.png'
file = open(file_dir, 'rb')
files = {'attachment_file': (file_dir, file, 'image/png', {})}
r1 = requests.post('http://172.18.3.183:99/upload', files=files)
print(r1.text)
file.close()
通过python模拟post请求，将图片上传到刚才nginx开放的接口，然后在刚才创建的目录查找文件
接下来，进行后端服务器的部署，可以使用django进行部署。
1.pip install Django==1.8.9
2. django-admin.py startproject UPload
3. cd UPload/
4. python manage.py startapp file_parse
5. cd file_parse
6. 将下面的view.py添加到view.py中
7. 在settings.py中添加file_parse
8. 在url.py中添加 url(r’^upload’,file_parse_views.upload),
9. python manage.py runserver 0.0.0.0:9999
#django中的view.py
import os
import json
import time
from django.views.decorators.csrf import csrf_exempt
date = time.strftime('%Y%m%d')
UPLOAD_FILE_PATH = '/tmp/nginx_upload/%s/' %date
isExists=os.path.exists(UPLOAD_FILE_PATH)
if not isExists:
os.makedirs(UPLOAD_FILE_PATH)
else:
print('path isexist!')
from django.shortcuts import render
from django.http import HttpResponse
# Create your views here.
def index(request):
return HttpResponse("Hello Django!")
@csrf_exempt
def upload(request):
request_params = request.POST
print(request_params)
file_name = request_params['file_name']
file_content_type = request_params['file_content_type']
file_md5 = request_params['file_md5']
file_path = request_params['file_path']
file_size = request_params['file_size']
ip_address = request.META.get('HTTP_X_REAL_IP') or request.META.get('HTTP_REMOTE_ADD')
# save file to tmp
new_file_name = '%s_%s' % (file_md5,file_name)
new_file_path = ''.join([UPLOAD_FILE_PATH, new_file_name])
with open(new_file_path, 'a') as new_file:
with open(file_path, 'rb') as f:
new_file.write(f.read())
content = json.dumps({
'name': file_name,
'content_type': file_content_type,
'md5': file_md5,
'path': file_path,
'size': file_size,
'ip': ip_address,
})
print(new_file_path)
path_dir = new_file_path.split('/')[2:]
res = '/'.join(path_dir)
download_url = '/'.join(('http://172.18.3.183:88',res))
response = HttpResponse(download_url, content_type='application/json; charset=utf-8')
return response实现效果图：
