node js批量导出PDF 导出压缩为zip 拆分设计 文件压缩 批量导出设计
node导出pdf网上有不少介绍的文章，之前我也有过这方面的需求，总结一下遇到的问题，给大家点参考建议。
导出PDF大概会遇到以下几个阶段：
1、导出单个PDF文件
2、导出多个PDF文件
下面着重说一下第二种情况，第一种情况比较简单，直接用phantomjs导出生成PDF即可。第二种情况，其实就是第一种情况的复杂化，我们先需要生成文件夹，在将生成的PDF放到文件夹中。这里有个问题，就是如果大数据同时导出生成会导致服务器内存打满，服务器挂掉，这种情况下必须得重启服务器才行，连登录都没法登录，死机了。我自己亲测的一个导出3页PDF的页面大概会占用20MB左右的内存，这个看你服务器多大了，我设置的是同时导出50个，50个成功后递归调用后续的，直到所有的都处理完。我把代码粘上面供大家参考一下，并且加上必要注释，同时上传项目到GIT上，有疑问的朋友可以留言一起学习探讨一下。GIT地址：https://github.com/seizeDev/node-pef
1、common.js
/**
* Created by lizhen on 2017/11/13.
*/
var commonJs = {};
exports.commonJs = commonJs;
//导出时生成的文件戳
commonJs.reserverTime = function () {
var thisDate = new Date();
var nameDate = thisDate.getFullYear().toString() + (thisDate.getMonth() + 1).toString() + thisDate.getDate().toString() + thisDate.getHours().toString() + thisDate.getMinutes().toString();
return nameDate
};
// 导出时生成的文件大小
commonJs.byteConversion = function (byte) {
try {
if (typeof byte == 'number') {
var calByte = byte;
var returnType = 'b';
if (calByte > 1024) {
calByte = calByte / 1024;
returnType = 'KB'
if (calByte>1024){
calByte = calByte / 1024;
returnType = 'MB'
if(calByte>1024){
calByte = calByte / 1024;
returnType = 'GB'
}
}
}
return calByte+returnType;
}
} catch (err) {
return err
}
};
// 解析传入的请求参数
commonJs.parseResult = function(body) {
var json = {};
try {
json = JSON.parse(body);
} catch (e) {
/*
Error(e);
console.log(Constant.ERROR, "解析返回数据出错！返回数据为：", body);
Res.end('{"code":500}');
return;*/
}
var rs = json;
return rs;
};
2、userMsg.js
/**
* Created by lizhen on 2017/11/13.
*/
var userMsg = {};
//导出文件存放的地址
var dirPath = "/Users/lizhen/Desktop/resurce/export_policy";
userMsg.dirPath = dirPath;
//导出文件的名字
userMsg.htmlMaps = {
'居间服务协议': 'intermediary_agreement',
}
//导出服务器信息
const productionEnv = {
hostname: '10.8.0.6',
port: 8180,
path: '/backstage/v1/sso/login'
}
userMsg.productionEnv = productionEnv;
//用户信息
//'lizhen_export', 'lizhenniubi', 'backstage_user'
userMsg.user = {
username:'lizhen_export',
pwd:'lizhenniubi',
type:'backstage_user'
}
//当前第几条
userMsg.total = 0;
// 导出返回给前端的下载路径
userMsg.exporttUrl = 'http://www.baidu.com';
userMsg.dataMsg = null;
userMsg.dataFile = null;
exports.userMsg = userMsg;
3、exportMethod.js
/**
* Created by lizhen on 2017/11/13.
*/
var exportMethod = {};
exportMethod.exportPdf = function () {
}
//登录成功后的回调生成PDF
var exec = require('child_process').exec;
var userMsg = require("../common/userMsg").userMsg;
var logger = require("../log4js/logHelper").helper;
var fs = require('fs');
exportMethod.loginCallback = function (data, order, pageHtml,yearRate , thisTime, cid ) {
return new Promise(function (resolve, reject) {
var productUrl = 'https://www.baidu.com';
var url = null;
if(!yearRate||yearRate === 'undefined'){
url = `phantomjs savePdf.js "${productUrl}${pageHtml}.html?sid=${data.__sid}×tamp=${data.timestamp}&signature=${data.signature}&orderNo=${order}&channel_id=${cid}" ${order} ${pageHtml} ${thisTime}`;
}else{
url = `phantomjs savePdf.js "${productUrl}${pageHtml}.html?sid=${data.__sid}×tamp=${data.timestamp}&signature=${data.signature}&orderNo=${order}&yearRate=${yearRate}&channel_id=${cid}" ${order} ${pageHtml} ${thisTime}`;
}
logger.writeInfo(url);
exec(url, {
encoding: 'utf-8',
timeout: 100000,
maxBuffer: 200 * 1024,
killSignal: 'SIGTERM',
cwd: null,
env: null
},
function (err, out) {
userMsg.total += 1;
var data = {
total:userMsg.total,
order
}
if (err) {
console.log(err);
data.success = false;
data.message = err;
resolve(data)
} else {
data.success = true;
data.message = '请求成功'
resolve(data)
}
});
})
}
/**
* 全部都导出成功后压缩整个文件夹
* @type {commonJs}
*/
var commonJs = require("../common/common").commonJs;
exportMethod.gzipPaf = function (dataMsg,thisName,thisTime) {
return new Promise(function (resolve, reject) {
if (dataMsg == null) {
dataMsg = '请求成功'
}
var fileName = thisName + thisTime;
var archiver = require('archiver');
//noinspection JSUnresolvedFunction
var output = fs.createWriteStream(userMsg.dirPath + '/' + fileName + '.zip');
//noinspection SpellCheckingInspection
var archive = archiver('zip', {
zlib: {level: 9} // Sets the compression level.
});
output.on('close', function () {
logger.writeInfo('导出成功:总大小' + commonJs.byteConversion(archive.pointer()));
//noinspection SpellCheckingInspection
var rsolveData = {
message: dataMsg,
name: `${userMsg.exporttUrl}/${thisName}${thisTime}.zip`
}
resolve(rsolveData)
});
//noinspection JSUnresolvedFunction
archive.on('warning', function (err) {
//noinspection SpellCheckingInspection
if (err.code === 'ENOENT') {
// log warning
} else {
// throw error
logger.writeErr('导出错误:' + err);
}
});
archive.on('error', function (err) {
logger.writeErr('导出失败:' + err);
});
archive.pipe(output);
var fsDir = userMsg.dirPath + '/' + fileName;
fs.readdirSync(fsDir).forEach(function (file) {
var pathname = userMsg.dirPath + '/' + fileName + '/' + file;
//noinspection JSUnresolvedFunction
archive.append(fs.createReadStream(pathname), {name: file});
});
//noinspection JSUnresolvedFunction
archive.finalize();
})
}
exportMethod.recursiveExport = function (orderList,config,thisName, rate, cid) {
return new Promise(function (resolve, reject) {
var thisTime = commonJs.reserverTime();
logger.writeInfo(cid);
/**
* 如果没有导出的文件夹，则创建文件夹
*/
if (!fs.existsSync(userMsg.dirPath + '/' + (thisName + thisTime))) {
fs.mkdirSync(userMsg.dirPath + '/' + (thisName + thisTime));
}
orderList.forEach((order) => {
exportMethod.loginCallback(config, order, thisName, rate, thisTime, cid).then(function (data) {
logger.writeInfo(data);
if (!data.success) {
userMsg.dataMsg += `${data.order}导出失败，请重新导出|`;
userMsg.dataFile += `${thisName} + ${thisTime}|`;
}
//导出成功！
if (data.total == orderList.length) {
exportMethod.gzipPaf(userMsg.dataMsg,thisName,thisTime).then(exportMsg => {
userMsg.total = 0;
userMsg.dataMsg = null;
userMsg.dataFile = null;
resolve(exportMsg);
})
}
});
})
})
};
exports.exportMethod = exportMethod;
4、savePdf.js
var system = require('system');
var args = system.args;
var url = args['1'];
var name = args['2'];
var filename = args['3'];
var filetime = args['4'];
var dirPath = "/Users/lizhen/Desktop/resurce/export_policy";
openPage(url);
function openPage(url) {
var page = require('webpage').create();
page.open(url, function (status) {
setTimeout(function () {
console.log(status)
if (status === "success") {
page.paperSize = {
format: 'A4',
orientation: 'portrait',
border: '1cm'
};
page.render(dirPath+'/'+(filename+filetime)+'/'+name + ".pdf");
} else {
console.log("Page failed to load.");
}
phantom.exit(0);
}, 3000);
});
}
5、logHelper.js
/**
* Created by lizhen on 2017/9/26.
*/
var helper = {};
exports.helper = helper;
var log4js = require('log4js');
// 目录创建完毕，才加载配置，不然会出异常
log4js.configure({
appenders: { cheese: { type: 'file', filename: 'cheese.log' } },
categories: { default: { appenders: ['cheese'], level: 'ALL' } }
});
var logDebug = log4js.getLogger('logDebug');
var logInfo = log4js.getLogger('logInfo');
var logWarn = log4js.getLogger('logWarn');
var logErr = log4js.getLogger('logErr');
helper.writeDebug = function (msg) {
if (msg == null)
msg = "";
logDebug.debug(msg);
};
helper.writeInfo = function (msg) {
if (msg == null)
msg = "";
console.log(msg)
logInfo.info(msg);
};
helper.writeWarn = function (msg) {
if (msg == null)
msg = "";
console.log(msg)
logWarn.warn(msg);
};
helper.writeErr = function (msg, exp) {
if (msg == null)
msg = "";
if (exp != null)
msg += "\r\n" + exp;
console.log(msg)
logErr.error(msg);
};
// 配合express用的方法
exports.use = function (app) {
//页面请求日志, level用auto时,默认级别是WARN
app.use(log4js.connectLogger(logInfo, {level: 'debug', format: ':method :url'}));
}
6、finlinkAgreement.js
// var http = require("http");
var https = require("https");
var fs = require("fs");
var md5 = require("md5");
var express = require('express');
var logger = require("./log4js/logHelper").helper;
var userMsg = require("./common/userMsg").userMsg;
var commonJs = require("./common/common").commonJs;
var exportMethod = require("./export/exportMethod").exportMethod;
var log = require('./log4js/logHelper');
var app = express();
log.use(app);
var exportHtml = null;
var yearRate = null;
var channelId = null;
var exportList = [];
var allList = [];
app.get('/getPdf', function (req, res) {
logger.writeInfo("开始记录日志");
userMsg.total = 0;
logger.writeInfo('/getPdf');
try {
allList = req.query.order_nos.split(',');
logger.writeInfo(allList);
} catch(error) {
res.send({
'code':100,
'message':'输入的列表分隔符错误！'
});
logger.writeErr(req.query.order_nos);
return;
}
exportHtml = req.query.html_name;
yearRate = req.query.year_rate;
channelId = req.query.channelId;
logger.writeInfo(exportHtml);
post(userMsg.user.username,userMsg.user.pwd,userMsg.user.type).then(function (result) {
logger.writeInfo(result);
res.send(result);
})
});
var globalMsg = [];
function post(username, pwd, type) {
return new Promise(function (resolve, reject) {
var params = {
username: username,
password: pwd,
type: type
};
var options = {
hostname: userMsg.productionEnv.hostname,
port: userMsg.productionEnv.port,
path: userMsg.productionEnv.path,
method: 'POST',
headers: {
"Content-Type": 'application/json;charset=utf-8'
}
};
//登录
var req = https.request(options, function (res) {
var body = "";
res.setEncoding('utf-8');
res.on('data', function (chunk) {
body += chunk;
}).on("end", function () {
var data = commonJs.parseResult(body);
var config = {};
var timestamp = (new Date()).valueOf();
config.__sid = data.data.__sid;
config.timestamp = timestamp;
config.signature = md5(data.data.key + timestamp.toString());
/**
* 根据用户传入的HTML名字来判断导出哪个文件，如果没有的话，则默认为intermediary_agreement
* @type {string}
*/
var thisName = exportHtml ? userMsg.htmlMaps[exportHtml] : 'intermediary_agreement';
function listTraverse(all,data,name,rate,cid) {
var angentList = all.splice(0, 50);
exportMethod.recursiveExport(angentList, data, name, rate, cid).then((exportMsg) => {
globalMsg.push(exportMsg);
logger.writeInfo(exportMsg);
if (all.length > 50) {
listTraverse(all, data, name)
} else if (all.length > 0 && all.length <= 50) {
listTraverse(all, data, name);
all = [];
} else {
resolve(globalMsg)
globalMsg = [];
}
})
}
listTraverse(allList,config,thisName, yearRate, channelId);
});
});
req.write(JSON.stringify(params));
req.end();
})
}
app.listen(8089, function () {
console.log('监听8089')
});
7、fileCopy.js
var fs=require('fs');
var exec = require('child_process').exec;
var args = process.argv;
var filePath, dirPath, outputdir;
while(args.length){
switch(args.shift()){
case "-f":
case "--file":
filePath = args.shift();
break;
case "-d":
case "--dir":
dirPath = args.shift();
break;
case "-od":
case "--outputdir":
outputdir = args.shift();
break;
}
}
var dirs = fs.readdirSync(dirPath);
var text = fs.readFileSync(filePath, 'UTF-8').match(/\S+/g);
dirs.forEach(function(dir){
for(var i=0, len=text.length;i