一.前言
大家都知道nodejs是一个单进程单线程的服务器引擎,不管有多么的强大硬件,只能利用到单个CPU进行计算。所以,有人开发了第三方的cluster,让node可以利用多核CPU实现并行。
随着nodejs的发展,让nodejs上生产环境,就必须是支持多进程多核处理!在V0.6.0版本,Nodejs内置了cluster的特性。自此,Nodejs终于可以作为一个独立的应用开发解决方案,映入大家眼帘了。
最简单的例子:
var cluster = require('cluster'); var http = require('http'); var numCPUs = require('os').cpus().length; if (cluster.isMaster) { console.log("master start..."); // Fork workers. for (var i = 0; i < numCPUs; i++) { cluster.fork(); } cluster.on('listening',function(worker,address){ console.log('listening: worker ' + worker.process.pid +', Address: '+address.address+":"+address.port); }); cluster.on('exit', function(worker, code, signal) { console.log('worker ' + worker.process.pid + ' died'); }); } else { http.createServer(function(req, res) { res.writeHead(200); res.end("hello world\n"); }).listen(0); }
二.cluster的工作原理
每个worker进程通过使用child_process.fork()函数,基于IPC(Inter-Process Communication,进程间通信),实现与master进程间通信。
当worker使用server.listen(...)函数时 ,将参数序列传递给master进程。如果master进程已经匹配workers,会将传递句柄给工人。如果master没有匹配好worker,那么会创建一个worker,再传递并句柄传递给worker。
在边界条件,有3个有趣的行为:
注:下面server.listen(),是对底层“http.Server-->net.Server”类的调用。
- 1. server.listen({fd: 7}):在master和worker通信过程,通过传递文件,master会监听“文件描述为7”,而不是传递“文件描述为7”的引用。
- 2. server.listen(handle):master和worker通信过程,通过handle函数进行通信,而不用进程联系
- 3. server.listen(0):在master和worker通信过程,集群中的worker会打开一个随机端口共用,通过socket通信,像上例中的57132
当多个进程都在 accept() 同样的资源的时候,操作系统的负载均衡非常高效。Node.js没有路由逻辑,worker之间没有共享状态。所以,程序要设计得简单一些,比如基于内存的session。
因为workers都是独力运行的,根据程序的需要,它们可以被独立删除或者重启,worker并不相互影响。只要还有workers存活,则master将继续接收连接。Node不会自动维护workers的数目。我们可以建立自己的连接池。
三. cluster的API
官网地址:http://nodejs.org/api/cluster.html#cluster_cluster
cluster对象
cluster的各种属性和函数
cluster.setttings:配置集群参数对象
cluster.isMaster:判断是不是master节点
cluster.isWorker:判断是不是worker节点
Event: 'fork': 监听创建worker进程事件
Event: 'online': 监听worker创建成功事件
Event: 'listening': 监听worker向master状态事件
Event: 'disconnect': 监听worker断线事件
Event: 'exit': 监听worker退出事件
Event: 'setup': 监听setupMaster事件
cluster.setupMaster([settings]): 设置集群参数
cluster.fork([env]): 创建worker进程
cluster.disconnect([callback]): 关闭worket进程
cluster.worker: 获得当前的worker对象
cluster.workers: 获得集群中所有存活的worker对象
worker对象
worker的各种属性和函数:可以通过cluster.workers, cluster.worket获得。
worker.id: 进程ID号
worker.process: ChildProcess对象
worker.suicide: 在disconnect()后,判断worker是否自杀
worker.send(message, [sendHandle]): master给worker发送消息。注:worker给发master发送消息要用process.send(message)
worker.kill([signal='SIGTERM']): 杀死指定的worker,别名destory()
worker.disconnect(): 断开worker连接,让worker自杀
Event: 'message': 监听master和worker的message事件
Event: 'online': 监听指定的worker创建成功事件
Event: 'listening': 监听master向worker状态事件
Event: 'disconnect': 监听worker断线事件
Event: 'exit': 监听worker退出事件
四.master和worker的通信
var cluster = require('cluster'); var http = require('http'); var numCPUs = require('os').cpus().length; if (cluster.isMaster) { console.log('[master] ' + "start master..."); for (var i = 0; i < numCPUs; i++) { var wk = cluster.fork(); wk.send('[master] ' + 'hi worker' + wk.id); } cluster.on('fork', function (worker) { console.log('[master] ' + 'fork: worker' + worker.id); }); cluster.on('online', function (worker) { console.log('[master] ' + 'online: worker' + worker.id); }); cluster.on('listening', function (worker, address) { console.log('[master] ' + 'listening: worker' + worker.id + ',pid:' + worker.process.pid + ', Address:' + address.address + ":" + address.port); }); cluster.on('disconnect', function (worker) { console.log('[master] ' + 'disconnect: worker' + worker.id); }); cluster.on('exit', function (worker, code, signal) { console.log('[master] ' + 'exit worker' + worker.id + ' died'); }); function eachWorker(callback) { for (var id in cluster.workers) { callback(cluster.workers[id]); } } setTimeout(function () { eachWorker(function (worker) { worker.send('[master] ' + 'send message to worker' + worker.id); }); }, 3000); Object.keys(cluster.workers).forEach(function(id) { cluster.workers[id].on('message', function(msg){ console.log('[master] ' + 'message ' + msg); }); }); } else if (cluster.isWorker) { console.log('[worker] ' + "start worker ..." + cluster.worker.id); process.on('message', function(msg) { console.log('[worker] '+msg); process.send('[worker] worker'+cluster.worker.id+' received!'); }); http.createServer(function (req, res) { res.writeHead(200, {"content-type": "text/html"}); res.end('worker'+cluster.worker.id+',PID:'+process.pid); }).listen(3000); }
五.从Redis队列中取数据,并将其写入到Mysql中
var cluster = require('cluster'); var http = require('http'); var numCPUs = require('os').cpus().length; var queueHost = '192.168.235.60'; var queuePort = 6379; var dataConnection = {}; dataConnection.host = 'localhost'; dataConnection.user = 'root'; dataConnection.password = ''; dataConnection.database = 'livedb'; if (cluster.isMaster) { for (var i = 0; i < numCPUs; i++) { cluster.fork(); } cluster.on('fork', function (worker) { console.log('[master] Create worker. pid: '+worker.process.pid); }); cluster.on('online', function (worker) { console.log('[master] Worker online. pid: ' +worker.process.pid); }); cluster.on('disconnect', function (worker) { console.log('[master] ' + 'disconnect: worker: ' + worker.process.pid); }); cluster.on('exit', function (worker, code, signal) { console.log('[master] ' + 'exit worker: ' + worker.process.pid + ' died'); }); function eachWorker(callback) { for (var id in cluster.workers) { callback(cluster.workers[id]); } } /* //send to child setTimeout(function () { eachWorker(function (worker) { worker.send('[master] ' + 'send message to worker' + worker.id); }); }, 3000); */ Object.keys(cluster.workers).forEach(function(id) { cluster.workers[id].on('message', function(message){ if(message.act === 'suicide'){ //cluster.fork(); } }); }); } else if (cluster.isWorker) { process.on('message', function(msg) { //send to parent //process.send('[worker] worker'+cluster.worker.id+' received!'); }); var redis = require("redis"), client = redis.createClient(queuePort, queueHost); client.on("error", function (err){ console.log("[process] Error " + err); }); var mysql = require('mysql'); var connection = mysql.createConnection(dataConnection); connection.connect(); process.on('uncaughtException', function(err){ console.log("[process] UncaughtException: " + err); connection.end(); process.send({act:'suicide'}); process.exit(1); }); setInterval(function(){ client.rpop("message",function (err,value){ if(value!=null){ //console.log("[process] Handler" + process.pid + ": " + value); var obj = JSON.parse(value); var account = {accountid: obj.time, openid: obj.stamp}; console.log(account); var query = connection.query('INSERT INTO pre_qqaccount SET ?', account, function(err, result) { if(err){ console.log('[process] insert database error: '+err); } }); } }); },1000); }