转载请注明：
原始地址：https://www.jianshu.com/p/5d77641b17b2
原作者：wonder
1、写在前面
前两篇（《owt-server 的集群管理者、集群工作站、消息队列（一）》、《owt-server 的集群管理者、集群工作站、消息队列（二）》）分别主要介绍了owt-server的 clusterManager 模块和 amqp_client 模块。clusterManager 模块是用于集群管理、调度的模块；amqp_client 模块是用于封装消息队列、为上层提供 rpc 接口的模块。
本文将介绍 clusterWorker 模块，该模块是集群工作站模块，clusterWorker是受 clusterManager 模块管理的对象。
除此以外，本篇还将：
1）介绍owt-server统计一个worker的硬件负载情况的策略
2）举一个栗子来说明一个clusterWorker的使用方式。
2、clusterWorker 模块分解
clusterWorker 模块位于owt-server源码目录下common/clusterWorker.js文件中。
var genID = (function() { //产生随机ID的函数
function s4() {
return Math.floor((1 + Math.random()) * 0x10000)
.toString(16)
.substring(1);
}
return function() {
return s4() + s4()
/*+ '-' + s4()
+ '-' + s4()
+ '-' + s4()
+ '-'*/ + s4() + s4() + s4();
};
})();
module.exports = function (spec) { //导出内容
var that = {};
/*'unregistered' | 'registered' | 'recovering'*/ //状态类型
var state = 'unregistered',
tasks = [];
var rpcClient = spec.rpcClient, //rpc实例
id = spec.purpose + '-' + genID() + '@' + (spec.info.hostname || spec.info.ip), //该clusterWorker的ID
purpose = spec.purpose, //该clusterWorker承载的应用类型
info = spec.info, //相关配置
cluster_name = spec.clusterName || 'owt-cluster', //该clusterWorker的名称
join_retry = spec.joinRetry || 60, //加入cluster的重试次数
keep_alive_period = 800/*MS*/, //保活时长
keep_alive_interval = undefined, //保活间隔
on_join_ok = spec.onJoinOK || function () {log.debug('Join cluster successfully.');}, //加入cluster成功时的回调函数
on_join_failed = spec.onJoinFailed || function (reason) {log.debug('Join cluster failed. reason:', reason);}, //加入失败时的回调函数
on_loss = spec.onLoss || function () {log.debug('Lost connection with cluster manager');}, //与cluster manager失联时的回调函数
on_recovery = spec.onRecovery || function () {log.debug('Rejoin cluster successfully.');}, //重新加入cluster的回调函数
on_overload = spec.onOverload || function () {log.debug('Overloaded!!');};; //该clusterWorker过载时的回调函数
var previous_load = 0.99; //当前负载，初始为0.99
var reportLoad = function (load) { //报告当前worker负载情况到manager
if (load == previous_load) { //与上次负载相同，不报告
return;
}
previous_load = load;
if (state === 'registered') {
rpcClient.remoteCast( //向cluster发送负载报告
cluster_name,
'reportLoad',
[id, load]);
}
if (load > 0.98/*FIXME: Introduce a configuration item to specify the overload threshold here.*/) {
on_overload(); //过载处理函数
}
};
var load_collector = loadCollector({period: spec.loadCollection.period, //负载统计间隔
item: spec.loadCollection.item, //指定负载收集依据（CPU、Network、GPU、Memory等）
onLoad: reportLoad}); //创建负载收集器，并制定发送负载报告的函数
var join = function (on_ok, on_failed) { //执行一次加入cluster的rpc过程
makeRPC( //执行一次rpc
rpcClient, //rpc实例
cluster_name, // cluster的名称
'join',
[purpose, id, info], // 消息包括：应用类型、该worker的id、配置信息
function (result) { //加入cluster成功的回调函数
state = 'registered'; //设置worker当前状态为“registered”
on_ok(result);
previous_load = 0.99; //设置初始负载
keepAlive(); //执行保活函数
}, on_failed);
};
var joinCluster = function (attempt) { //加入cluster
var countDown = attempt; //尝试次数
var tryJoin = function (countDown) {
log.debug('Try joining cluster', cluster_name, ', retry count:', attempt - countDown);
join(function (result) { //执行 join 的rpc
on_join_ok(id);
log.info('Join cluster', cluster_name, 'OK.');
}, function (error_reason) { //加入失败的回调函数
if (state === 'unregistered') {
log.info('Join cluster', cluster_name, 'failed.');
if (countDown <= 0) {
log.error('Join cluster', cluster_name, 'failed. reason:', error_reason);
on_join_failed(error_reason);
} else {
tryJoin(countDown - 1); //重试
}
}
});
};
tryJoin(attempt); //尝试加入，尝试若干次
};
var keepAlive = function () { //保活函数
keep_alive_interval && clearInterval(keep_alive_interval); //清空计时器
var tryRecovery = function (on_success) { //尝试恢复函数
clearInterval(keep_alive_interval); //清空计时
keep_alive_interval = undefined;
state = 'recovering'; //状态置为“recovering”
var tryJoining = function () { //尝试加入
log.debug('Try rejoining cluster', cluster_name, '....');
join(function (result) { //执行一次join rpc过程
log.debug('Rejoining result', result);
if (result === 'initializing') { // manager是正处于始化状态
tasks.length > 0 && pickUpTasks(tasks); //该worker向manager申请记录worker上的所有任务
} else { //manager是其他状态
on_loss(); //worker掉线，处理掉线处理函数
tasks = []; //任务置为空
}
on_success(); //与manager成功重连
}, function (reason) { //重连失败
if (state === 'recovering') { //该worker处于重连状态
log.debug('Rejoin cluster', cluster_name, 'failed. reason:', reason);
tryJoining(); //再次尝试加入cluster
}
});
};
tryJoining(); //开始尝试加入cluster
};
var loss_count = 0;
keep_alive_interval = setInterval(function () { //保活计时器启动
makeRPC( //执行保活rpc
rpcClient,
cluster_name,
'keepAlive',
[id],
function (result) {
loss_count = 0;
if (result === 'whoareyou') { //manager未记录该worker
if (state !== 'recovering') { //该worker不处于“recovering”时，需重新加入
log.info('Unknown by cluster manager', cluster_name);
tryRecovery(function () {
log.info('Rejoin cluster', cluster_name, 'OK.');
});
}
}
}, function (error_reason) { //保活失败
loss_count += 1;
if (loss_count > 3) { //保活失败超过3次
if (state !== 'recovering') { //该worker不处于“recovering”时，需重新加入
log.info('Lost connection with cluster', cluster_name);
tryRecovery(function () {
log.info('Rejoin cluster', cluster_name, 'OK.');
on_recovery(id);
});
}
}
});
}, keep_alive_period); //保活间隔
};
var pickUpTasks = function (taskList) { //通知manager：该worker正在执行某些任务
rpcClient.remoteCast(
cluster_name,
'pickUpTasks',
[id, taskList]);
};
var layDownTask = function (task) { //通知manager：该worker结束某些任务
rpcClient.remoteCast(
cluster_name,
'layDownTask',
[id, task]);
};
var doRejectTask = function (task) { //通知manager：该worker拒绝某些任务
rpcClient.remoteCast(
cluster_name,
'unschedule',
[id, task]);
};
that.quit = function () { //导出函数：退出
if (state === 'registered') {
if (keep_alive_interval) {
clearInterval(keep_alive_interval); //清理保活计时器
keep_alive_interval = undefined;
}
rpcClient.remoteCast( //通知manager：该worker退出
cluster_name,
'quit',
[id]);
} else if (state === 'recovering') {
keep_alive_interval && clearInterval(keep_alive_interval); //清理保活计时器
}
load_collector && load_collector.stop(); //停止该worker的负载统计
};
that.reportState = function (st) { //导出函数：状态汇报
if (state === 'registered') {
rpcClient.remoteCast( //通知mananger：该worker的状态
cluster_name,
'reportState',
[id, st]);
}
};
that.addTask = function (task) { //导出函数：添加任务
var i = tasks.indexOf(task);
if (i === -1) { //向该worker任务列表增加新任务
tasks.push(task);
if (state === 'registered') { //向manager通知开始执该任务
pickUpTasks([task]);
}
}
};
that.removeTask = function (task) { //导出函数：移除某任务
var i = tasks.indexOf(task);
if (i !== -1) { //从worker任务列表中移除该任务
tasks.splice(i, 1);
if (state === 'registered') { //向manager通知移除该任务
layDownTask(task);
}
}
};
that.rejectTask = function (task) { //拒绝该任务
doRejectTask(task); //向manager通知，拒绝该任务
};
joinCluster(join_retry); //开始加入cluster集群
return that;
};
3、loadCollector模块分解
在第2节中，clusterWorker 需要向manager 汇报自身硬件的负载情况。这里就对 负载统计模块（loadCollector ）进行分解，以形成一个直观的认识。该模块位于owt-server源码目录下的common/loadCollector.js文件
var child_process = require('child_process'); //引用 node.js 的子进程模块
var os = require('os'); //引用 node.js 的操作系统模块
var cpuCollector = function (period, onLoad) { //基于CPU进行负载统计
var olds = os.cpus(); //初始时，获取逻辑CPU内核的信息列表
var begin = 0;
var end = olds.length - 1;
var interval = setInterval(function() { //启动计时器
var cpus = os.cpus(); //获取当前逻辑CPU内核的信息列表
var idle = 0;
var total = 0;
for (let i = begin; i <= end; i++) { //逐个CPU计算其负载
for (let key in cpus[i].times) { //统计没在模式下CPU花费的ms数，共[user , nice , sys, idle, irq ] 5种模式
let diff = cpus[i].times[key] - olds[i].times[key];
if (key === 'idle') { //累计该CPU在 “空闲” 状态下的耗时
idle += diff;
}
total += diff; //累计该CPU该时段的总耗时
}
}
olds = cpus; //记录当前各CPU状态
onLoad(1 - idle/total); //使用所有CPU的 “1-空闲比” 作为当前负载情况进行汇报
log.debug('cpu usage:', 1 - idle/total);
}, period); //计时器触发间隔
this.stop = function () { //停止统计
log.debug("To stop cpu load collector.");
clearInterval(interval);
};
};
var memCollector = function (period, onLoad) { //基于Memory进行负载统计
var interval = setInterval(function() { //启动计时器
var usage = 1 - os.freemem() / os.totalmem(); //内存负载为： 1-空闲比
onLoad(usage); //汇报Memeory负载
log.debug('mem usage:', usage);
}, period);
this.stop = function () { //停止计时器
log.debug("To mem cpu load collector.");
clearInterval(interval);
};
};
var diskCollector = function (period, drive, on_load) { //基于 Disk 进行负载统计
var interval = setInterval(function () {
var total = 1, free = 0;
child_process.exec("df -k '" + drive.replace(/'/g,"'\\''") + "'", function(err, stdout, stderr) { //在指定目录执行 df -k
if (err) {
log.error(stderr);
} else {
var lines = stdout.trim().split('\n');
var str_disk_info = lines[lines.length - 1].replace( /[\s\n\r]+/g,' ');
var disk_info = str_disk_info.split(' ');
total = disk_info[1];
free = disk_info[3];
on_load(Math.round((1.0 - free / total) * 1000) / 1000); //汇报
}
});
}, period);
this.stop = function () {
log.debug("To stop disk load collector.");
clearInterval(interval);
};
};
var networkCollector = function (period, interf, max_scale, on_load) { //使用 Network 进行负载统计
var rx_Mbps = 0, tx_Mbps = 0, rx_bytes = 0, tx_bytes = 0;
var meter = setInterval(function () { //启动定时器，1s统计一下网络收发情况
child_process.exec("awk 'NR>2{if (index($1, \"" + interf + "\")==1){print $2, $10}}' /proc/net/dev", function (err, stdout, stderr) { //在 网口interf 上执行该awk语句，从proc/net/dev抽取出“接收bytes” 和 “发送bytes”
if (err) {
log.error(stderr);
} else {
var fields = stdout.trim().split(" ");
if (fields.length < 2) {
return log.warn('not ordinary network load data');
}
var rx = Number(fields[0]), tx = Number(fields[1]); //分别赋值接收字节 和 发送字节
if (rx >= rx_bytes && rx_bytes > 0) { //转换单位为Mbps
rx_Mbps = Math.round(((rx - rx_bytes) * 8 / 1048576) * 1000) / 1000;
}
if (tx >= tx_bytes && tx_bytes > 0) { //转换单位为Mbps
tx_Mbps = Math.round(((tx - tx_bytes) * 8 / 1048576) * 1000) / 1000;
}
rx_bytes = rx;
tx_bytes = tx;
}
});
}, 1000); //1s统计一次
var reporter = setInterval(function () { //启动负载汇报定时器
var rt_load = Math.round(Math.max(rx_Mbps / max_scale, tx_Mbps / max_scale) * 1000) / 1000; //计算收发速率与基准速率比例
on_load(rt_load); //汇报负载
}, period); //汇报间隔
this.stop = function () { //停止统计
log.debug("To stop network load collector.");
meter && clearInterval(meter);
reporter && clearInterval(reporter);
meter = undefined;
reporter = undefined;
};
};
var gpuCollector = function (period, on_load) { //基于 Gpu 进行负载统计
var child = child_process.exec('stdbuf -o0 metrics_monitor 100 1000'); //这一条使用了子进程 metrics_monitor 进行资源统计
var cpu_load = 0,
cpu_collector = new cpuCollector(period, function (data) {cpu_load = data;}); //启用一个CPU负载统计
var load = 0;
child.stdout.on('data', function (data) { //当上述命令得到结果时
var usage_sum = 0, samples = 0;
var lines = data.toString().split('\n');
var i = lines.length > 10 ? lines.length - 10 : 0;
for (; i < lines.length; i++) { //处理每行
var engine_list = lines[i].split('\t');
var engine_max_usage = 0;
for (var engine of engine_list) { //处理每个引擎
var m = null;
if ((m = engine.match(/\s+usage:\s+(\d+\.\d+)/)) && m !== null && m.length > 1) { //匹配到“usage字段”
var engine_usage = Number(m[1]); //获取使用率
if (engine_max_usage < engine_usage) //更新最大使用率
engine_max_usage = engine_usage;
}
}
usage_sum = usage_sum + engine_max_usage; //累计最大使用率
samples = samples + 1; //采样数增加
}
if (samples > 0)
load = (usage_sum / samples) / 100; //平均最大使用率
else
load = 0;
});
var interval = setInterval(function () { //启动汇报计时器
var result = Math.max(load, cpu_load); //报告CPU 和 GPU中最大的使用负载
on_load(result);
}, period);
this.stop = function () { //停止计时器
log.debug("To stop gpu load collector.");
cpu_collector && cpu_collector.stop();
cpu_collector = undefined;
child && child.kill(); //关闭子进程
child = undefined;
interval && clearInterval(interval);
interval = undefined;
};
};
exports.LoadCollector = function (spec) { //导出负载统计模块
var that = {};
var period = spec.period || 1000, //统计间隔
item = spec.item, //统计策略
on_load = spec.onLoad || function (load) {log.debug('Got', item.name, 'load:', load);}, //回调函数
collector = undefined;
that.stop = function () { //停止统计
log.info("To stop load collector.");
collector && collector.stop();
collector = undefined;
};
switch (item.name) {
case 'network':
collector = new networkCollector(period, item.interf, item.max_scale, on_load);
break;
case 'cpu':
collector = new cpuCollector(period, on_load);
break;
case 'gpu':
collector = new gpuCollector(period, on_load);
break;
case 'memory':
collector = new memCollector(period, on_load);
break;
case 'disk':
collector = new diskCollector(period, item.drive, on_load);
break;
default:
log.error('Unknown load item');
return undefined;
//break;
}
return that;
};
4、makeRPC 模块
第2节中出现了makeRPC函数，该模makeRPC块位于该模块位于owt-server源码目录下的common/makeRPC.js文件。仅仅是对amqp_client模块封装了一个响应处理函数，用于代处理“error” 、“timeout” 和 正常消息。
exports.makeRPC = function (rpcClient, remote_node, remote_function, parameters_list, on_ok, on_error) {
rpcClient.remoteCall(
remote_node,
remote_function,
parameters_list,
{callback: function (result, error_reason) {
if (result === 'error') {
typeof on_error === 'function' && on_error(error_reason);
} else if (result === 'timeout') {
typeof on_error === 'function' && on_error('Timeout to make rpc to ' + remote_node + '.' + remote_function);
} else {
typeof on_ok === 'function' && on_ok(result);
}
}}
);
};
5、makeRPC 模块