| 1 | | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ |
| 2 | | |
| 3 | 2 | var fs = require('fs'); |
| 4 | 2 | var util = require('util'); |
| 5 | 2 | var Emitter = require('events').EventEmitter; |
| 6 | 2 | var child_process = require('child_process'); |
| 7 | | |
| 8 | 2 | var common = require('./common'); |
| 9 | 2 | var STATUS = common.STATUS; |
| 10 | 2 | var MESSAGE = common.MESSAGE; |
| 11 | 2 | var debug = common.debug; |
| 12 | 2 | var listen = common.listen; |
| 13 | 2 | var MAX_HEARTBEAT_INTERVAL = common.MAX_HEARTBEAT_INTERVAL; |
| 14 | | |
| 15 | 2 | var NOTICE = function (name, message) { |
| 16 | 4 | debug('master', Array.prototype.join.call(arguments, ' ')); |
| 17 | | }; |
| 18 | | |
| 19 | 2 | var WRITE_STATUS_FILE = function (name, pid, message) { |
| 20 | | }; |
| 21 | | |
| 22 | | /* {{{ private function _normalize() */ |
| 23 | 2 | var _normalize = function (name) { |
| 24 | 5 | return name.toString().trim().toLowerCase(); |
| 25 | | }; |
| 26 | | /* }}} */ |
| 27 | | |
| 28 | 2 | var _workerPair = function (argv, options, name) { |
| 29 | | |
| 30 | | /** |
| 31 | | * @ _options |
| 32 | | */ |
| 33 | 3 | var _options = { |
| 34 | | 'listen' : [], |
| 35 | | 'children' : require('os').cpus().length, |
| 36 | | 'max_request' : 0, |
| 37 | | 'max_fatal_restart': 5, |
| 38 | | }; |
| 39 | 3 | var groupName = name; |
| 40 | | |
| 41 | | /* {{{ options rewrite */ |
| 42 | | |
| 43 | 3 | for (var k in options) { |
| 44 | 6 | var value = options[k]; |
| 45 | 6 | switch (k) { |
| 46 | | case 'listen': |
| 47 | 3 | _options[k] = Array.isArray(value) ? value : [value]; |
| 48 | 3 | break; |
| 49 | | default: |
| 50 | 3 | _options[k] = value; |
| 51 | 3 | break; |
| 52 | | } |
| 53 | | } |
| 54 | | /* }}} */ |
| 55 | | |
| 56 | | /** |
| 57 | | * @ 请求队列 |
| 58 | | */ |
| 59 | 3 | var _fdqueue = []; |
| 60 | | |
| 61 | | /** |
| 62 | | * @ 子进程列表 |
| 63 | | */ |
| 64 | 3 | var _pobject = []; |
| 65 | | |
| 66 | | /** |
| 67 | | * @ 子进程状态 |
| 68 | | */ |
| 69 | 3 | var _pstatus = {}; |
| 70 | | |
| 71 | 3 | var _killall = function (signal) { |
| 72 | 6 | _pobject.forEach(function (sub) { |
| 73 | 12 | try { |
| 74 | 12 | sub.kill(signal || 'SIGTERM'); |
| 75 | | } catch (e) { |
| 76 | | } |
| 77 | | }); |
| 78 | 6 | _pobject = []; |
| 79 | | }; |
| 80 | | |
| 81 | | /** |
| 82 | | * @ fatal restarts |
| 83 | | */ |
| 84 | 3 | var _fatals = []; |
| 85 | | |
| 86 | | /* {{{ function _newChild() */ |
| 87 | | /** |
| 88 | | * fork a new child process |
| 89 | | * @return {Object} child_process |
| 90 | | */ |
| 91 | 3 | var _command = argv.join(' '); |
| 92 | 3 | var _execpath = argv.shift(); |
| 93 | 3 | var _newChild = function () { |
| 94 | | |
| 95 | 34 | var sub = child_process.fork(_execpath, argv, { |
| 96 | | 'cwd' : process.cwd(), |
| 97 | | 'env' : process.env, |
| 98 | | }); |
| 99 | 34 | var pid = sub.pid; |
| 100 | | |
| 101 | 34 | _pobject.push(sub); |
| 102 | 34 | _pstatus[pid] = { |
| 103 | | 'status' : STATUS.UNKNOWN, |
| 104 | | }; |
| 105 | | |
| 106 | 34 | NOTICE(util.format('new worker forked (%d) as "%s"', pid, _command)); |
| 107 | | |
| 108 | | /** |
| 109 | | * @ on exit |
| 110 | | */ |
| 111 | | /* {{{ */ |
| 112 | 34 | sub.on('exit', function (code, signal) { |
| 113 | 16 | var _objects = []; |
| 114 | 16 | _pobject.forEach(function (item) { |
| 115 | 69 | if (item.pid !== pid) { |
| 116 | 55 | _objects.push(item); |
| 117 | | } |
| 118 | | }); |
| 119 | 16 | _pobject = _objects; |
| 120 | 16 | delete _pstatus[pid]; |
| 121 | | |
| 122 | 16 | if (STATUS.STOPING === _wstatus) { |
| 123 | 2 | return; |
| 124 | | } |
| 125 | | |
| 126 | 14 | if (code && 'SIGKILL' !== signal) { |
| 127 | 6 | var now = Date.now(); |
| 128 | 6 | if (_fatals.unshift(now) > _options.max_fatal_restart) { |
| 129 | 1 | _fatals = _fatals.slice(0, _options.max_fatal_restart); |
| 130 | | } |
| 131 | | |
| 132 | 6 | if (_fatals.length >= _options.max_fatal_restart && _fatals[_fatals.length - 1] + 60000 >= now) { |
| 133 | 2 | NOTICE('max fatal restarts (' + _options.max_fatal_restart + ') arrived, give up to fork'); |
| 134 | | |
| 135 | 2 | __GLOBAL_MASTER.emit('giveup', groupName, _fatals.length); |
| 136 | | // after 1 minute, start again |
| 137 | 2 | setTimeout(function () { |
| 138 | 0 | _me.start(); |
| 139 | | }, 60100); |
| 140 | | |
| 141 | 2 | return; |
| 142 | | } |
| 143 | | } |
| 144 | 12 | _me.start(); |
| 145 | | }); |
| 146 | | /* }}} */ |
| 147 | | |
| 148 | | /** |
| 149 | | * @on message |
| 150 | | */ |
| 151 | | /* {{{ */ |
| 152 | 34 | var _send = function (type, data, handle, port) { |
| 153 | 6 | try { |
| 154 | 6 | sub.send({ |
| 155 | | 'type' : type, |
| 156 | | 'data' : data, |
| 157 | | 'port' : port |
| 158 | | }, handle); |
| 159 | | } catch (e) { |
| 160 | 0 | NOTICE('SEND', e.stack); |
| 161 | | } |
| 162 | | }; |
| 163 | | |
| 164 | 34 | sub.on('message', function (msg) { |
| 165 | 43 | if (!msg || !msg.type) { |
| 166 | 16 | return; |
| 167 | | } |
| 168 | | |
| 169 | 27 | switch (msg.type) { |
| 170 | | case MESSAGE.GET_FD: |
| 171 | 6 | var _item = _fdqueue.shift(); |
| 172 | 6 | if (!_item) { |
| 173 | 2 | _send(MESSAGE.REQ_FD, 0); |
| 174 | 2 | break; |
| 175 | | } |
| 176 | 4 | var _fd = _item[1]; |
| 177 | 4 | _send(MESSAGE.REQ_FD, _fdqueue.length, _fd, _item[0]); |
| 178 | 4 | _fd.close(); |
| 179 | 4 | _fd = null; |
| 180 | 4 | break; |
| 181 | | |
| 182 | | case MESSAGE.STATUS: |
| 183 | 13 | var pstat = _pstatus[pid] = msg.data; |
| 184 | 13 | pstat._time = Date.now(); |
| 185 | 13 | WRITE_STATUS_FILE(groupName, pid, pstat); |
| 186 | 13 | _check_stat_change(); |
| 187 | 13 | if (_options.max_request && pstat.scores > _options.max_request) { |
| 188 | 4 | _newChild(); |
| 189 | 4 | _tobekill.push(pid); |
| 190 | | } |
| 191 | 13 | break; |
| 192 | | |
| 193 | | case MESSAGE.RELOAD: |
| 194 | 1 | if (msg.data) { |
| 195 | 1 | var gname = _normalize(msg.data.name || groupName); |
| 196 | 1 | if (__WORKERS_LIST[gname]) { |
| 197 | 1 | __WORKERS_LIST[gname].reload(); |
| 198 | | } |
| 199 | | } |
| 200 | 1 | break; |
| 201 | | |
| 202 | | case MESSAGE.SENDTO: |
| 203 | 3 | if (msg.data && msg.data.name && msg.data.data) { |
| 204 | 1 | var gname = _normalize(msg.data.name); |
| 205 | 1 | if (__WORKERS_LIST[gname]) { |
| 206 | | // TODO: gname or groupName? |
| 207 | 1 | __WORKERS_LIST[gname].accept(msg.data.data, groupName, pid); |
| 208 | | } |
| 209 | | } |
| 210 | 3 | break; |
| 211 | | |
| 212 | | default: |
| 213 | 4 | break; |
| 214 | | } |
| 215 | | }); |
| 216 | | /* }}} */ |
| 217 | | |
| 218 | 34 | return sub; |
| 219 | | |
| 220 | | }; |
| 221 | | /* }}} */ |
| 222 | | |
| 223 | 3 | var _running = 0; |
| 224 | | |
| 225 | | /* {{{ */ |
| 226 | 3 | var _check_stat_change = function () { |
| 227 | 13 | var _floor = _options.children / 2; |
| 228 | 13 | var _count = 0; |
| 229 | 13 | for (var i in _pstatus) { |
| 230 | 53 | var _stat = _pstatus[i]; |
| 231 | 53 | if (STATUS.RUNNING === _stat.status /** && _stat._time >= ? */) { |
| 232 | 31 | _count++; |
| 233 | | } |
| 234 | | } |
| 235 | 13 | if (_count !== _running) { |
| 236 | 9 | __GLOBAL_MASTER.emit('state', groupName, _count, _running); |
| 237 | | } |
| 238 | | |
| 239 | 13 | _running = _count; |
| 240 | 13 | if (_running >= _floor) { |
| 241 | 12 | _tobekill.forEach(function (pid) { |
| 242 | 8 | process.kill(pid, 'SIGTERM'); |
| 243 | | }); |
| 244 | 12 | _tobekill = []; |
| 245 | | } |
| 246 | | }; |
| 247 | | /* }}} */ |
| 248 | | |
| 249 | | /** |
| 250 | | * @process object |
| 251 | | */ |
| 252 | 3 | var _wakeups = 0; |
| 253 | | |
| 254 | | /** |
| 255 | | * @to be killed process |
| 256 | | */ |
| 257 | 3 | var _tobekill = []; |
| 258 | | |
| 259 | | /** |
| 260 | | * @listeners |
| 261 | | */ |
| 262 | 3 | var _listener = {}; |
| 263 | | |
| 264 | | /** |
| 265 | | * @worker status |
| 266 | | */ |
| 267 | 3 | var _wstatus = STATUS.PENDING; |
| 268 | | |
| 269 | | /** |
| 270 | | * @timer to check heartbeat |
| 271 | | */ |
| 272 | | /* {{{ */ |
| 273 | 3 | var _timer = setInterval(function () { |
| 274 | 0 | var _expire = Date.now() - MAX_HEARTBEAT_INTERVAL; |
| 275 | 0 | for (var pid in _pstatus) { |
| 276 | 0 | var _stat = _pstatus[pid]; |
| 277 | | // `MAX_HEARTBEAT_INTERVAL` no heartbeat, kill the worker |
| 278 | 0 | if (_stat._time < _expire) { |
| 279 | 0 | _newChild(); |
| 280 | 0 | _tobekill.push(pid); |
| 281 | | } |
| 282 | | } |
| 283 | | }, 2000); |
| 284 | | |
| 285 | | /* }}} */ |
| 286 | | |
| 287 | 3 | var _me = {}; |
| 288 | | |
| 289 | | /* {{{ public function start() */ |
| 290 | 3 | _me.start = function () { |
| 291 | 16 | _wstatus = STATUS.PENDING; |
| 292 | 16 | for (var i = _pobject.length; i < _options.children; i++) { |
| 293 | 11 | _newChild(); |
| 294 | | } |
| 295 | | |
| 296 | 16 | var usepush = 2 * _options.children; |
| 297 | 16 | _options.listen.forEach(function (item) { |
| 298 | 17 | if (_listener[item]) { |
| 299 | 13 | return; |
| 300 | | } |
| 301 | 4 | _listener[item] = listen(item, function (handle, port) { |
| 302 | 4 | if (_fdqueue.push([port, handle]) <= usepush) { |
| 303 | 4 | _wakeups = (_wakeups + 1) % _pobject.length; |
| 304 | 4 | try { |
| 305 | 4 | _pobject[_wakeups].send({ |
| 306 | | 'type' : MESSAGE.WAKEUP, |
| 307 | | }); |
| 308 | | } catch (e) { |
| 309 | | } |
| 310 | | } |
| 311 | | }); |
| 312 | | }); |
| 313 | | }; |
| 314 | 3 | _me.start(); |
| 315 | | /* }}} */ |
| 316 | | |
| 317 | | /* {{{ public function stop() */ |
| 318 | | /** |
| 319 | | * stop worker process |
| 320 | | */ |
| 321 | 3 | _me.stop = function (callback, signal) { |
| 322 | 6 | _wstatus = STATUS.STOPING; |
| 323 | 6 | for (var idx in _listener) { |
| 324 | 4 | var _fd = _listener[idx]; |
| 325 | 4 | _fd.close(); |
| 326 | 4 | _fd = null; |
| 327 | | } |
| 328 | 6 | _listener = {}; |
| 329 | 6 | _killall(signal); |
| 330 | | |
| 331 | 6 | callback && callback(); |
| 332 | | |
| 333 | | // var _timer1, _timer2; |
| 334 | | // _timer1 = setTimeout(function () { |
| 335 | | // _killall(signal); |
| 336 | | // clearInterval(_timer2); |
| 337 | | // }, 1000); |
| 338 | | |
| 339 | | // _timer2 = setInterval(function () { |
| 340 | | // if (0 === _fdqueue.length) { |
| 341 | | // _killall(signal); |
| 342 | | // clearTimeout(_timer1); |
| 343 | | // clearInterval(_timer2); |
| 344 | | // } |
| 345 | | // }, 20); |
| 346 | | }; |
| 347 | | /* }}} */ |
| 348 | | |
| 349 | | /* {{{ public function reload() */ |
| 350 | 3 | _me.reload = function () { |
| 351 | 5 | for (var pid in _pstatus) { |
| 352 | 19 | _newChild(); |
| 353 | 19 | pid = parseInt(pid, 10); |
| 354 | 19 | if (pid) { |
| 355 | 19 | _tobekill.push(pid); |
| 356 | | } |
| 357 | | } |
| 358 | | }; |
| 359 | | /* }}} */ |
| 360 | | |
| 361 | | /* {{{ public function accept() */ |
| 362 | 3 | _me.accept = function (data, from, pid) { |
| 363 | 1 | _pobject.forEach(function (sub) { |
| 364 | 4 | try { |
| 365 | 4 | sub.send({ |
| 366 | | 'type' : MESSAGE.COMMAND, |
| 367 | | 'data' : data, |
| 368 | | 'from' : from, |
| 369 | | 'pid' : pid, |
| 370 | | }); |
| 371 | | } catch (e) { |
| 372 | 0 | NOTICE('SEND', e.stack); |
| 373 | | } |
| 374 | | }); |
| 375 | | }; |
| 376 | | /* }}} */ |
| 377 | | |
| 378 | 3 | return _me; |
| 379 | | |
| 380 | | }; |
| 381 | | |
| 382 | | /* {{{ function _createPidFile() */ |
| 383 | 2 | var _createPidFile = function (fname) { |
| 384 | 2 | var pid = String(process.pid); |
| 385 | 2 | fs.writeFile(fname, pid, function (error) { |
| 386 | 2 | if (error) { |
| 387 | 0 | throw error; |
| 388 | | } |
| 389 | | }); |
| 390 | | |
| 391 | 2 | process.on('exit', function () { |
| 392 | 0 | try { |
| 393 | 0 | if (pid !== fs.readFileSync(fname, 'utf-8').trim()) { |
| 394 | 0 | return; |
| 395 | | } |
| 396 | 0 | fs.unlinkSync(fname); |
| 397 | | } catch (e) { |
| 398 | | } |
| 399 | | }); |
| 400 | | }; |
| 401 | | /* }}} */ |
| 402 | | |
| 403 | | /* {{{ Master constructor */ |
| 404 | | |
| 405 | 2 | var Master = function (options) { |
| 406 | 2 | if (!(this instanceof Master)) { |
| 407 | 0 | return new Master(); |
| 408 | | } |
| 409 | | |
| 410 | 2 | if (options.pidfile) { |
| 411 | 2 | _createPidFile(options.pidfile); |
| 412 | | } |
| 413 | | |
| 414 | 2 | if (options.statusfile) { |
| 415 | 2 | WRITE_STATUS_FILE = function (name, pid, message) { |
| 416 | 13 | fs.createWriteStream(options.statusfile, { |
| 417 | | flags: 'a+', |
| 418 | | encoding: 'utf-8', |
| 419 | | mode: 0644 |
| 420 | | }).end(util.format( |
| 421 | | '%d:\t%s\t%d\t%j\n', process.pid, name, pid, message |
| 422 | | )); |
| 423 | | }; |
| 424 | | |
| 425 | | // XXX: fork |
| 426 | | } |
| 427 | | |
| 428 | 2 | Emitter.call(this); |
| 429 | | }; |
| 430 | 2 | util.inherits(Master, Emitter); |
| 431 | | |
| 432 | | /* }}} */ |
| 433 | | |
| 434 | 2 | var __WORKERS_LIST = {}; |
| 435 | 2 | var __GLOBAL_MASTER = null; |
| 436 | | |
| 437 | | /** |
| 438 | | * Register a group of workers. |
| 439 | | * @param {String} name |
| 440 | | * @param {String} file, worker file path. |
| 441 | | * @param {Object} options |
| 442 | | * - {Array|Number|String} listen, listen port or domain sock, e.g.: `80`, `[80, 8080]` or `[80, '/tmp/web.sock']`. |
| 443 | | * - {Number} [children], worker number, default is `os.cpu` number. |
| 444 | | * - {Number} [max_request], max request number per child worker. Dafault is `0`, no limit. |
| 445 | | * - {Number} [max_fatal_restart], max fatal to giveup restart. Dafault is `5`. |
| 446 | | * - {Bool} [trace_gc], trace gc, default is `false`. |
| 447 | | * @param {Array} [argv] process start argv. |
| 448 | | * @api public |
| 449 | | */ |
| 450 | 2 | Master.prototype.register = function (name, file, options, argv) { |
| 451 | 3 | name = _normalize(name); |
| 452 | 3 | if (__WORKERS_LIST[name]) { |
| 453 | 0 | __WORKERS_LIST[name].stop(function () { |
| 454 | | }, 'SIGKILL'); |
| 455 | | } |
| 456 | | |
| 457 | 3 | argv = Array.isArray(argv) ? argv : []; |
| 458 | 3 | argv.unshift(file); |
| 459 | | |
| 460 | 3 | if (options.trace_gc) { |
| 461 | 0 | argv.unshift('--trace_gc'); |
| 462 | 0 | delete options.trace_gc; |
| 463 | | } |
| 464 | | |
| 465 | 3 | __WORKERS_LIST[name] = _workerPair(argv, options, name); |
| 466 | 3 | return this; |
| 467 | | }; |
| 468 | | |
| 469 | 2 | Master.prototype.shutdown = function (callback, signal, name) { |
| 470 | 7 | if (name) { |
| 471 | 4 | if (__WORKERS_LIST[name]) { |
| 472 | 3 | __WORKERS_LIST[name].stop(callback, signal); |
| 473 | | } else { |
| 474 | 1 | callback && callback(); |
| 475 | | } |
| 476 | | } else { |
| 477 | 3 | for (var k in __WORKERS_LIST) { |
| 478 | 3 | __WORKERS_LIST[k].stop(null, signal); |
| 479 | | } |
| 480 | 3 | callback && callback(); |
| 481 | | } |
| 482 | 7 | return this; |
| 483 | | }; |
| 484 | | |
| 485 | 2 | Master.prototype.reload = function (name) { |
| 486 | 4 | if (name) { |
| 487 | 2 | if (__WORKERS_LIST[name]) { |
| 488 | 2 | __WORKERS_LIST[name].reload(); |
| 489 | | } |
| 490 | | } else { |
| 491 | 2 | for (var k in __WORKERS_LIST) { |
| 492 | 2 | __WORKERS_LIST[k].reload(); |
| 493 | | } |
| 494 | | } |
| 495 | 4 | return this; |
| 496 | | }; |
| 497 | | |
| 498 | 2 | Master.prototype.dispatch = function () { |
| 499 | 1 | for (var name in __WORKERS_LIST) { |
| 500 | 1 | __WORKERS_LIST[name].start(); |
| 501 | | } |
| 502 | 1 | return this; |
| 503 | | }; |
| 504 | | |
| 505 | 2 | exports.create = function (options) { |
| 506 | | |
| 507 | 3 | if (!__GLOBAL_MASTER) { |
| 508 | 2 | __GLOBAL_MASTER = new Master(options); |
| 509 | | |
| 510 | 2 | process.on('SIGHUB', function () {}); |
| 511 | 2 | process.on('exit', function () { |
| 512 | 1 | NOTICE('About to exit ...'); |
| 513 | 1 | __GLOBAL_MASTER.shutdown(function () { |
| 514 | 1 | NOTICE('SIGKILL exited'); |
| 515 | | }, 'SIGKILL'); |
| 516 | | }); |
| 517 | | |
| 518 | 2 | process.on('SIGTERM', function () { |
| 519 | 1 | NOTICE('Got SIGTERM, about to exit...'); |
| 520 | 1 | __GLOBAL_MASTER.shutdown(function () { |
| 521 | 1 | NOTICE('SIGTERM exited'); |
| 522 | | }); |
| 523 | | }); |
| 524 | | |
| 525 | 2 | process.on('SIGUSR1', function () { |
| 526 | 1 | NOTICE('Got SIGUSR1, about to reload...'); |
| 527 | 1 | __GLOBAL_MASTER.reload(); |
| 528 | | }); |
| 529 | | } |
| 530 | | |
| 531 | 3 | return __GLOBAL_MASTER; |
| 532 | | |
| 533 | | }; |
| 534 | | |