diff --git a/conf/config.json b/conf/config.json index e8c8623..c05126a 100644 --- a/conf/config.json +++ b/conf/config.json @@ -1,6 +1,16 @@ { "name": "Hero Union", "version": "0.1", + + "task_data_max_size": 1024, + "task_cache_time": 86400, + + "heroHeartTimeout": 20, + "notify_timeout": 8, + + "reloadConfigFrequence": 10, + "heroHeartCheckFrequence": 10, + "tokens": { "demo": "hello#world!" } diff --git a/heroUnion.mjs b/heroUnion.mjs index 74500e6..f6f7bf4 100644 --- a/heroUnion.mjs +++ b/heroUnion.mjs @@ -30,10 +30,12 @@ class HeroUnion { constructor() { this.config = null; + //默认配置 //this.task_data_dir = path.resolve('./tmp/data/'); //任务数据保存目录 this.task_cache_time = 86400; //任务数据最长缓存时间,单位:秒 this.task_data_max_size = 1024; //任务数据最大字节数,单位:KB this.notify_timeout = 8; //回调通知请求超时时长,单位:秒 + this.heroHeartTimeout = 600; //爬虫心跳超时时长,单位:秒 this.stats = { start_time: common.getTimestampInSeconds() @@ -66,7 +68,6 @@ class HeroUnion { }; //爬虫相关数据 - this.heroHeartTimeout = 600; //爬虫心跳超时时长,单位:秒 this.heroStatus = { 'total': 0, 'idle': 0, //空闲 @@ -82,6 +83,23 @@ class HeroUnion { async getConfig(forceReload) { if ( !this.config || (typeof(forceReload) != 'undefined' && forceReload) ) { this.config = await common.getConfigFromJsonFile('config.json'); + + //覆盖默认配置 + if (typeof(this.config.task_cache_time) != 'undefined' && this.config.task_cache_time) { + this.task_cache_time = this.config.task_cache_time; //任务数据最长缓存时间,单位:秒 + } + + if (typeof(this.config.task_data_max_size) != 'undefined' && this.config.task_data_max_size) { + this.task_data_max_size = this.config.task_data_max_size; //任务数据最大字节数,单位:KB + } + + if (typeof(this.config.notify_timeout) != 'undefined' && this.config.notify_timeout) { + this.notify_timeout = this.config.notify_timeout; //回调通知请求超时时长,单位:秒 + } + + if (typeof(this.config.heroHeartTimeout) != 'undefined' && this.config.heroHeartTimeout) { + this.heroHeartTimeout = this.config.heroHeartTimeout; //爬虫心跳超时时长,单位:秒 + } } return this.config; @@ -278,20 +296,19 @@ class HeroUnion { * lang */ heroOnboard(bot) { - let cachedBot = this.heros.find((item) => item.name == bot.name); + let cachedBotIndex = this.heros.findIndex((item) => item.name == bot.name), + cachedBot = cachedBotIndex > -1 ? this.heros[cachedBotIndex] : null; if (cachedBot) { //如果是已经存在的爬虫 if (cachedBot.status != bot.status) { - if (bot.status == 'idle') { - this.heroStatus.idle ++; - this.heroStatus.busy --; - }else { - this.heroStatus.busy ++; - this.heroStatus.idle --; - } + console.log('Hero %s status change from %s to %s', cachedBot.name, cachedBot.status, bot.status); + this.heroStatus[cachedBot.status] --; + this.heroStatus[bot.status] ++; } - cachedBot = bot; //数据更新 + this.heros[cachedBotIndex] = bot; //数据更新 + + console.log('Hero %s is %s at %s', bot.name, bot.status, bot.timestamp); }else { this.heros.push(bot); //添加新爬虫 @@ -301,6 +318,8 @@ class HeroUnion { }else { this.heroStatus.busy ++; } + + console.log('Hero %s is onboard at %s', bot.name, bot.timestamp); } } @@ -309,7 +328,8 @@ class HeroUnion { heroHeartCheck() { let _self = this; - const frequence = 60; //1 分钟检查一次 + const frequence = typeof(this.config.heroHeartCheckFrequence) != 'undefined' + && this.config.heroHeartCheckFrequence ? this.config.heroHeartCheckFrequence : 60; //1 分钟检查一次 const cronjob = cron.schedule(`*/${frequence} * * * * *`, () => { let timestamp = common.getTimestampInSeconds(); _self.heros.forEach(function(item, index) { @@ -318,7 +338,7 @@ class HeroUnion { _self.heros[index].status = 'offline'; _self.heroStatus.offline ++; - console.log('Hero %s is offline, its last heart beat is %s', item.name, item.timestamp); + console.log('Hero %s is offline, last heart beat at %s', item.name, item.timestamp); } }); }, { @@ -333,7 +353,8 @@ class HeroUnion { autoReloadConfigs() { let _self = this; - const frequence = 300; //5 分钟重新加载一次 + const frequence = typeof(this.config.reloadConfigFrequence) != 'undefined' + && this.config.reloadConfigFrequence ? this.config.reloadConfigFrequence : 300; //5 分钟重新加载一次 const cronjob = cron.schedule(`*/${frequence} * * * * *`, () => { const forceReload = true; _self.getConfig(forceReload); @@ -355,8 +376,8 @@ class HeroUnion { } //初始化 - init() { - this.getConfig(); + async init() { + await this.getConfig(); this.autoReloadConfigs(); this.heroHeartCheck(); }