|
|
|
@ -40,6 +40,7 @@ class HeroUnion {
@@ -40,6 +40,7 @@ class HeroUnion {
|
|
|
|
|
this.heros = []; //hero爬虫队列
|
|
|
|
|
this.tasks = []; //任务队列
|
|
|
|
|
|
|
|
|
|
//任务相关数据
|
|
|
|
|
this.taskStatus = { |
|
|
|
|
'total': 0, |
|
|
|
|
'waiting': 0, |
|
|
|
@ -47,6 +48,7 @@ class HeroUnion {
@@ -47,6 +48,7 @@ class HeroUnion {
|
|
|
|
|
'done': 0, |
|
|
|
|
'failed': 0 |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
this.statusCode = { |
|
|
|
|
'waiting': '待处理', |
|
|
|
|
'running': '处理中', |
|
|
|
@ -60,6 +62,15 @@ class HeroUnion {
@@ -60,6 +62,15 @@ class HeroUnion {
|
|
|
|
|
'xigua': true, |
|
|
|
|
'bilibili': true |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
//爬虫相关数据
|
|
|
|
|
this.heroHeartTimeout = 600; //爬虫心跳超时时长,单位:秒
|
|
|
|
|
this.heroStatus = { |
|
|
|
|
'total': 0, |
|
|
|
|
'idle': 0, //空闲
|
|
|
|
|
'busy': 0, //繁忙
|
|
|
|
|
'offline': 0 //离线
|
|
|
|
|
}; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
//公用方法
|
|
|
|
@ -264,7 +275,45 @@ class HeroUnion {
@@ -264,7 +275,45 @@ class HeroUnion {
|
|
|
|
|
//--爬虫相关功能--
|
|
|
|
|
|
|
|
|
|
//接收爬虫状态上报
|
|
|
|
|
heroOnboard() { |
|
|
|
|
/** |
|
|
|
|
* bot爬虫属性 |
|
|
|
|
* name |
|
|
|
|
* description |
|
|
|
|
* status: [idle, busy] |
|
|
|
|
* timestamp |
|
|
|
|
* country |
|
|
|
|
* lang |
|
|
|
|
*/ |
|
|
|
|
heroOnboard(bot) { |
|
|
|
|
let cachedBot = this.heros.find((item) => item.name == bot.name); |
|
|
|
|
|
|
|
|
|
if (cachedBot) { //如果是已经存在的爬虫
|
|
|
|
|
if (cachedBot.status != bot.status) { |
|
|
|
|
if (bot.status == 'idle') { |
|
|
|
|
this.heroStatus.idle ++; |
|
|
|
|
this.heroStatus.busy --; |
|
|
|
|
}else { |
|
|
|
|
this.heroStatus.busy ++; |
|
|
|
|
this.heroStatus.idle --; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
cachedBot = bot; //数据更新
|
|
|
|
|
}else { |
|
|
|
|
this.heros.push(bot); //添加新爬虫
|
|
|
|
|
|
|
|
|
|
this.heroStatus.total ++; |
|
|
|
|
if (bot.status == 'idle') { |
|
|
|
|
this.heroStatus.idle ++; |
|
|
|
|
}else { |
|
|
|
|
this.heroStatus.busy ++; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
//定期检查爬虫是否在线
|
|
|
|
|
//如果上一次上报状态时间在10分钟前,则设置该爬虫已下线
|
|
|
|
|
beroHeartCheck() { |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|