|
|
|
@ -73,15 +73,6 @@ class HeroUnion {
@@ -73,15 +73,6 @@ class HeroUnion {
|
|
|
|
|
}; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
//公用方法
|
|
|
|
|
getTimestamp() { |
|
|
|
|
return Math.floor(Date.now()); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
getTimestampInSeconds() { |
|
|
|
|
return Math.floor(Date.now() / 1000); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
isDataTooLarge(data) { |
|
|
|
|
return JSON.stringify(data).length > this.task_data_max_size * 1024; |
|
|
|
|
} |
|
|
|
@ -98,7 +89,7 @@ class HeroUnion {
@@ -98,7 +89,7 @@ class HeroUnion {
|
|
|
|
|
|
|
|
|
|
//根据任务提交者ID和时间戳生成任务ID编号
|
|
|
|
|
generateTaskId(uuid) { |
|
|
|
|
let timestamp = this.getTimestamp(); |
|
|
|
|
let timestamp = common.getTimestamp(); |
|
|
|
|
return `${uuid}_${timestamp}`; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -125,7 +116,7 @@ class HeroUnion {
@@ -125,7 +116,7 @@ class HeroUnion {
|
|
|
|
|
* } |
|
|
|
|
**/ |
|
|
|
|
createTask(uuid, url, platform, data_mode, notify_url, country, lang) { |
|
|
|
|
let timestamp = this.getTimestampInSeconds(); |
|
|
|
|
let timestamp = common.getTimestampInSeconds(); |
|
|
|
|
|
|
|
|
|
let task = { |
|
|
|
|
id: this.generateTaskId(uuid), |
|
|
|
@ -220,7 +211,7 @@ class HeroUnion {
@@ -220,7 +211,7 @@ class HeroUnion {
|
|
|
|
|
this.tasks[taskIndex].results[resIndex] = data; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
this.tasks[taskIndex].updated = this.getTimestampInSeconds(); |
|
|
|
|
this.tasks[taskIndex].updated = common.getTimestampInSeconds(); |
|
|
|
|
this.tasks[taskIndex].status = 'done'; |
|
|
|
|
|
|
|
|
|
done = true; |
|
|
|
@ -252,7 +243,7 @@ class HeroUnion {
@@ -252,7 +243,7 @@ class HeroUnion {
|
|
|
|
|
let params = { |
|
|
|
|
"task_id": task.id, |
|
|
|
|
"task_result": task.results, |
|
|
|
|
"timestamp": this.getTimestamp(), |
|
|
|
|
"timestamp": common.getTimestamp(), |
|
|
|
|
}; |
|
|
|
|
let token = await this.getUserToken(task.uuid); |
|
|
|
|
params.sign = common.sign(params, token); |
|
|
|
@ -314,7 +305,26 @@ class HeroUnion {
@@ -314,7 +305,26 @@ class HeroUnion {
|
|
|
|
|
//定期检查爬虫是否在线
|
|
|
|
|
//如果上一次上报状态时间在10分钟前,则设置该爬虫已下线
|
|
|
|
|
beroHeartCheck() { |
|
|
|
|
let _self = this; |
|
|
|
|
|
|
|
|
|
const frequence = 60; //1 分钟检查一次
|
|
|
|
|
const cronjob = cron.schedule(`*/${frequence} * * * * *`, () => { |
|
|
|
|
let timestamp = common.getTimestampInSeconds(); |
|
|
|
|
|
|
|
|
|
_self.heros.forEach(function(item, index) { |
|
|
|
|
if (item.status != 'offline' && timestamp - item.timestamp > _self.heroHeartTimeout) { |
|
|
|
|
_self.heroStatus[item.status] --; |
|
|
|
|
|
|
|
|
|
_self.heros[index].status = 'offline'; |
|
|
|
|
_self.heroStatus.offline ++; |
|
|
|
|
console.log('Hero %s is offline, its last heart beat is %s', item.name, item.timestamp); |
|
|
|
|
} |
|
|
|
|
}); |
|
|
|
|
}, { |
|
|
|
|
scheduled: false |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
cronjob.start(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
//获取联盟状态
|
|
|
|
|