|
|
@ -35,6 +35,7 @@ class HeroUnion { |
|
|
|
this.task_cache_time = 86400; //任务数据最长缓存时间,单位:秒
|
|
|
|
this.task_cache_time = 86400; //任务数据最长缓存时间,单位:秒
|
|
|
|
this.task_data_max_size = 1024; //任务数据最大字节数,单位:KB
|
|
|
|
this.task_data_max_size = 1024; //任务数据最大字节数,单位:KB
|
|
|
|
this.notify_timeout = 8; //回调通知请求超时时长,单位:秒
|
|
|
|
this.notify_timeout = 8; //回调通知请求超时时长,单位:秒
|
|
|
|
|
|
|
|
this.notify_max_try = 5; //回调通知最多尝试次数
|
|
|
|
this.heroHeartTimeout = 600; //爬虫心跳超时时长,单位:秒
|
|
|
|
this.heroHeartTimeout = 600; //爬虫心跳超时时长,单位:秒
|
|
|
|
|
|
|
|
|
|
|
|
this.stats = { |
|
|
|
this.stats = { |
|
|
@ -97,6 +98,10 @@ class HeroUnion { |
|
|
|
this.notify_timeout = this.config.notify_timeout; //回调通知请求超时时长,单位:秒
|
|
|
|
this.notify_timeout = this.config.notify_timeout; //回调通知请求超时时长,单位:秒
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (typeof(this.config.notify_max_try) != 'undefined' && this.config.notify_max_try) { |
|
|
|
|
|
|
|
this.notify_max_try = this.config.notify_max_try; //最多回调通知次数
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if (typeof(this.config.heroHeartTimeout) != 'undefined' && this.config.heroHeartTimeout) { |
|
|
|
if (typeof(this.config.heroHeartTimeout) != 'undefined' && this.config.heroHeartTimeout) { |
|
|
|
this.heroHeartTimeout = this.config.heroHeartTimeout; //爬虫心跳超时时长,单位:秒
|
|
|
|
this.heroHeartTimeout = this.config.heroHeartTimeout; //爬虫心跳超时时长,单位:秒
|
|
|
|
} |
|
|
|
} |
|
|
@ -133,7 +138,9 @@ class HeroUnion { |
|
|
|
* results: [], |
|
|
|
* results: [], |
|
|
|
* created: 0, //timestamp in seconds
|
|
|
|
* created: 0, //timestamp in seconds
|
|
|
|
* updated: 0, //timestamp in seconds
|
|
|
|
* updated: 0, //timestamp in seconds
|
|
|
|
* error: '' |
|
|
|
* error: '', |
|
|
|
|
|
|
|
* notified: false //是否成功发送回调通知
|
|
|
|
|
|
|
|
* notify_time: 0 //回调通知次数
|
|
|
|
* } |
|
|
|
* } |
|
|
|
**/ |
|
|
|
**/ |
|
|
|
createTask(uuid, url, platform, contract, data_mode, notify_url, country, lang) { |
|
|
|
createTask(uuid, url, platform, contract, data_mode, notify_url, country, lang) { |
|
|
@ -143,6 +150,9 @@ class HeroUnion { |
|
|
|
id: this.generateTaskId(uuid), |
|
|
|
id: this.generateTaskId(uuid), |
|
|
|
status: 'waiting', |
|
|
|
status: 'waiting', |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
notified: false, |
|
|
|
|
|
|
|
notify_time: 0, |
|
|
|
|
|
|
|
|
|
|
|
//必选
|
|
|
|
//必选
|
|
|
|
uuid: uuid, |
|
|
|
uuid: uuid, |
|
|
|
url: url, |
|
|
|
url: url, |
|
|
@ -276,7 +286,11 @@ class HeroUnion { |
|
|
|
const response = await axios.post(notify_url, params, {timeout: this.notify_timeout*1000}); |
|
|
|
const response = await axios.post(notify_url, params, {timeout: this.notify_timeout*1000}); |
|
|
|
if (response.status == 200) { |
|
|
|
if (response.status == 200) { |
|
|
|
notified = true; |
|
|
|
notified = true; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
//TODO: 更新任务notified状态以及notify_time通知次数
|
|
|
|
}else { |
|
|
|
}else { |
|
|
|
|
|
|
|
//TODO: 检查任务通知次数是否达到最大尝试次数
|
|
|
|
|
|
|
|
|
|
|
|
common.error('[FAILED] Notify to %s failed, response status: %s, status text: %s, result: %s', |
|
|
|
common.error('[FAILED] Notify to %s failed, response status: %s, status text: %s, result: %s', |
|
|
|
notify_url, response.status, response.statusText, response.data); |
|
|
|
notify_url, response.status, response.statusText, response.data); |
|
|
|
} |
|
|
|
} |
|
|
|