add task fail retry and finally failed report to HeroUnion

config.mjs

@@ -5,6 +5,7 @@ let configs = {
     task_list_dir: 'todo/',       //待抓取任务文件保存目录
     data_save_dir: 'data/',       //抓取完成数据保存目录，文件格式：.url快捷方式，详细说明见：https://filesite.io
 
+    max_fail_retry: 5,            //任务失败最多重试次数
 
     //HeroUnion英雄联盟对接配置
     herounion: {

lib/heroBot.mjs

@@ -122,12 +122,15 @@ class HeroBot {
     }
 
     //回传任务数据给联盟
-    async saveTaskData(task_id, task_token, task_data) {
+    async saveTaskData(task_id, task_token, task_data, task_status) {
         let params = {
             name: this.name,
             task_id: task_id,
             task_result: task_data
         };
+        if (typeof(task_status) != 'undefined') {
+            params.status = task_status;
+        }
         params.sign = this.sign(params, task_token);    //对参数进行签名
 
         let response = null;

lib/tajian.mjs

@@ -19,16 +19,14 @@ class TaJian {
         URL=https://microsoft.com/
     */
     async saveUrlShortcut(filename, data) {
-    //console.log('TaJian try to save shortcut url');
-
         try {
 
             const dirPath = path.resolve(this.save_dir);
             const filepath = `${dirPath}/${filename}.url`;
 
             const shortUrlContent = `\[InternetShortcut\]
-URL=${data.url}
+            URL=${data.url}
-`;
+            `;
             await writeFile(filepath, shortUrlContent, { encoding: 'utf8' });
         }catch(error) {
             console.error('Save short url file failed: %s', error);
@@ -39,8 +37,6 @@ URL=${data.url}
     }
 
     async saveDescriptionFiles(filename, data) {
-    //console.log('TaJian try to save description files');
-
         try {
             const dirPath = path.resolve(this.save_dir);
 

lib/taskMoniter.mjs

@@ -104,8 +104,12 @@ class TaskMoniter {
         this.taskStatus[this.statusCode.done] ++;
         this.taskStatus[this.statusCode.running] --;
 
+        //如果不是联盟的任务，则把本地任务文件删除
+        if (typeof(this.tasks[task_id].from) == 'undefined' || this.tasks[task_id].from != 'HeroUnion') {
             const filepath = this.getTaskFilePath(task_id);
             common.removeFile(filepath);        //async delete
+        }
+
         return true;
     }
 

spider.mjs

@@ -1,6 +1,8 @@
 /**
  * 爬虫主程序
  * 负责监听任务目录里的新任务，并自动抓取数据保存到数据目录。
+ * 增加失败任务的重试机制
+ * 增加失败任务上报
  */
 import getConfigs from './config.mjs';
 import common from './lib/common.mjs';
@@ -93,7 +95,21 @@ import cron from 'node-cron';
                     taskMoniter.setTaskFailed(task.id);
                 }
             }else {
+                //失败后最多重试 5 次
+                if (typeof(task.fail_retry) == 'undefined') {
+                    task.fail_retry = 0;
+                }else {
+                    task.fail_retry ++;
+                }
+
+                taskMoniter.updateTask(task.id, task);
+
+                if (task.fail_retry > configs.max_fail_retry) {
                     taskMoniter.setTaskFailed(task.id);
+
+                    //上报联盟，任务失败
+                    heroBot.saveTaskData(task.id, task.token, [], 'failed');
+                }
             }
 
             spider_is_running = false;