api newtask test done

common.mjs

@@ -137,7 +137,7 @@ class Common {
 
     //检查url是否符合要求
     isUrlOk(url) {
-        return /^http(s)?:\/\/[\w\.]{6,100}$/i.test(url);
+        return /^http(s)?:\/\/[\w\.\/]{6,100}$/i.test(url);
     }
 
     //检查uuid是否符合要求：6-32位的英文字符串
@@ -159,14 +159,12 @@ class Common {
         if (arguments[0]) {
             let logFormat = `[%s] ${arguments[0]}`;
             args.push(logFormat);
+            args.push(localTime);
         }
 
-        if (arguments) {
+        if (arguments && arguments.length > 1) {
             for (const index in arguments) {
-                if (index > 1) {
+                if (index > 0) {
-                    args.push(arguments[index]);
-                }else if (index == 1) {
-                    args.push(localTime);
                     args.push(arguments[index]);
                 }
             }

conf/config.json

@@ -13,6 +13,6 @@
     "heroHeartCheckFrequence": 60,
 
     "tokens": {
-        "demo": "hello#world!"
+        "herounion_demo": "hello#world!"
     }
 }

heroUnion.mjs

@@ -22,7 +22,7 @@ import path from 'node:path';
 import cron from 'node-cron';
 import axios from 'axios';
 import common from './common.mjs';
-
+import md5 from 'md5';
 
 class HeroUnion {
 
@@ -118,6 +118,12 @@ class HeroUnion {
         return `${uuid}_${timestamp}`;
     }
 
+    //根据当前时间生成任务的密钥
+    generateTaskToken(id) {
+        let timestamp = common.getTimestamp();
+        return md5(`${id}_${timestamp}`);
+    }
+
     isSupportedPlatform(platform) {
         return typeof(this.supportedPlatforms[platform]) != 'undefined' && this.supportedPlatforms[platform];
     }
@@ -139,8 +145,9 @@ class HeroUnion {
      *  created: 0,    //timestamp in seconds
      *  updated: 0,    //timestamp in seconds
      *  error: '',
-     *  notified: false      //是否成功发送回调通知
+     *  notified: false,      //是否成功发送回调通知
-     *  notify_time: 0       //回调通知次数
+     *  notify_time: 0,       //回调通知次数
+     *  token: ''             //任务密钥，爬虫完成任务回传数据的时候用它签名
      * }
      **/
     createTask(uuid, url, platform, contract, data_mode, notify_url, country, lang) {
@@ -220,6 +227,9 @@ class HeroUnion {
 
         if (taskIndex > -1) {
             this.tasks[taskIndex].status = 'running';
+            //为task生成一个随机密钥，便于爬虫处理完成后回传的时候对数据进行签名
+            this.tasks[taskIndex].token = this.generateTaskToken(this.tasks[taskIndex].id);
+
             searchResult = this.tasks[taskIndex];
         }
 

router_api.mjs

@@ -13,6 +13,7 @@ heroUnion.init();
 
 const router = express.Router();
 
+//获取联盟公开接口列表
 router.get('/', async (req, res) => {
     const apiList = {
         "/api/": "查看所有API",
@@ -32,6 +33,8 @@ router.get('/', async (req, res) => {
 });
 
 /**
+ * 联盟成员向联盟提交数据抓取任务
+ * 
  * 参数列表
  * uuid: 用户ID
  * url: 目标网址
@@ -106,22 +109,45 @@ router.post('/newtask/', async (req, res) => {
     return res.status(200).json(data);
 });
 
+/**
+ * 联盟成员向联盟查询某个任务的数据
+ * 
+ **/
+router.get('/querytask/', async (req, res) => {
+    
+    return res.send('api/querytask/');
+});
+
+/**
+ * hero爬虫从联盟获取等待中的数据抓取任务
+ * 
+ * 参数：
+ * platform: 爬虫支持的平台
+ * contract：爬虫支持的合约
+ * country：爬虫所在国家
+ * lang：爬虫支持的语言
+ * data_mode：爬虫支持的返回数据格式
+ **/
 router.get('/gettask/', async (req, res) => {
 
     return res.send('api/gettask/');
 });
 
+/**
+ * hero爬虫向联盟提交某个任务的抓取结果
+ * 
+ **/
 router.post('/savetask/', async (req, res) => {
+    //TODO: 签名检查
 
-    return res.send('api/savetask/');
+    //TODO: 任务状态检查，如果已经完成，则忽略当前请求
-});
-
-router.get('/querytask/', async (req, res) => {
     
-    return res.send('api/querytask/');
+    return res.send('api/savetask/');
 });
 
 /**
+ * 爬虫向联盟上报自己的状态，以保持在线
+ * 
  * 参数列表
  * name
  * description
@@ -194,6 +220,7 @@ router.post('/onboard/', async (req, res) => {
     return res.status(200).json(data);
 });
 
+//获取联盟的hero爬虫列表
 router.get('/heros/', async (req, res) => {
     let page = req.query.page,
         limit = req.query.limit;
@@ -208,6 +235,7 @@ router.get('/heros/', async (req, res) => {
     return res.status(200).json(heroUnion.getHeros(page, limit));
 });
 
+//获取联盟状态
 router.get('/stats/', async (req, res) => {
     return res.status(200).json(heroUnion.getStats());
 });

test/common.test.mjs

@@ -54,8 +54,14 @@ test('Common function getLocalTimeString test', async (t) => {
 
 test('Common function log/info/warn/error test', async (t) => {
     let string = '测试log输出';
-    let args = common.log('console.log替换测试：%s', string);
+    let args = [];
 
+    args = common.log(string);
+    assert.ok(args);
+    assert.equal(/^\[%s\] /i.test(args[0]), true);
+    assert.equal(args.length, 2);
+
+    args = common.log('console.log替换测试：%s', string);
     assert.ok(args);
     assert.equal(/^\[%s\] /i.test(args[0]), true);
     assert.equal(args.length, 3);
@@ -73,10 +79,10 @@ test('Common function log/info/warn/error test', async (t) => {
     assert.equal(args.length, 3);
     assert.equal(args[args.length - 1], string);
 
-    args = common.error('console.error替换测试：%s', string);
+    args = common.error('console.error替换测试：%s，再重复一次：%s', string, string);
     assert.ok(args);
     assert.equal(/^\[%s\] /i.test(args[0]), true);
-    assert.equal(args.length, 3);
+    assert.equal(args.length, 4);
     assert.equal(args[args.length - 1], string);
 
     console.log("插入日期后的参数：\n%s", args);

test/heroUnion.test.mjs

@@ -53,8 +53,8 @@ test('Hero onboard test', async (t) => {
     assert.equal(response2.data.code, 1);
 });
 
-test('HeroUnion stats test', async (t) => {
+test('HeroUnion get heros test', async (t) => {
-    let api = 'http://127.0.0.1:8080/api/stats/';
+    let api = 'http://127.0.0.1:8080/api/heros/';
 
     const response = await axios.get(api, axiosConfig);
     console.log(response.data);
@@ -62,8 +62,33 @@ test('HeroUnion stats test', async (t) => {
     assert.equal(response.status, 200);
 });
 
-test('HeroUnion get heros test', async (t) => {
+test('HeroUnion create task test', async (t) => {
-    let api = 'http://127.0.0.1:8080/api/heros/';
+    let params = {
+         uuid: 'herounion_demo',
+         url: 'https://v.douyin.com/xxx',
+         platform: 'douyin',
+         contract: 'tajiantv',
+         data_mode: 'json',
+         country: 'cn',
+         lang: 'zh',
+         notify_url: 'https://tajian.tv/test/'
+    };
+    let token = 'hello#world!';
+    params.sign = common.sign(params, token);
+
+    let api = 'http://127.0.0.1:8080/api/newtask/';
+
+    const response = await axios.post(api, params, axiosConfig);
+    console.log(response.data);
+
+    assert.equal(response.status, 200);
+    assert.equal(response.data.code, 1);
+});
+
+
+
+test('HeroUnion stats test', async (t) => {
+    let api = 'http://127.0.0.1:8080/api/stats/';
 
     const response = await axios.get(api, axiosConfig);
     console.log(response.data);