diff --git a/common.mjs b/common.mjs index d3eacfc..db457dc 100644 --- a/common.mjs +++ b/common.mjs @@ -137,7 +137,7 @@ class Common { //检查url是否符合要求 isUrlOk(url) { - return /^http(s)?:\/\/[\w\.]{6,100}$/i.test(url); + return /^http(s)?:\/\/[\w\.\/]{6,100}$/i.test(url); } //检查uuid是否符合要求：6-32位的英文字符串 @@ -159,14 +159,12 @@ class Common { if (arguments[0]) { let logFormat = `[%s] ${arguments[0]}`; args.push(logFormat); + args.push(localTime); } - if (arguments) { + if (arguments && arguments.length > 1) { for (const index in arguments) { - if (index > 1) { - args.push(arguments[index]); - }else if (index == 1) { - args.push(localTime); + if (index > 0) { args.push(arguments[index]); } } diff --git a/conf/config.json b/conf/config.json index d2c642b..0cb9ba4 100644 --- a/conf/config.json +++ b/conf/config.json @@ -13,6 +13,6 @@ "heroHeartCheckFrequence": 60, "tokens": { - "demo": "hello#world!" + "herounion_demo": "hello#world!" } } \ No newline at end of file diff --git a/heroUnion.mjs b/heroUnion.mjs index 2feeeda..f78069e 100644 --- a/heroUnion.mjs +++ b/heroUnion.mjs @@ -22,7 +22,7 @@ import path from 'node:path'; import cron from 'node-cron'; import axios from 'axios'; import common from './common.mjs'; - +import md5 from 'md5'; class HeroUnion { @@ -118,6 +118,12 @@ class HeroUnion { return `${uuid}_${timestamp}`; } + //根据当前时间生成任务的密钥 + generateTaskToken(id) { + let timestamp = common.getTimestamp(); + return md5(`${id}_${timestamp}`); + } + isSupportedPlatform(platform) { return typeof(this.supportedPlatforms[platform]) != 'undefined' && this.supportedPlatforms[platform]; } @@ -139,8 +145,9 @@ class HeroUnion { * created: 0, //timestamp in seconds * updated: 0, //timestamp in seconds * error: '', - * notified: false //是否成功发送回调通知 - * notify_time: 0 //回调通知次数 + * notified: false, //是否成功发送回调通知 + * notify_time: 0, //回调通知次数 + * token: '' //任务密钥，爬虫完成任务回传数据的时候用它签名 * } **/ createTask(uuid, url, platform, contract, data_mode, notify_url, country, lang) { @@ -220,9 +227,12 @@ class HeroUnion { if (taskIndex > -1) { this.tasks[taskIndex].status = 'running'; + //为task生成一个随机密钥，便于爬虫处理完成后回传的时候对数据进行签名 + this.tasks[taskIndex].token = this.generateTaskToken(this.tasks[taskIndex].id); + searchResult = this.tasks[taskIndex]; } - + return searchResult; } diff --git a/router_api.mjs b/router_api.mjs index b2b2d0e..db30b3a 100644 --- a/router_api.mjs +++ b/router_api.mjs @@ -13,6 +13,7 @@ heroUnion.init(); const router = express.Router(); +//获取联盟公开接口列表 router.get('/', async (req, res) => { const apiList = { "/api/": "查看所有API", @@ -32,6 +33,8 @@ router.get('/', async (req, res) => { }); /** + * 联盟成员向联盟提交数据抓取任务 + * * 参数列表 * uuid: 用户ID * url: 目标网址 @@ -106,22 +109,45 @@ router.post('/newtask/', async (req, res) => { return res.status(200).json(data); }); -router.get('/gettask/', async (req, res) => { +/** + * 联盟成员向联盟查询某个任务的数据 + * + **/ +router.get('/querytask/', async (req, res) => { + return res.send('api/querytask/'); +}); + +/** + * hero爬虫从联盟获取等待中的数据抓取任务 + * + * 参数： + * platform: 爬虫支持的平台 + * contract：爬虫支持的合约 + * country：爬虫所在国家 + * lang：爬虫支持的语言 + * data_mode：爬虫支持的返回数据格式 + **/ +router.get('/gettask/', async (req, res) => { + return res.send('api/gettask/'); }); +/** + * hero爬虫向联盟提交某个任务的抓取结果 + * + **/ router.post('/savetask/', async (req, res) => { - - return res.send('api/savetask/'); -}); + //TODO: 签名检查 -router.get('/querytask/', async (req, res) => { + //TODO: 任务状态检查，如果已经完成，则忽略当前请求 - return res.send('api/querytask/'); + return res.send('api/savetask/'); }); /** + * 爬虫向联盟上报自己的状态，以保持在线 + * * 参数列表 * name * description @@ -194,6 +220,7 @@ router.post('/onboard/', async (req, res) => { return res.status(200).json(data); }); +//获取联盟的hero爬虫列表 router.get('/heros/', async (req, res) => { let page = req.query.page, limit = req.query.limit; @@ -208,6 +235,7 @@ router.get('/heros/', async (req, res) => { return res.status(200).json(heroUnion.getHeros(page, limit)); }); +//获取联盟状态 router.get('/stats/', async (req, res) => { return res.status(200).json(heroUnion.getStats()); }); diff --git a/test/common.test.mjs b/test/common.test.mjs index 6cb8675..c4c4934 100644 --- a/test/common.test.mjs +++ b/test/common.test.mjs @@ -54,8 +54,14 @@ test('Common function getLocalTimeString test', async (t) => { test('Common function log/info/warn/error test', async (t) => { let string = '测试log输出'; - let args = common.log('console.log替换测试：%s', string); + let args = []; + args = common.log(string); + assert.ok(args); + assert.equal(/^\[%s\] /i.test(args[0]), true); + assert.equal(args.length, 2); + + args = common.log('console.log替换测试：%s', string); assert.ok(args); assert.equal(/^\[%s\] /i.test(args[0]), true); assert.equal(args.length, 3); @@ -73,10 +79,10 @@ test('Common function log/info/warn/error test', async (t) => { assert.equal(args.length, 3); assert.equal(args[args.length - 1], string); - args = common.error('console.error替换测试：%s', string); + args = common.error('console.error替换测试：%s，再重复一次：%s', string, string); assert.ok(args); assert.equal(/^\[%s\] /i.test(args[0]), true); - assert.equal(args.length, 3); + assert.equal(args.length, 4); assert.equal(args[args.length - 1], string); console.log("插入日期后的参数：\n%s", args); diff --git a/test/heroUnion.test.mjs b/test/heroUnion.test.mjs index 38a197a..607212e 100644 --- a/test/heroUnion.test.mjs +++ b/test/heroUnion.test.mjs @@ -53,8 +53,8 @@ test('Hero onboard test', async (t) => { assert.equal(response2.data.code, 1); }); -test('HeroUnion stats test', async (t) => { - let api = 'http://127.0.0.1:8080/api/stats/'; +test('HeroUnion get heros test', async (t) => { + let api = 'http://127.0.0.1:8080/api/heros/'; const response = await axios.get(api, axiosConfig); console.log(response.data); @@ -62,8 +62,33 @@ test('HeroUnion stats test', async (t) => { assert.equal(response.status, 200); }); -test('HeroUnion get heros test', async (t) => { - let api = 'http://127.0.0.1:8080/api/heros/'; +test('HeroUnion create task test', async (t) => { + let params = { + uuid: 'herounion_demo', + url: 'https://v.douyin.com/xxx', + platform: 'douyin', + contract: 'tajiantv', + data_mode: 'json', + country: 'cn', + lang: 'zh', + notify_url: 'https://tajian.tv/test/' + }; + let token = 'hello#world!'; + params.sign = common.sign(params, token); + + let api = 'http://127.0.0.1:8080/api/newtask/'; + + const response = await axios.post(api, params, axiosConfig); + console.log(response.data); + + assert.equal(response.status, 200); + assert.equal(response.data.code, 1); +}); + + + +test('HeroUnion stats test', async (t) => { + let api = 'http://127.0.0.1:8080/api/stats/'; const response = await axios.get(api, axiosConfig); console.log(response.data);