diff --git a/common.mjs b/common.mjs index ec4c700..3db03f1 100644 --- a/common.mjs +++ b/common.mjs @@ -43,28 +43,6 @@ class Common { }, {}); } - joinDict(obj, glue, separator) { //dict拼接成字符串 - if (typeof(glue) == 'undefined') { - glue = '='; - } - - if (typeof(separator) == 'undefined') { - separator = '&'; - } - - return Object.keys(obj).map(function(key) { - let arr = [key]; - let val = obj[key]; - if (typeof(val) == 'string' || typeof(val) == 'number') { - arr.push(val); - }else if (typeof(val) == 'object') { - arr.push(JSON.stringify(val)); - } - - return arr.join(glue); - }).join(separator); - } - sign(params, token) { //对参数做MD5签名 return md5( JSON.stringify(this.sortDict(params)) + token ); } diff --git a/heroBot.mjs b/heroBot.mjs new file mode 100644 index 0000000..bc5b655 --- /dev/null +++ b/heroBot.mjs @@ -0,0 +1,150 @@ +/** + * HeroUnion Bot SDK + */ + +import test from 'node:test'; +import assert from 'node:assert'; +import axios from 'axios'; +import md5 from 'md5'; + + +class HeroBot { + constructor( + server_url, + bot_name, + bot_description, + support_platforms, + support_contracts, + bot_country, + bot_lang, + bot_contact, + data_mode + ) { + //必填参数 + this.union_server = server_url; + this.name = bot_name; + this.description = bot_description; + this.platforms = support_platforms; + this.contracts = support_contracts; + + //可选参数 + this.country = typeof(bot_country) != 'undefined' ? bot_country : 'cn'; + this.lang = typeof(bot_lang) != 'undefined' ? bot_lang : 'zh'; + this.contact = typeof(bot_contact) != 'undefined' ? bot_contact : ''; + this.data_mode = typeof(data_mode) != 'undefined' ? data_mode : 'json'; + + //联盟API地址 + this.apis = { + "heartBeat": `${server_url}/api/onboard/`, + "getNewTask": `${server_url}/api/gettask/`, + "saveTaskData": `${server_url}/api/savetask/`, + }; + + //axios请求配置 + this.axiosConfig = { + timeout: 8000, //请求超时 + proxy: false //是否走代理 + }; + } + + getTimestampInSeconds() { + return Math.floor(Date.now() / 1000); + } + + sortDict(obj) { //dict按key排序 + return Object.keys(obj).sort().reduce(function(result, key) { + result[key] = obj[key]; + return result; + }, {}); + } + + sign(params, token) { //对参数做MD5签名 + return md5( JSON.stringify(this.sortDict(params)) + token ); + } + + //向联盟发送心跳数据 + async heartBeat(status) { + let params = { + name: this.name, + description: this.description, + status: status, + timestamp: this.getTimestampInSeconds(), + platforms: this.platforms, + contracts: this.contracts, + country: this.country, + lang: this.lang, + contact: this.contact + }; + + let response = null; + + try { + response = await axios.post(this.apis.heartBeat, params, this.axiosConfig); + }catch(err) { + console.error('[ERROR] HeroBot heart beat failed: %s, api: %s, params: %s', + err, + this.apis.heartBeat, + JSON.stringify(params) + ); + } + + return response ? response.data : false; + } + + //从联盟领取任务 + async getNewTask() { + let params = { + platforms: this.platforms, + contracts: this.contracts, + data_mode: this.data_mode, + country: this.country, + lang: this.lang + }; + + let queryOption = this.axiosConfig; + queryOption.method = 'get'; + queryOption.url = this.apis.getNewTask; + queryOption.params = params; + + let response = null; + + try { + response = await await axios(queryOption); + }catch(err) { + console.error('[ERROR] HeroBot get new task failed: %s, api: %s, params: %s', + err, + this.apis.getNewTask, + JSON.stringify(params) + ); + } + + return response && response.data.code == 1 ? response.data.task : false; + } + + //回传任务数据给联盟 + async saveTaskData(task_id, task_token, task_data) { + let params = { + name: this.name, + task_id: task_id, + task_result: task_data + }; + params.sign = this.sign(params, task_token); //对参数进行签名 + + let response = null; + + try { + response = await axios.post(this.apis.saveTaskData, params, this.axiosConfig); + }catch(err) { + console.error('[ERROR] HeroBot save task data failed: %s, api: %s, params: %s', + err, + this.apis.saveTaskData, + JSON.stringify(params) + ); + } + + return response ? response.data : false; + } + +} + +export default HeroBot; \ No newline at end of file diff --git a/heroUnion.mjs b/heroUnion.mjs index 1edcf3d..04e8123 100644 --- a/heroUnion.mjs +++ b/heroUnion.mjs @@ -265,6 +265,8 @@ class HeroUnion { this.tasks[taskIndex].status = 'failed'; this.tasks[taskIndex].error = 'Result is too large to save.'; + + common.error('Task %s save data failed by bot %s, data is too large.', id, bot_name); return false; } @@ -285,6 +287,7 @@ class HeroUnion { this.tasks[taskIndex].status = 'done'; + common.log('Task %s save data done by bot %s.', id, bot_name); done = true; } diff --git a/test/common.test.mjs b/test/common.test.mjs index 16a3032..570199e 100644 --- a/test/common.test.mjs +++ b/test/common.test.mjs @@ -21,17 +21,6 @@ test('Common function sortDict test', (t) => { assert.deepEqual(common.sortDict(params), expectRes); }); -test('Common function joinDict test', (t) => { - let params = { - b: 2, - a: 1 - }; - - const expectRes = "a=1&b=2"; - - assert.strictEqual(common.joinDict(common.sortDict(params)), expectRes); -}); - test('Common function getConfigFromJsonFile test', async (t) => { let filename = 'config.json'; let config = await common.getConfigFromJsonFile(filename); diff --git a/test/heroBot.test.mjs b/test/heroBot.test.mjs new file mode 100644 index 0000000..92baacc --- /dev/null +++ b/test/heroBot.test.mjs @@ -0,0 +1,58 @@ +/** + * HeroBot测试用例 + * 执行此测试之前，请先启动主程序，在根目录执行命令：npm start + */ + +import test from 'node:test'; +import assert from 'node:assert'; +import axios from 'axios'; +import common from '../common.mjs'; +import HeroBot from '../heroBot.mjs'; + +let server_url = 'http://127.0.0.1:8080', + bot_name = 'test_hero_bot', + bot_description = '测试爬虫 test', + support_platforms = 'douyin,xigua', + support_contracts = 'tajiantv', + bot_country = 'cn', + bot_lang = 'zh', + bot_contact = 'https://tajian.tv', + data_mode = 'json'; + +let heroBot = new HeroBot( + server_url, + bot_name, + bot_description, + support_platforms, + support_contracts, + bot_country, + bot_lang, + bot_contact, + data_mode + ); + +test('Hero onboard test', async (t) => { + let status = 'idle'; + const res = await heroBot.heartBeat(status); + console.log(res); + + assert.ok(res); + assert.equal(res.code, 1); +}); + +test('Hero get task and data save test', async (t) => { + const task = await heroBot.getNewTask(); + console.log(task); + + assert.ok(task); + + let task_data = { + "title": "标题测试：HeroUnion英雄联盟", + "description": "描述内容，联盟简介", + "others": "其它内容" + }; + let res = await heroBot.saveTaskData(task.id, task.token, task_data); + console.log(res); + + assert.equal(res.code, 1); +}); \ No newline at end of file diff --git a/test/heroUnion.test.mjs b/test/heroUnion.test.mjs index 4dedef1..5567206 100644 --- a/test/heroUnion.test.mjs +++ b/test/heroUnion.test.mjs @@ -25,15 +25,15 @@ test('HeroUnion api list test', async (t) => { test('Hero onboard test', async (t) => { let params = { - name: 'test_hero', - description: 'Hero test 测试爬虫', - status: 'idle', - timestamp: common.getTimestampInSeconds(), - platforms: 'douyin,xigua', - contracts: 'tajiantv', - country: 'cn', - lang: 'zh', - contact: 'https://tajian.tv' + name: 'test_hero', + description: 'Hero test 测试爬虫', + status: 'idle', + timestamp: common.getTimestampInSeconds(), + platforms: 'douyin,xigua', + contracts: 'tajiantv', + country: 'cn', + lang: 'zh', + contact: 'https://tajian.tv' }; let api = 'http://127.0.0.1:8080/api/onboard/'; @@ -64,14 +64,14 @@ test('HeroUnion get heros test', async (t) => { test('HeroUnion create task test', async (t) => { let params = { - uuid: 'herounion_demo', - url: 'https://v.douyin.com/xxx', - platform: 'douyin', - contract: 'tajiantv', - data_mode: 'json', - country: 'cn', - lang: 'zh', - notify_url: 'https://tajian.tv/test/' + uuid: 'herounion_demo', + url: 'https://v.douyin.com/xxx', + platform: 'douyin', + contract: 'tajiantv', + data_mode: 'json', + country: 'cn', + lang: 'zh', + notify_url: 'https://tajian.tv/test/' }; let token = 'hello#world!'; params.sign = common.sign(params, token); @@ -87,14 +87,14 @@ test('HeroUnion create task test', async (t) => { test('HeroUnion task query test', async (t) => { let params = { - uuid: 'herounion_demo', - url: 'https://v.douyin.com/yyy', - platform: 'douyin', - contract: 'tajiantv', - data_mode: 'json', - country: 'cn', - lang: 'zh', - notify_url: 'http://127.0.0.1:8080/test/' + uuid: 'herounion_demo', + url: 'https://v.douyin.com/yyy', + platform: 'douyin', + contract: 'tajiantv', + data_mode: 'json', + country: 'cn', + lang: 'zh', + notify_url: 'http://127.0.0.1:8080/test/' }; let token = 'hello#world!'; params.sign = common.sign(params, token); @@ -136,11 +136,11 @@ test('HeroUnion task query test', async (t) => { test('HeroUnion get waiting task test', async (t) => { //case 1 let params = { - platforms: 'douyin,kuaishou,xigua,bilibili', - contracts: 'tajiantv', - data_mode: 'json', - country: 'cn', - lang: 'zh' + platforms: 'douyin,kuaishou,xigua,bilibili', + contracts: 'tajiantv', + data_mode: 'json', + country: 'cn', + lang: 'zh' }; let api = 'http://127.0.0.1:8080/api/gettask/'; @@ -169,11 +169,11 @@ test('HeroUnion get waiting task test', async (t) => { test('HeroUnion task data save test', async (t) => { let params = { - platforms: 'douyin,kuaishou,xigua,bilibili', - contracts: 'tajiantv', - data_mode: 'json', - country: 'cn', - lang: 'zh' + platforms: 'douyin,kuaishou,xigua,bilibili', + contracts: 'tajiantv', + data_mode: 'json', + country: 'cn', + lang: 'zh' }; let api = 'http://127.0.0.1:8080/api/gettask/';