Browse Source

add herBot sdk for bots

master
filesite 8 months ago
parent
commit
de70260512
  1. 22
      common.mjs
  2. 150
      heroBot.mjs
  3. 3
      heroUnion.mjs
  4. 11
      test/common.test.mjs
  5. 58
      test/heroBot.test.mjs
  6. 70
      test/heroUnion.test.mjs

22
common.mjs

@ -43,28 +43,6 @@ class Common {
}, {}); }, {});
} }
joinDict(obj, glue, separator) { //dict拼接成字符串
if (typeof(glue) == 'undefined') {
glue = '=';
}
if (typeof(separator) == 'undefined') {
separator = '&';
}
return Object.keys(obj).map(function(key) {
let arr = [key];
let val = obj[key];
if (typeof(val) == 'string' || typeof(val) == 'number') {
arr.push(val);
}else if (typeof(val) == 'object') {
arr.push(JSON.stringify(val));
}
return arr.join(glue);
}).join(separator);
}
sign(params, token) { //对参数做MD5签名 sign(params, token) { //对参数做MD5签名
return md5( JSON.stringify(this.sortDict(params)) + token ); return md5( JSON.stringify(this.sortDict(params)) + token );
} }

150
heroBot.mjs

@ -0,0 +1,150 @@
/**
* HeroUnion Bot SDK
*/
import test from 'node:test';
import assert from 'node:assert';
import axios from 'axios';
import md5 from 'md5';
class HeroBot {
constructor(
server_url,
bot_name,
bot_description,
support_platforms,
support_contracts,
bot_country,
bot_lang,
bot_contact,
data_mode
) {
//必填参数
this.union_server = server_url;
this.name = bot_name;
this.description = bot_description;
this.platforms = support_platforms;
this.contracts = support_contracts;
//可选参数
this.country = typeof(bot_country) != 'undefined' ? bot_country : 'cn';
this.lang = typeof(bot_lang) != 'undefined' ? bot_lang : 'zh';
this.contact = typeof(bot_contact) != 'undefined' ? bot_contact : '';
this.data_mode = typeof(data_mode) != 'undefined' ? data_mode : 'json';
//联盟API地址
this.apis = {
"heartBeat": `${server_url}/api/onboard/`,
"getNewTask": `${server_url}/api/gettask/`,
"saveTaskData": `${server_url}/api/savetask/`,
};
//axios请求配置
this.axiosConfig = {
timeout: 8000, //请求超时
proxy: false //是否走代理
};
}
getTimestampInSeconds() {
return Math.floor(Date.now() / 1000);
}
sortDict(obj) { //dict按key排序
return Object.keys(obj).sort().reduce(function(result, key) {
result[key] = obj[key];
return result;
}, {});
}
sign(params, token) { //对参数做MD5签名
return md5( JSON.stringify(this.sortDict(params)) + token );
}
//向联盟发送心跳数据
async heartBeat(status) {
let params = {
name: this.name,
description: this.description,
status: status,
timestamp: this.getTimestampInSeconds(),
platforms: this.platforms,
contracts: this.contracts,
country: this.country,
lang: this.lang,
contact: this.contact
};
let response = null;
try {
response = await axios.post(this.apis.heartBeat, params, this.axiosConfig);
}catch(err) {
console.error('[ERROR] HeroBot heart beat failed: %s, api: %s, params: %s',
err,
this.apis.heartBeat,
JSON.stringify(params)
);
}
return response ? response.data : false;
}
//从联盟领取任务
async getNewTask() {
let params = {
platforms: this.platforms,
contracts: this.contracts,
data_mode: this.data_mode,
country: this.country,
lang: this.lang
};
let queryOption = this.axiosConfig;
queryOption.method = 'get';
queryOption.url = this.apis.getNewTask;
queryOption.params = params;
let response = null;
try {
response = await await axios(queryOption);
}catch(err) {
console.error('[ERROR] HeroBot get new task failed: %s, api: %s, params: %s',
err,
this.apis.getNewTask,
JSON.stringify(params)
);
}
return response && response.data.code == 1 ? response.data.task : false;
}
//回传任务数据给联盟
async saveTaskData(task_id, task_token, task_data) {
let params = {
name: this.name,
task_id: task_id,
task_result: task_data
};
params.sign = this.sign(params, task_token); //对参数进行签名
let response = null;
try {
response = await axios.post(this.apis.saveTaskData, params, this.axiosConfig);
}catch(err) {
console.error('[ERROR] HeroBot save task data failed: %s, api: %s, params: %s',
err,
this.apis.saveTaskData,
JSON.stringify(params)
);
}
return response ? response.data : false;
}
}
export default HeroBot;

3
heroUnion.mjs

@ -265,6 +265,8 @@ class HeroUnion {
this.tasks[taskIndex].status = 'failed'; this.tasks[taskIndex].status = 'failed';
this.tasks[taskIndex].error = 'Result is too large to save.'; this.tasks[taskIndex].error = 'Result is too large to save.';
common.error('Task %s save data failed by bot %s, data is too large.', id, bot_name);
return false; return false;
} }
@ -285,6 +287,7 @@ class HeroUnion {
this.tasks[taskIndex].status = 'done'; this.tasks[taskIndex].status = 'done';
common.log('Task %s save data done by bot %s.', id, bot_name);
done = true; done = true;
} }

11
test/common.test.mjs

@ -21,17 +21,6 @@ test('Common function sortDict test', (t) => {
assert.deepEqual(common.sortDict(params), expectRes); assert.deepEqual(common.sortDict(params), expectRes);
}); });
test('Common function joinDict test', (t) => {
let params = {
b: 2,
a: 1
};
const expectRes = "a=1&b=2";
assert.strictEqual(common.joinDict(common.sortDict(params)), expectRes);
});
test('Common function getConfigFromJsonFile test', async (t) => { test('Common function getConfigFromJsonFile test', async (t) => {
let filename = 'config.json'; let filename = 'config.json';
let config = await common.getConfigFromJsonFile(filename); let config = await common.getConfigFromJsonFile(filename);

58
test/heroBot.test.mjs

@ -0,0 +1,58 @@
/**
* HeroBot测试用例
* 执行此测试之前请先启动主程序在根目录执行命令npm start
*/
import test from 'node:test';
import assert from 'node:assert';
import axios from 'axios';
import common from '../common.mjs';
import HeroBot from '../heroBot.mjs';
let server_url = 'http://127.0.0.1:8080',
bot_name = 'test_hero_bot',
bot_description = '测试爬虫 test',
support_platforms = 'douyin,xigua',
support_contracts = 'tajiantv',
bot_country = 'cn',
bot_lang = 'zh',
bot_contact = 'https://tajian.tv',
data_mode = 'json';
let heroBot = new HeroBot(
server_url,
bot_name,
bot_description,
support_platforms,
support_contracts,
bot_country,
bot_lang,
bot_contact,
data_mode
);
test('Hero onboard test', async (t) => {
let status = 'idle';
const res = await heroBot.heartBeat(status);
console.log(res);
assert.ok(res);
assert.equal(res.code, 1);
});
test('Hero get task and data save test', async (t) => {
const task = await heroBot.getNewTask();
console.log(task);
assert.ok(task);
let task_data = {
"title": "标题测试:HeroUnion英雄联盟",
"description": "描述内容,联盟简介",
"others": "其它内容"
};
let res = await heroBot.saveTaskData(task.id, task.token, task_data);
console.log(res);
assert.equal(res.code, 1);
});

70
test/heroUnion.test.mjs

@ -25,15 +25,15 @@ test('HeroUnion api list test', async (t) => {
test('Hero onboard test', async (t) => { test('Hero onboard test', async (t) => {
let params = { let params = {
name: 'test_hero', name: 'test_hero',
description: 'Hero test 测试爬虫', description: 'Hero test 测试爬虫',
status: 'idle', status: 'idle',
timestamp: common.getTimestampInSeconds(), timestamp: common.getTimestampInSeconds(),
platforms: 'douyin,xigua', platforms: 'douyin,xigua',
contracts: 'tajiantv', contracts: 'tajiantv',
country: 'cn', country: 'cn',
lang: 'zh', lang: 'zh',
contact: 'https://tajian.tv' contact: 'https://tajian.tv'
}; };
let api = 'http://127.0.0.1:8080/api/onboard/'; let api = 'http://127.0.0.1:8080/api/onboard/';
@ -64,14 +64,14 @@ test('HeroUnion get heros test', async (t) => {
test('HeroUnion create task test', async (t) => { test('HeroUnion create task test', async (t) => {
let params = { let params = {
uuid: 'herounion_demo', uuid: 'herounion_demo',
url: 'https://v.douyin.com/xxx', url: 'https://v.douyin.com/xxx',
platform: 'douyin', platform: 'douyin',
contract: 'tajiantv', contract: 'tajiantv',
data_mode: 'json', data_mode: 'json',
country: 'cn', country: 'cn',
lang: 'zh', lang: 'zh',
notify_url: 'https://tajian.tv/test/' notify_url: 'https://tajian.tv/test/'
}; };
let token = 'hello#world!'; let token = 'hello#world!';
params.sign = common.sign(params, token); params.sign = common.sign(params, token);
@ -87,14 +87,14 @@ test('HeroUnion create task test', async (t) => {
test('HeroUnion task query test', async (t) => { test('HeroUnion task query test', async (t) => {
let params = { let params = {
uuid: 'herounion_demo', uuid: 'herounion_demo',
url: 'https://v.douyin.com/yyy', url: 'https://v.douyin.com/yyy',
platform: 'douyin', platform: 'douyin',
contract: 'tajiantv', contract: 'tajiantv',
data_mode: 'json', data_mode: 'json',
country: 'cn', country: 'cn',
lang: 'zh', lang: 'zh',
notify_url: 'http://127.0.0.1:8080/test/' notify_url: 'http://127.0.0.1:8080/test/'
}; };
let token = 'hello#world!'; let token = 'hello#world!';
params.sign = common.sign(params, token); params.sign = common.sign(params, token);
@ -136,11 +136,11 @@ test('HeroUnion task query test', async (t) => {
test('HeroUnion get waiting task test', async (t) => { test('HeroUnion get waiting task test', async (t) => {
//case 1 //case 1
let params = { let params = {
platforms: 'douyin,kuaishou,xigua,bilibili', platforms: 'douyin,kuaishou,xigua,bilibili',
contracts: 'tajiantv', contracts: 'tajiantv',
data_mode: 'json', data_mode: 'json',
country: 'cn', country: 'cn',
lang: 'zh' lang: 'zh'
}; };
let api = 'http://127.0.0.1:8080/api/gettask/'; let api = 'http://127.0.0.1:8080/api/gettask/';
@ -169,11 +169,11 @@ test('HeroUnion get waiting task test', async (t) => {
test('HeroUnion task data save test', async (t) => { test('HeroUnion task data save test', async (t) => {
let params = { let params = {
platforms: 'douyin,kuaishou,xigua,bilibili', platforms: 'douyin,kuaishou,xigua,bilibili',
contracts: 'tajiantv', contracts: 'tajiantv',
data_mode: 'json', data_mode: 'json',
country: 'cn', country: 'cn',
lang: 'zh' lang: 'zh'
}; };
let api = 'http://127.0.0.1:8080/api/gettask/'; let api = 'http://127.0.0.1:8080/api/gettask/';

Loading…
Cancel
Save