v0.0.1 ready

.gitignore

@@ -0,0 +1,3 @@
+package-lock.json
+node_modules/
+log/*.log

README.md

@@ -1,3 +1,46 @@
 # monit-via-herounion
 
-Website monitor via HeroUnion.
+Website monitor via HeroUnion.
+基于HeroUnion的网站监控程序。
+
+
+## 使用方法
+
+1. 下载源码：
+```
+git clone "https://git.filesite.io/filesite/monit-via-herounion.git"
+```
+
+2. 安装node依赖包
+```
+npm install
+```
+
+3. 配置需要监控的网站
+
+修改文件：conf/config.json
+
+在**monit_urls**里添加网址，例如：
+```
+"https://tajian.tv",
+"https://filesite.io"
+```
+
+4. 启动监控程序
+```
+npm start
+```
+
+
+## 查看监控结果
+
+在log/目录下会生成两个日志文件：
+
+* ok.log 	- 成功访问日志
+* fail.log 	- 访问失败日志
+
+
+## HeroUnion账号获取
+
+请打开网站，查看底部的联系方式：
+[FileSite.io](https://filesite.io)

conf/config.json

@@ -0,0 +1,15 @@
+{
+    "systemLogDir": "log/",
+
+    "herounion_server": "https://herounion.filesite.io",
+    "herounion_id": "herounion_demo",
+    "herounion_token": "hello#world!",
+
+    "request_timeout": 10,
+    "monitFrequence": 10,
+    "resultQueryFrequence": 1,
+    "monit_urls": [
+        "https://tajian.tv",
+        "https://filesite.io"
+    ]
+}

lib/common.mjs

@@ -0,0 +1,209 @@
+/**
+ * 公用方法
+ */
+
+import fs from 'node:fs';
+import { readdir, readFile, appendFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { Buffer } from 'node:buffer';
+import axios from 'axios';
+import md5 from 'md5';
+
+class Common {
+
+    //构造函数，设置默认配置
+    constructor() {
+        this.configDir = resolve('conf/');
+    }
+
+    byteSize(str) {
+        return Buffer.byteLength(str, 'utf8');
+    }
+
+    getTimestamp() {
+        return Math.floor(Date.now());
+    }
+
+    getTimestampInSeconds() {
+        return Math.floor(Date.now() / 1000);
+    }
+
+    getLocalTimeString(locales, timezone) {
+        if (typeof(locales) == 'undefined' || !locales) {
+            locales = 'zh-Hans-CN';
+        }
+
+        if (typeof(timezone) == 'undefined' || !timezone) {
+            timezone = 'Asia/Shanghai';
+        }
+
+        let date = new Date();
+        let option = {"timeZone": timezone};
+        return date.toLocaleString(locales, option);
+    }
+
+    sortDict(obj) {                         //dict按key排序
+        return Object.keys(obj).sort().reduce(function(result, key) {
+            result[key] = obj[key];
+            return result;
+        }, {});
+    }
+
+    sign(params, token) {                    //对参数做MD5签名
+        return md5( JSON.stringify(this.sortDict(params)) + token );
+    }
+
+    //从conf/目录读取配置文件内容
+    async getConfigFromJsonFile(filename) {
+        let data = null;
+
+        let filePath = this.configDir + `/${filename}`;
+        if (fs.existsSync(filePath)) {
+            try {
+                const contents = await readFile(filePath, { encoding: 'utf8' });
+                if (contents) {
+                    data = JSON.parse(contents);
+                }
+            } catch (err) {
+                console.error(`[FAILED] get config content from %s failed, error: %s`, filePath, err.message);
+            }
+        }else {
+            console.error("[ERROR] file %s not exist.", filePath);
+        }
+
+        return data;
+    }
+
+    getLogArguments() {
+        let args = [];
+        let localTime = this.getLocalTimeString('zh-Hans-CN', 'Asia/Shanghai');
+
+        if (arguments[0]) {
+            let logFormat = `[%s] ${arguments[0]}`;
+            args.push(logFormat);
+            args.push(localTime);
+        }
+
+        if (arguments && arguments.length > 1) {
+            for (const index in arguments) {
+                if (index > 0) {
+                    args.push(arguments[index]);
+                }
+            }
+        }
+
+        return args;
+    }
+
+    log() {
+        let args = this.getLogArguments.apply(this, arguments);
+        console.log.apply(this, args);
+        return args;
+    }
+
+    info() {
+        let args = this.getLogArguments.apply(this, arguments);
+        console.info.apply(this, args);
+        return args;
+    }
+
+    warn() {
+        let args = this.getLogArguments.apply(this, arguments);
+        console.warn.apply(this, args);
+        return args;
+    }
+
+    error() {
+        let args = this.getLogArguments.apply(this, arguments);
+        console.error.apply(this, args);
+        return args;
+    }
+
+    //保存log到指定文件
+    async saveLog(filePath, content) {
+        let saved = false;
+
+        try {
+            let saveRes = await appendFile(filePath, content);
+            if (saveRes == undefined) {
+                saved = true;
+            }
+        } catch (err) {
+            console.error(`Log save to %s failed: %s`, filePath, err.message);
+        }
+
+        return saved;
+    }
+
+    async delay(seconds) {
+        await setTimeout(seconds * 1000);
+    }
+
+    //创建HeroUnion的爬虫任务
+    //参考：
+    //* https://github.com/filesite-io/herounion
+    //* https://github.com/filesite-io/machete_hero
+    async createHeroUnionTask(targetUrl, notifyUrl, configs) {
+        let params = {
+            uuid: typeof(configs.herounion_id) != 'undefined' && configs.herounion_id ? configs.herounion_id : 'herounion_demo',
+            url: targetUrl,
+            platform: 'website',    //爬虫支持的平台：任意网站
+            contract: 'tajiantv',   //爬虫支持的合约：tajiantv
+            data_mode: 'json',
+            country: 'cn',
+            lang: 'zh',
+            notify_url: notifyUrl
+        };
+        let token = typeof(configs.herounion_token) != 'undefined' && configs.herounion_token ? configs.herounion_token : 'hello#world!';
+        params.sign = this.sign(params, token);
+
+        let api = typeof(configs.herounion_server) != 'undefined' && configs.herounion_server ?
+                    configs.herounion_server + '/api/newtask/' : 'http://127.0.0.1:8080/api/newtask/';
+
+        const axiosConfig = {
+            timeout: typeof(configs.request_timeout) != 'undefined' && configs.request_timeout ? configs.request_timeout*1000 : 10000,
+            proxy: false
+        };
+
+        const response = await axios.post(api, params, axiosConfig);
+        if (response.status == 200) {
+            return response.data;
+        }
+
+        return false;
+    }
+
+    //查询HeroUnion任务结果
+    async queryHeroUnionTask(task_id, configs) {
+        let params = {
+            uuid: typeof(configs.herounion_id) != 'undefined' && configs.herounion_id ? configs.herounion_id : 'herounion_demo',
+            task_id: task_id
+        };
+        let token = typeof(configs.herounion_token) != 'undefined' && configs.herounion_token ? configs.herounion_token : 'hello#world!';
+        params.sign = this.sign(params, token);
+
+        let api = typeof(configs.herounion_server) != 'undefined' && configs.herounion_server ?
+                    configs.herounion_server + '/api/querytask/' : 'http://127.0.0.1:8080/api/querytask/';
+
+        const axiosConfig = {
+            timeout: typeof(configs.request_timeout) != 'undefined' && configs.request_timeout ? configs.request_timeout*1000 : 10000,
+            proxy: false
+        };
+
+        let queryOption = axiosConfig;
+        queryOption.method = 'get';
+        queryOption.url = api;
+        queryOption.params = params;
+
+        const response = await axios(queryOption);
+        if (response.status == 200) {
+            return response.data;
+        }
+
+        return false;
+    }
+
+}
+
+let commonFuns = new Common();
+export default commonFuns;

log/README.md

@@ -0,0 +1,7 @@
+
+# 日志存放目录
+
+日志文件命名格式：
+```
+xxx.log
+```

monitor.mjs

@@ -0,0 +1,167 @@
+/**
+ * Monitor via HeroUnion
+ */
+
+import path from 'node:path';
+import cron from 'node-cron';
+import axios from 'axios';
+import common from './lib/common.mjs';
+import md5 from 'md5';
+
+class Monitor {
+
+    //构造函数，设置默认配置
+    constructor() {
+        this.config = null;
+
+        //默认配置
+        this.systemLogDir = 'log/';                             //系统日志保存目录
+        this.reloadConfigFrequence = 5;                         //单位：分钟，配置重新加载时间间隔
+        this.monitFrequence = 10;                                 //单位：分钟，检测时间间隔
+        this.resultQueryFrequence = 1;                             //单位：分钟，检测任务结果查询时间间隔
+
+        this.tasks = [];                                        //HeroUnion检测任务队列
+    }
+
+    async getConfig(forceReload) {
+        const _self = this;
+
+        if ( !this.config || (typeof(forceReload) != 'undefined' && forceReload) ) {
+            let config = await common.getConfigFromJsonFile('config.json');
+
+            //覆盖默认配置
+            for (const key in config) {
+                if (typeof(_self[key]) != 'undefined') {
+                    _self[key] = config[key];
+                }
+            }
+
+            this.config = config;
+        }
+
+        return this.config;
+    }
+
+    //自动重新加载配置文件
+    autoReloadConfigs() {
+        const _self = this;
+
+        const frequence = typeof(this.config.reloadConfigFrequence) != 'undefined'
+                            && this.config.reloadConfigFrequence ? this.config.reloadConfigFrequence : 5;    //5 分钟重新加载一次
+        const cronjob = cron.schedule(`*/${frequence} * * * *`, () =>  {
+            const forceReload = true;
+            _self.getConfig(forceReload);
+        }, {
+            scheduled: false
+        });
+
+        cronjob.start();
+        common.log('Cronjob of config auto reload started.');
+    }
+
+    //自动向HeroUnion提交检测任务
+    autoCreateMonitTask() {
+        const _self = this;
+
+        const frequence = typeof(this.config.monitFrequence) != 'undefined'
+                            && this.config.monitFrequence ? this.config.monitFrequence : 10;    //10 分钟检测一次
+        const cronjob = cron.schedule(`*/${frequence} * * * *`, async () =>  {
+            let configs = await _self.getConfig();
+            if (configs.monit_urls.length == 0) {
+                console.error("No monit urls");
+                return false;
+            }
+
+            let taskRes;
+            let total = configs.monit_urls.length;
+            for (let i=0; i<total; i++) {
+            	if (_self.tasks.find((item) => item.url == configs.monit_urls[i] && item.stats != 'done')) {continue;}
+
+                console.log("Checking url %s ...", configs.monit_urls[i]);
+                taskRes = await common.createHeroUnionTask(configs.monit_urls[i], '', configs);
+                if (taskRes && taskRes.code == 1) {
+                    _self.tasks.push(taskRes.task);
+                    console.log("Monit task", taskRes.task);
+                }else {
+                    console.error("Monit task create failed", taskRes);
+                }
+            }
+        }, {
+            scheduled: false
+        });
+
+        cronjob.start();
+        common.log('Cronjob of url monit started.');
+    }
+
+    async queryTasks() {
+    	const _self = this;
+        let configs = await _self.getConfig();
+
+    	console.log('Task number', _self.tasks.length);
+
+    	let task, taskRes;
+        for(let index = 0; index < _self.tasks.length; index ++) {
+        	task = _self.tasks[index];
+        	if (task.status == 'done') {continue;}
+
+            console.log('Query task result', task);
+            taskRes = await common.queryHeroUnionTask(task.id, configs);
+        	if (taskRes && taskRes.code == 1) {
+                console.log("Task result", taskRes);
+                _self.tasks[index] = taskRes.task;
+
+                common.log('Connect success, url: %s, task id: %s', task.url, task.id);
+
+                let currentTime = common.getLocalTimeString();
+                let logFile = path.resolve(_self.systemLogDir) + '/ok.log';
+                common.saveLog(logFile, `[${currentTime}] Url request success: ${task.url}, task id: ${task.id}\n`);
+            }else {
+                console.error("Monit task query failed", taskRes);
+
+                //TODO: 写入日志，或发送告警
+                common.error('Connect warning, url: %s, task id: %s', task.url, task.id);
+
+                let currentTime = common.getLocalTimeString();
+                let logFile = path.resolve(_self.systemLogDir) + '/fail.log';
+                common.saveLog(logFile, `[${currentTime}] Url request failed: ${task.url}, task id: ${task.id}\n`);
+            }
+        }
+
+        //更新tasks，去掉已完成的
+        _self.tasks = _self.tasks.filter((item) => item.status != 'done');
+    }
+
+    //自动查询监控任务结果
+    autoQueryTaskResult() {
+        const _self = this;
+
+        const frequence = typeof(this.config.resultQueryFrequence) != 'undefined'
+                            && this.config.resultQueryFrequence ? this.config.resultQueryFrequence : 5;    //5 分钟检测一次
+        const cronjob = cron.schedule(`*/${frequence} * * * *`, async () =>  {
+            if (_self.tasks.length == 0) {
+                console.error("No tasks");
+                return false;
+            }
+
+            await _self.queryTasks();
+        }, {
+            scheduled: false
+        });
+
+        cronjob.start();
+        common.log('Cronjob of monit task result query started.');
+    }
+
+    //初始化
+    async init() {
+        await this.getConfig();
+        this.autoReloadConfigs();
+        this.autoCreateMonitTask();
+        this.autoQueryTaskResult();
+    }
+
+
+}
+
+export default Monitor;

package.json

@@ -0,0 +1,23 @@
+{
+    "name": "@filesite/monit-via-herounion",
+    "description": "Website monitor via HeroUnion: https://herounion.filesite.io.",
+    "version": "0.0.1",
+    "author": "filesite.io",
+    "repository": {
+        "type": "git",
+        "url": "https://git.filesite.io/filesite/monit-via-herounion.git"
+    },
+    "license": "MIT",
+    "type": "module",
+    "engines": {
+        "node": ">=18"
+    },
+    "dependencies": {
+        "node-cron": "^3.0.2",
+        "axios": "^1.3.3",
+        "md5": "^2.3.0"
+    },
+    "scripts": {
+        "start": "node server.mjs"
+    }
+}

server.mjs

@@ -0,0 +1,8 @@
+import common from './lib/common.mjs';
+import Monitor from './monitor.mjs';
+
+(async () => {
+    let monitor = new Monitor();
+    await monitor.init();
+
+})();

test/monitor.test.mjs

@@ -0,0 +1,38 @@
+/**
+ * Monitor测试用例
+ */
+
+import test from 'node:test';
+import assert from 'node:assert';
+import axios from 'axios';
+import common from '../lib/common.mjs';
+import Monitor from '../monitor.mjs';
+
+const axiosConfig = {
+    timeout: 5000,
+    proxy: false
+};
+
+test('common.createHeroUnionTask test', async (t) => {
+    let configFile = 'config_test.json';
+    let configs = await common.getConfigFromJsonFile(configFile);
+    console.log("configs from %s", configFile, configs);
+
+    let targetUrl = 'https://tajian.tv';
+    let notifyUrl = '';
+
+    let taskRes = await common.createHeroUnionTask(targetUrl, notifyUrl, configs);
+    console.log("Task create result", taskRes);
+    assert.equal(taskRes.code, 1);
+});
+
+test('common.queryHeroUnionTask test', async (t) => {
+    let configFile = 'config_test.json';
+    let configs = await common.getConfigFromJsonFile(configFile);
+    console.log("configs from %s", configFile, configs);
+
+    let task_id = 'machete_tajian_1717495858207';
+    let taskRes = await common.queryHeroUnionTask(task_id, configs);
+    console.log("Task data", taskRes);
+    assert.equal(taskRes.code, 1);
+});