From 880b042807cc547e2318c97e62203a0c7845c054 Mon Sep 17 00:00:00 2001 From: filesite-io Date: Tue, 4 Jun 2024 21:10:51 +0800 Subject: [PATCH] v0.0.1 ready --- .gitignore | 3 + README.md | 45 ++++++++- conf/config.json | 15 +++ lib/common.mjs | 209 ++++++++++++++++++++++++++++++++++++++++++ log/README.md | 7 ++ monitor.mjs | 167 +++++++++++++++++++++++++++++++++ package.json | 23 +++++ server.mjs | 8 ++ test/monitor.test.mjs | 38 ++++++++ 9 files changed, 514 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 conf/config.json create mode 100644 lib/common.mjs create mode 100644 log/README.md create mode 100644 monitor.mjs create mode 100644 package.json create mode 100644 server.mjs create mode 100644 test/monitor.test.mjs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f44e58f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +package-lock.json +node_modules/ +log/*.log \ No newline at end of file diff --git a/README.md b/README.md index b491b1b..566be21 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,46 @@ # monit-via-herounion -Website monitor via HeroUnion. \ No newline at end of file +Website monitor via HeroUnion. +基于HeroUnion的网站监控程序。 + + +## 使用方法 + +1. 下载源码： +``` +git clone "https://git.filesite.io/filesite/monit-via-herounion.git" +``` + +2. 安装node依赖包 +``` +npm install +``` + +3. 配置需要监控的网站 + +修改文件：conf/config.json + +在**monit_urls**里添加网址，例如： +``` +"https://tajian.tv", +"https://filesite.io" +``` + +4. 启动监控程序 +``` +npm start +``` + + +## 查看监控结果 + +在log/目录下会生成两个日志文件： + +* ok.log - 成功访问日志 +* fail.log - 访问失败日志 + + +## HeroUnion账号获取 + +请打开网站，查看底部的联系方式： +[FileSite.io](https://filesite.io) diff --git a/conf/config.json b/conf/config.json new file mode 100644 index 0000000..6c620a7 --- /dev/null +++ b/conf/config.json @@ -0,0 +1,15 @@ +{ + "systemLogDir": "log/", + + "herounion_server": "https://herounion.filesite.io", + "herounion_id": "herounion_demo", + "herounion_token": "hello#world!", + + "request_timeout": 10, + "monitFrequence": 10, + "resultQueryFrequence": 1, + "monit_urls": [ + "https://tajian.tv", + "https://filesite.io" + ] +} \ No newline at end of file diff --git a/lib/common.mjs b/lib/common.mjs new file mode 100644 index 0000000..1fa0e4f --- /dev/null +++ b/lib/common.mjs @@ -0,0 +1,209 @@ +/** + * 公用方法 + */ + +import fs from 'node:fs'; +import { readdir, readFile, appendFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { Buffer } from 'node:buffer'; +import axios from 'axios'; +import md5 from 'md5'; + +class Common { + + //构造函数，设置默认配置 + constructor() { + this.configDir = resolve('conf/'); + } + + byteSize(str) { + return Buffer.byteLength(str, 'utf8'); + } + + getTimestamp() { + return Math.floor(Date.now()); + } + + getTimestampInSeconds() { + return Math.floor(Date.now() / 1000); + } + + getLocalTimeString(locales, timezone) { + if (typeof(locales) == 'undefined' || !locales) { + locales = 'zh-Hans-CN'; + } + + if (typeof(timezone) == 'undefined' || !timezone) { + timezone = 'Asia/Shanghai'; + } + + let date = new Date(); + let option = {"timeZone": timezone}; + return date.toLocaleString(locales, option); + } + + sortDict(obj) { //dict按key排序 + return Object.keys(obj).sort().reduce(function(result, key) { + result[key] = obj[key]; + return result; + }, {}); + } + + sign(params, token) { //对参数做MD5签名 + return md5( JSON.stringify(this.sortDict(params)) + token ); + } + + //从conf/目录读取配置文件内容 + async getConfigFromJsonFile(filename) { + let data = null; + + let filePath = this.configDir + `/${filename}`; + if (fs.existsSync(filePath)) { + try { + const contents = await readFile(filePath, { encoding: 'utf8' }); + if (contents) { + data = JSON.parse(contents); + } + } catch (err) { + console.error(`[FAILED] get config content from %s failed, error: %s`, filePath, err.message); + } + }else { + console.error("[ERROR] file %s not exist.", filePath); + } + + return data; + } + + getLogArguments() { + let args = []; + let localTime = this.getLocalTimeString('zh-Hans-CN', 'Asia/Shanghai'); + + if (arguments[0]) { + let logFormat = `[%s] ${arguments[0]}`; + args.push(logFormat); + args.push(localTime); + } + + if (arguments && arguments.length > 1) { + for (const index in arguments) { + if (index > 0) { + args.push(arguments[index]); + } + } + } + + return args; + } + + log() { + let args = this.getLogArguments.apply(this, arguments); + console.log.apply(this, args); + return args; + } + + info() { + let args = this.getLogArguments.apply(this, arguments); + console.info.apply(this, args); + return args; + } + + warn() { + let args = this.getLogArguments.apply(this, arguments); + console.warn.apply(this, args); + return args; + } + + error() { + let args = this.getLogArguments.apply(this, arguments); + console.error.apply(this, args); + return args; + } + + //保存log到指定文件 + async saveLog(filePath, content) { + let saved = false; + + try { + let saveRes = await appendFile(filePath, content); + if (saveRes == undefined) { + saved = true; + } + } catch (err) { + console.error(`Log save to %s failed: %s`, filePath, err.message); + } + + return saved; + } + + async delay(seconds) { + await setTimeout(seconds * 1000); + } + + //创建HeroUnion的爬虫任务 + //参考： + //* https://github.com/filesite-io/herounion + //* https://github.com/filesite-io/machete_hero + async createHeroUnionTask(targetUrl, notifyUrl, configs) { + let params = { + uuid: typeof(configs.herounion_id) != 'undefined' && configs.herounion_id ? configs.herounion_id : 'herounion_demo', + url: targetUrl, + platform: 'website', //爬虫支持的平台：任意网站 + contract: 'tajiantv', //爬虫支持的合约：tajiantv + data_mode: 'json', + country: 'cn', + lang: 'zh', + notify_url: notifyUrl + }; + let token = typeof(configs.herounion_token) != 'undefined' && configs.herounion_token ? configs.herounion_token : 'hello#world!'; + params.sign = this.sign(params, token); + + let api = typeof(configs.herounion_server) != 'undefined' && configs.herounion_server ? + configs.herounion_server + '/api/newtask/' : 'http://127.0.0.1:8080/api/newtask/'; + + const axiosConfig = { + timeout: typeof(configs.request_timeout) != 'undefined' && configs.request_timeout ? configs.request_timeout*1000 : 10000, + proxy: false + }; + + const response = await axios.post(api, params, axiosConfig); + if (response.status == 200) { + return response.data; + } + + return false; + } + + //查询HeroUnion任务结果 + async queryHeroUnionTask(task_id, configs) { + let params = { + uuid: typeof(configs.herounion_id) != 'undefined' && configs.herounion_id ? configs.herounion_id : 'herounion_demo', + task_id: task_id + }; + let token = typeof(configs.herounion_token) != 'undefined' && configs.herounion_token ? configs.herounion_token : 'hello#world!'; + params.sign = this.sign(params, token); + + let api = typeof(configs.herounion_server) != 'undefined' && configs.herounion_server ? + configs.herounion_server + '/api/querytask/' : 'http://127.0.0.1:8080/api/querytask/'; + + const axiosConfig = { + timeout: typeof(configs.request_timeout) != 'undefined' && configs.request_timeout ? configs.request_timeout*1000 : 10000, + proxy: false + }; + + let queryOption = axiosConfig; + queryOption.method = 'get'; + queryOption.url = api; + queryOption.params = params; + + const response = await axios(queryOption); + if (response.status == 200) { + return response.data; + } + + return false; + } + +} + +let commonFuns = new Common(); +export default commonFuns; \ No newline at end of file diff --git a/log/README.md b/log/README.md new file mode 100644 index 0000000..52edf45 --- /dev/null +++ b/log/README.md @@ -0,0 +1,7 @@ + +# 日志存放目录 + +日志文件命名格式： +``` +xxx.log +``` \ No newline at end of file diff --git a/monitor.mjs b/monitor.mjs new file mode 100644 index 0000000..fb62b21 --- /dev/null +++ b/monitor.mjs @@ -0,0 +1,167 @@ +/** + * Monitor via HeroUnion + */ + +import path from 'node:path'; +import cron from 'node-cron'; +import axios from 'axios'; +import common from './lib/common.mjs'; +import md5 from 'md5'; + +class Monitor { + + //构造函数，设置默认配置 + constructor() { + this.config = null; + + //默认配置 + this.systemLogDir = 'log/'; //系统日志保存目录 + this.reloadConfigFrequence = 5; //单位：分钟，配置重新加载时间间隔 + this.monitFrequence = 10; //单位：分钟，检测时间间隔 + this.resultQueryFrequence = 1; //单位：分钟，检测任务结果查询时间间隔 + + this.tasks = []; //HeroUnion检测任务队列 + } + + async getConfig(forceReload) { + const _self = this; + + if ( !this.config || (typeof(forceReload) != 'undefined' && forceReload) ) { + let config = await common.getConfigFromJsonFile('config.json'); + + //覆盖默认配置 + for (const key in config) { + if (typeof(_self[key]) != 'undefined') { + _self[key] = config[key]; + } + } + + this.config = config; + } + + return this.config; + } + + //自动重新加载配置文件 + autoReloadConfigs() { + const _self = this; + + const frequence = typeof(this.config.reloadConfigFrequence) != 'undefined' + && this.config.reloadConfigFrequence ? this.config.reloadConfigFrequence : 5; //5 分钟重新加载一次 + const cronjob = cron.schedule(`*/${frequence} * * * *`, () => { + const forceReload = true; + _self.getConfig(forceReload); + }, { + scheduled: false + }); + + cronjob.start(); + common.log('Cronjob of config auto reload started.'); + } + + //自动向HeroUnion提交检测任务 + autoCreateMonitTask() { + const _self = this; + + const frequence = typeof(this.config.monitFrequence) != 'undefined' + && this.config.monitFrequence ? this.config.monitFrequence : 10; //10 分钟检测一次 + const cronjob = cron.schedule(`*/${frequence} * * * *`, async () => { + let configs = await _self.getConfig(); + if (configs.monit_urls.length == 0) { + console.error("No monit urls"); + return false; + } + + let taskRes; + let total = configs.monit_urls.length; + for (let i=0; i item.url == configs.monit_urls[i] && item.stats != 'done')) {continue;} + + console.log("Checking url %s ...", configs.monit_urls[i]); + taskRes = await common.createHeroUnionTask(configs.monit_urls[i], '', configs); + if (taskRes && taskRes.code == 1) { + _self.tasks.push(taskRes.task); + console.log("Monit task", taskRes.task); + }else { + console.error("Monit task create failed", taskRes); + } + } + }, { + scheduled: false + }); + + cronjob.start(); + common.log('Cronjob of url monit started.'); + } + + async queryTasks() { + const _self = this; + let configs = await _self.getConfig(); + + console.log('Task number', _self.tasks.length); + + let task, taskRes; + for(let index = 0; index < _self.tasks.length; index ++) { + task = _self.tasks[index]; + if (task.status == 'done') {continue;} + + console.log('Query task result', task); + taskRes = await common.queryHeroUnionTask(task.id, configs); + if (taskRes && taskRes.code == 1) { + console.log("Task result", taskRes); + _self.tasks[index] = taskRes.task; + + common.log('Connect success, url: %s, task id: %s', task.url, task.id); + + let currentTime = common.getLocalTimeString(); + let logFile = path.resolve(_self.systemLogDir) + '/ok.log'; + common.saveLog(logFile, `[${currentTime}] Url request success: ${task.url}, task id: ${task.id}\n`); + }else { + console.error("Monit task query failed", taskRes); + + //TODO: 写入日志，或发送告警 + common.error('Connect warning, url: %s, task id: %s', task.url, task.id); + + let currentTime = common.getLocalTimeString(); + let logFile = path.resolve(_self.systemLogDir) + '/fail.log'; + common.saveLog(logFile, `[${currentTime}] Url request failed: ${task.url}, task id: ${task.id}\n`); + } + } + + //更新tasks，去掉已完成的 + _self.tasks = _self.tasks.filter((item) => item.status != 'done'); + } + + //自动查询监控任务结果 + autoQueryTaskResult() { + const _self = this; + + const frequence = typeof(this.config.resultQueryFrequence) != 'undefined' + && this.config.resultQueryFrequence ? this.config.resultQueryFrequence : 5; //5 分钟检测一次 + const cronjob = cron.schedule(`*/${frequence} * * * *`, async () => { + if (_self.tasks.length == 0) { + console.error("No tasks"); + return false; + } + + await _self.queryTasks(); + }, { + scheduled: false + }); + + cronjob.start(); + common.log('Cronjob of monit task result query started.'); + } + + //初始化 + async init() { + await this.getConfig(); + this.autoReloadConfigs(); + this.autoCreateMonitTask(); + this.autoQueryTaskResult(); + } + + +} + +export default Monitor; \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..ae23a6b --- /dev/null +++ b/package.json @@ -0,0 +1,23 @@ +{ + "name": "@filesite/monit-via-herounion", + "description": "Website monitor via HeroUnion: https://herounion.filesite.io.", + "version": "0.0.1", + "author": "filesite.io", + "repository": { + "type": "git", + "url": "https://git.filesite.io/filesite/monit-via-herounion.git" + }, + "license": "MIT", + "type": "module", + "engines": { + "node": ">=18" + }, + "dependencies": { + "node-cron": "^3.0.2", + "axios": "^1.3.3", + "md5": "^2.3.0" + }, + "scripts": { + "start": "node server.mjs" + } +} \ No newline at end of file diff --git a/server.mjs b/server.mjs new file mode 100644 index 0000000..6ec1ee0 --- /dev/null +++ b/server.mjs @@ -0,0 +1,8 @@ +import common from './lib/common.mjs'; +import Monitor from './monitor.mjs'; + +(async () => { + let monitor = new Monitor(); + await monitor.init(); + +})(); \ No newline at end of file diff --git a/test/monitor.test.mjs b/test/monitor.test.mjs new file mode 100644 index 0000000..2f817a5 --- /dev/null +++ b/test/monitor.test.mjs @@ -0,0 +1,38 @@ +/** + * Monitor测试用例 + */ + +import test from 'node:test'; +import assert from 'node:assert'; +import axios from 'axios'; +import common from '../lib/common.mjs'; +import Monitor from '../monitor.mjs'; + +const axiosConfig = { + timeout: 5000, + proxy: false +}; + +test('common.createHeroUnionTask test', async (t) => { + let configFile = 'config_test.json'; + let configs = await common.getConfigFromJsonFile(configFile); + console.log("configs from %s", configFile, configs); + + let targetUrl = 'https://tajian.tv'; + let notifyUrl = ''; + + let taskRes = await common.createHeroUnionTask(targetUrl, notifyUrl, configs); + console.log("Task create result", taskRes); + assert.equal(taskRes.code, 1); +}); + +test('common.queryHeroUnionTask test', async (t) => { + let configFile = 'config_test.json'; + let configs = await common.getConfigFromJsonFile(configFile); + console.log("configs from %s", configFile, configs); + + let task_id = 'machete_tajian_1717495858207'; + let taskRes = await common.queryHeroUnionTask(task_id, configs); + console.log("Task data", taskRes); + assert.equal(taskRes.code, 1); +}); \ No newline at end of file