machete_hero/spider.mjs

/**
 * 爬虫主程序
 * 负责监听任务目录里的新任务，并自动抓取数据保存到数据目录。
 */
import configs from './config.mjs';
import common from './lib/common.mjs';
import TaskMoniter from "./lib/taskMoniter.mjs";
import TaJian from "./lib/tajian.mjs";
import HeroBot from "./lib/heroBot.mjs";

import Douyin from './bot/Douyin.mjs';
import Kuaishou from './bot/Kuaishou.mjs';
import Xigua from './bot/Xigua.mjs';
import Bilibili from './bot/Bilibili.mjs';

import cron from 'node-cron';

(async () => {

    const taskMoniter = new TaskMoniter(configs.task_list_dir);
    const tajian = new TaJian(configs.data_save_dir);

    taskMoniter.run();      //监控新任务

    //HeroUnion英雄联盟对接
    let heroUnionConfig = configs.herounion;
    let heroBot = new HeroBot(
            heroUnionConfig.server_url,
            heroUnionConfig.name,
            heroUnionConfig.description,
            heroUnionConfig.platforms,
            heroUnionConfig.contracts,
            heroUnionConfig.country,
            heroUnionConfig.lang,
            heroUnionConfig.contact,
            heroUnionConfig.data_mode
        );


    //配置本地cloud server地址，cloud安装参考：./install_cloud.sh
    const heroCloudServer = typeof(configs.cloud_server) != 'undefined' && configs.cloud_server ? configs.cloud_server : '';

    //spider run
    let spider_is_running = false;
    const task_check_time = 20;     //每 20 秒抓取一次
    const task_auto_run = cron.schedule(`*/${task_check_time} * * * * *`, async () =>  {
        if (spider_is_running == true) {return false;}      //避免同时执行多个爬虫任务

        const task = taskMoniter.getNewTask();
        if (!task) {return false;}

        const botName = common.getBotName(task.url);
        console.log('New task %s handle by bot %s.', task.url, botName);
        let bot = null;
        switch (botName) {
            case 'douyin':
                bot = new Douyin(heroCloudServer);
                break;
            case 'kuaishou':
                bot = new Kuaishou(heroCloudServer);
                break;
            case 'xigua':
                bot = new Xigua(heroCloudServer);
                break;
            case 'bilibili':
                bot = new Bilibili(heroCloudServer);
                break;
        }

        if (bot) {
            spider_is_running = true;

            taskMoniter.setTaskRunning(task.id);
            const data = await bot.scrap(task.url);
            //console.log('Data got by bot', data);

            if (typeof(data.done) != 'undefined' && data.done == true) {
                if (
                    await tajian.saveUrlShortcut(task.id, data)
                    && await tajian.saveDescriptionFiles(task.id, data)
                ) {
                    taskMoniter.setTaskDone(task.id);

                    //保存数据到HeroUnion联盟
                    if (typeof(task.from) != 'undefined' && task.from == 'HeroUnion') {
                        heroBot.saveTaskData(task.id, task.token, data);
                    }
                }else {
                    taskMoniter.setTaskFailed(task.id);
                }
            }else {
                taskMoniter.setTaskFailed(task.id);
            }

            spider_is_running = false;
        }else {
            console.error('No bot matched with url %s', task.url);
            taskMoniter.setTaskRunning(task.id);
            taskMoniter.setTaskFailed(task.id);
        }
    }, {
        scheduled: false
    });

    task_auto_run.start();
    console.log('[%s] Spider started.', common.getTimeString());


    //爬虫心跳上报
    const heartBeatFrequence = 5;    //5 分钟上报一次
    const heroUnionHeartBeat = cron.schedule(`*/${heartBeatFrequence} * * * *`, async () =>  {
        let status = spider_is_running ? 'busy' : 'idle';
        const res = await heroBot.heartBeat(status);
        console.log('HeroUnion bot heart beat result', res);
    }, {scheduled: false});
    heroUnionHeartBeat.start();
    console.log('[%s] HeroUnion bot heart beat started.', common.getTimeString());
})().catch(error => {
    console.error("Spider error got:\n%s", error);
    process.exit(1);
});