From debbf606017cf4fd667f4bd131521a24ebe6e3fc Mon Sep 17 00:00:00 2001 From: filesite Date: Wed, 13 Sep 2023 21:33:02 +0800 Subject: [PATCH] add client log plugin --- bot/Bilibili.mjs | 6 +++++- bot/Douyin.mjs | 6 +++++- bot/HeroBot.mjs | 2 ++ bot/Kuaishou.mjs | 6 +++++- bot/Xigua.mjs | 6 +++++- config.mjs | 4 ++-- plugin/ClientLogPlugin.mjs | 22 ++++++++++++++++++++++ test/scrap_test.mjs | 2 ++ 8 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 plugin/ClientLogPlugin.mjs diff --git a/bot/Bilibili.mjs b/bot/Bilibili.mjs index fa5a6b1..4a71bb2 100644 --- a/bot/Bilibili.mjs +++ b/bot/Bilibili.mjs @@ -1,11 +1,12 @@ import Hero from '@ulixee/hero'; import configs from '../config.mjs'; import HeroBot from './HeroBot.mjs'; +import ClientLogPlugin from '../plugin/ClientLogPlugin.mjs'; class Bilibili extends HeroBot { async scrap(url) { - let data = {}; + let data = {url: url, done: false}; try { let options = { @@ -23,6 +24,7 @@ class Bilibili extends HeroBot { } const hero = new Hero(options); + hero.use(ClientLogPlugin); //开启log await hero.goto(url, configs.heroBotOptions); //等待所有内容加载完成 @@ -56,6 +58,8 @@ class Bilibili extends HeroBot { } await hero.close(); + + data.done = true; }catch(error) { console.error("Error got when request %s via hero: %s", url, error); } diff --git a/bot/Douyin.mjs b/bot/Douyin.mjs index b6e40bd..76219dc 100644 --- a/bot/Douyin.mjs +++ b/bot/Douyin.mjs @@ -1,11 +1,12 @@ import Hero from '@ulixee/hero'; import configs from '../config.mjs'; import HeroBot from './HeroBot.mjs'; +import ClientLogPlugin from '../plugin/ClientLogPlugin.mjs'; class Douyin extends HeroBot { async scrap(url) { - let data = {}; + let data = {url: url, done: false}; try { let options = { @@ -23,6 +24,7 @@ class Douyin extends HeroBot { } const hero = new Hero(options); + hero.use(ClientLogPlugin); //开启log await hero.goto(url, configs.heroBotOptions); //等待所有内容加载完成 @@ -56,6 +58,8 @@ class Douyin extends HeroBot { } await hero.close(); + + data.done = true; }catch(error) { console.error("Error got when request %s via hero: %s", url, error); } diff --git a/bot/HeroBot.mjs b/bot/HeroBot.mjs index 8af2c3a..78ca4e6 100644 --- a/bot/HeroBot.mjs +++ b/bot/HeroBot.mjs @@ -2,6 +2,7 @@ import Hero from '@ulixee/hero'; import configs from '../config.mjs'; import fs from 'node:fs'; import path from 'node:path'; +import ClientLogPlugin from '../plugin/ClientLogPlugin.mjs'; class HeroBot { constructor(heroCloudServer) { @@ -46,6 +47,7 @@ class HeroBot { console.log('Hero init配置', configs); const hero = new Hero(options); + hero.use(ClientLogPlugin); //开启log await hero.goto(base_url, configs.heroBotOptions); //等待所有内容加载完成 diff --git a/bot/Kuaishou.mjs b/bot/Kuaishou.mjs index edfd9a2..43c4769 100644 --- a/bot/Kuaishou.mjs +++ b/bot/Kuaishou.mjs @@ -1,11 +1,12 @@ import Hero from '@ulixee/hero'; import configs from '../config.mjs'; import HeroBot from './HeroBot.mjs'; +import ClientLogPlugin from '../plugin/ClientLogPlugin.mjs'; class Kuaishou extends HeroBot { async scrap(url) { - let data = {}; + let data = {url: url, done: false}; try { let options = { @@ -23,6 +24,7 @@ class Kuaishou extends HeroBot { } const hero = new Hero(options); + hero.use(ClientLogPlugin); //开启log await hero.goto(url, configs.heroBotOptions); //等待所有内容加载完成 @@ -40,6 +42,8 @@ class Kuaishou extends HeroBot { } await hero.close(); + + data.done = true; }catch(error) { console.error("Error got when request %s via hero: %s", url, error); }; diff --git a/bot/Xigua.mjs b/bot/Xigua.mjs index 631c4eb..430a6e6 100644 --- a/bot/Xigua.mjs +++ b/bot/Xigua.mjs @@ -1,11 +1,12 @@ import Hero from '@ulixee/hero'; import configs from '../config.mjs'; import HeroBot from './HeroBot.mjs'; +import ClientLogPlugin from '../plugin/ClientLogPlugin.mjs'; class Xigua extends HeroBot { async scrap(url) { - let data = {}; + let data = {url: url, done: false}; try { let options = { @@ -23,6 +24,7 @@ class Xigua extends HeroBot { } const hero = new Hero(options); + hero.use(ClientLogPlugin); //开启log await hero.goto(url, configs.heroBotOptions); //等待所有内容加载完成 @@ -49,6 +51,8 @@ class Xigua extends HeroBot { } await hero.close(); + + data.done = true; }catch(error) { console.error("Error got when request %s via hero: %s", url, error); }; diff --git a/config.mjs b/config.mjs index 0bffbed..7da4b4b 100644 --- a/config.mjs +++ b/config.mjs @@ -18,13 +18,13 @@ export default { //请求参数 heroBotOptions: { - timeoutMs: 10000, + timeoutMs: 20000, referrer: '', }, //网页tab参数 heroTabOptions: { - timeoutMs: 5000 + timeoutMs: 30000 }, //常用浏览器user-agent diff --git a/plugin/ClientLogPlugin.mjs b/plugin/ClientLogPlugin.mjs new file mode 100644 index 0000000..84d073a --- /dev/null +++ b/plugin/ClientLogPlugin.mjs @@ -0,0 +1,22 @@ +import { ClientPlugin } from '@ulixee/hero-plugin-utils'; + +class ClientLogPlugin extends ClientPlugin { + + async onHero(hero, sendToCore) { + const logTime = Date.now(); + console.log('%s - New Hero is initialized, session id %s.', logTime, await hero.sessionId); + } + + async onTab(hero, tab, sendToCore) { + const logTime = Date.now(); + console.log('%s - New Tab is initialized, id %s.', logTime, await tab.tabId); + } + + async onFrameEnvironment(hero, frameEnvironment, sendToCore) { + const logTime = Date.now(); + console.log('%s - New FrameEnvironment is initialized.', logTime); + } + +} + +export default ClientLogPlugin; diff --git a/test/scrap_test.mjs b/test/scrap_test.mjs index 1efc1ec..050f565 100644 --- a/test/scrap_test.mjs +++ b/test/scrap_test.mjs @@ -64,6 +64,8 @@ import configs from '../config.mjs'; configs.userAgent = configs.userAgents.mac_chrome; configs.viewport = configs.viewports.pc; + configs.heroTabOptions.timeoutMs = 20000; //所有内容加载完成超时 + console.log('Hero配置', configs); const xigua = new Xigua(heroCloudServer);