Browse Source

add mode switch for bot

master
filesite 7 months ago
parent
commit
dd3f5acc52
  1. 3
      bot/Douyin.mjs
  2. 4
      bot/HeroBot.mjs
  3. 1
      config.mjs
  4. 7
      spider.mjs
  5. 4
      test/scrap_test.mjs

3
bot/Douyin.mjs

@ -71,7 +71,6 @@ class Douyin extends HeroBot {
await hero.close(); await hero.close();
//删除profile文件后重试 //删除profile文件后重试
await this.deleteProfile(); await this.deleteProfile();
this.ua = 'pc'; //切换到pc模式重试
return false; return false;
} }
@ -116,8 +115,6 @@ class Douyin extends HeroBot {
//删除profile文件后重试 //删除profile文件后重试
await this.deleteProfile(); await this.deleteProfile();
//切换模式
this.ua = this.ua == 'mob' ? 'pc' : 'mob';
} }
return data; return data;

4
bot/HeroBot.mjs

@ -26,6 +26,10 @@ class HeroBot {
this.ua = 'pc'; this.ua = 'pc';
} }
setMode(mode) {
this.ua = mode == 'mob' ? 'mob' : 'pc';
}
//返回profile对象 //返回profile对象
async init(botName) { async init(botName) {
if (typeof(this.supportedBots[botName]) == 'undefined') { if (typeof(this.supportedBots[botName]) == 'undefined') {

1
config.mjs

@ -27,6 +27,7 @@ let configs = {
//bot相关配置 //bot相关配置
cloud_server: '', cloud_server: '',
default_mode: 'pc',
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36', userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
viewport: { viewport: {
width: 1440, width: 1440,

7
spider.mjs

@ -49,6 +49,7 @@ import path from 'node:path';
//spider run //spider run
let spider_is_running = false, let spider_is_running = false,
last_run_time = 0; last_run_time = 0;
let last_douyin_mode = 'pc';
const task_check_time = 20; //每 20 秒抓取一次 const task_check_time = 20; //每 20 秒抓取一次
const task_auto_run = cron.schedule(`*/${task_check_time} * * * * *`, async () => { const task_auto_run = cron.schedule(`*/${task_check_time} * * * * *`, async () => {
const current_time = common.getTimestampInSeconds(); const current_time = common.getTimestampInSeconds();
@ -70,6 +71,7 @@ import path from 'node:path';
switch (botName) { switch (botName) {
case 'douyin': case 'douyin':
bot = new Douyin(heroCloudServer); bot = new Douyin(heroCloudServer);
bot.setMode(last_douyin_mode);
break; break;
case 'kuaishou': case 'kuaishou':
bot = new Kuaishou(heroCloudServer); bot = new Kuaishou(heroCloudServer);
@ -90,6 +92,11 @@ import path from 'node:path';
const data = await bot.scrap(task.url); const data = await bot.scrap(task.url);
//console.log('Data got by bot', data); //console.log('Data got by bot', data);
//为抖音切换模式
if (bot.name == 'douyin') {
last_douyin_mode = last_douyin_mode == 'pc' ? 'mob' : 'pc';
}
if (typeof(data.done) != 'undefined' && data.done == true) { if (typeof(data.done) != 'undefined' && data.done == true) {
task.data = data; //把抓取到的数据保存到任务里 task.data = data; //把抓取到的数据保存到任务里
taskMoniter.updateTask(task.id, task); taskMoniter.updateTask(task.id, task);

4
test/scrap_test.mjs

@ -31,6 +31,10 @@ import getConfigs from '../config.mjs';
console.log('Hero配置', configs); console.log('Hero配置', configs);
const douyin = new Douyin(heroCloudServer); const douyin = new Douyin(heroCloudServer);
//使用手机模式,默认为pc
douyin.setMode('mob');
console.log('请求中: %s ...', url); console.log('请求中: %s ...', url);
data = await douyin.scrap(url); data = await douyin.scrap(url);
console.log("解析结果:\n%s", JSON.stringify(data)); console.log("解析结果:\n%s", JSON.stringify(data));

Loading…
Cancel
Save