Browse Source

add configs for bot

master
filesite 1 year ago
parent
commit
28cdf7d197
  1. 9
      bot/Douyin.mjs
  2. 15
      config.mjs
  3. 2
      test/scrap_test.mjs

9
bot/Douyin.mjs

@ -1,4 +1,5 @@
import Hero from '@ulixee/hero'; import Hero from '@ulixee/hero';
import configs from '../config.mjs';
class Douyin { class Douyin {
constructor(heroCloudServer) { constructor(heroCloudServer) {
@ -15,15 +16,11 @@ class Douyin {
} }
const hero = new Hero(options); const hero = new Hero(options);
await hero.goto(url, { await hero.goto(url, configs.heroBotOptions);
timeoutMs: 10000,
referrer: 'https://wechat.com',
userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1'
});
//等待所有内容加载完成 //等待所有内容加载完成
const tab = await hero.activeTab; const tab = await hero.activeTab;
await tab.waitForLoad('AllContentLoaded', {timeoutMs: 5000}); await tab.waitForLoad('AllContentLoaded', {timeoutMs: configs.heroTabOptions.timeoutMs});
const elems = await hero.detach( hero.document.querySelectorAll('meta') ); const elems = await hero.detach( hero.document.querySelectorAll('meta') );
let meta_name = ''; let meta_name = '';

15
config.mjs

@ -0,0 +1,15 @@
export default {
//请求参数
heroBotOptions: {
timeoutMs: 10000,
referrer: 'https://wechat.com',
userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1'
},
//网页tab参数
heroTabOptions: {
timeoutMs: 5000
}
};

2
test/scrap_test.mjs

@ -6,5 +6,5 @@ import Douyin from '../bot/Douyin.mjs';
const url = 'https://v.douyin.com/iJr1NsJJ/'; const url = 'https://v.douyin.com/iJr1NsJJ/';
console.log('请求中: %s ...', url); console.log('请求中: %s ...', url);
const data = await douyin.scrap(url); const data = await douyin.scrap(url);
console.log('抖音网址 %s 解析结果: %s', url, JSON.stringify(data)); console.log("解析结果:\n%s", JSON.stringify(data));
})(); })();

Loading…
Cancel
Save