Browse Source

add bot for kuaishou

master
filesite 1 year ago
parent
commit
7f54b50387
  1. 1
      bot/Douyin.mjs
  2. 40
      bot/Kuaishou.mjs
  3. 20
      test/scrap_test.mjs

1
bot/Douyin.mjs

@ -22,6 +22,7 @@ class Douyin {
const tab = await hero.activeTab; const tab = await hero.activeTab;
await tab.waitForLoad('AllContentLoaded', {timeoutMs: configs.heroTabOptions.timeoutMs}); await tab.waitForLoad('AllContentLoaded', {timeoutMs: configs.heroTabOptions.timeoutMs});
//解析网页HTML数据
const elems = await hero.detach( hero.document.querySelectorAll('meta') ); const elems = await hero.detach( hero.document.querySelectorAll('meta') );
let meta_name = ''; let meta_name = '';
for (const elem of elems) { for (const elem of elems) {

40
bot/Kuaishou.mjs

@ -0,0 +1,40 @@
import Hero from '@ulixee/hero';
import configs from '../config.mjs';
class Kuaishou {
constructor(heroCloudServer) {
this.heroServer = heroCloudServer ? heroCloudServer : '';
}
async scrap(url) {
let data = {};
try {
let options = {};
if (this.heroServer) {
options.connectionToCore = this.heroServer;
}
const hero = new Hero(options);
await hero.goto(url, configs.heroBotOptions);
//等待所有内容加载完成
const tab = await hero.activeTab;
await tab.waitForLoad('AllContentLoaded', {timeoutMs: configs.heroTabOptions.timeoutMs});
//解析网页HTML数据
data.title = await hero.document.title;
const elem = await hero.detach( hero.document.querySelector('.video-container-player') );
data.cover = elem.getAttribute('poster');
await hero.close();
}catch(error) {
console.error("Error got when request %s via hero: %s", url, error);
};
return data;
}
}
export default Kuaishou;

20
test/scrap_test.mjs

@ -1,10 +1,24 @@
import Douyin from '../bot/Douyin.mjs'; import Douyin from '../bot/Douyin.mjs';
import Kuaishou from '../bot/Kuaishou.mjs';
import configs from '../config.mjs';
(async () => { (async () => {
configs.heroTabOptions.timeoutMs = 10000; //所有内容加载完成超时
console.log('Hero配置', configs);
const heroCloudServer = 'ws://192.168.3.13:1818'; const heroCloudServer = 'ws://192.168.3.13:1818';
const douyin = new Douyin(heroCloudServer);
const url = 'https://v.douyin.com/iJr1NsJJ/'; //抖音测试
//const douyin = new Douyin(heroCloudServer);
//const url = 'https://v.douyin.com/iJr1NsJJ/';
//console.log('请求中: %s ...', url);
//const data = await douyin.scrap(url);
//console.log("解析结果:\n%s", JSON.stringify(data));
//快手测试
const kuaishou = new Kuaishou(heroCloudServer);
const url = 'https://www.kuaishou.com/f/X8FTguiIjZQVwE7';
console.log('请求中: %s ...', url); console.log('请求中: %s ...', url);
const data = await douyin.scrap(url); const data = await kuaishou.scrap(url);
console.log("解析结果:\n%s", JSON.stringify(data)); console.log("解析结果:\n%s", JSON.stringify(data));
})(); })();

Loading…
Cancel
Save