diff --git a/spider_watcher.sh b/spider_watcher.sh new file mode 100755 index 0000000..224adae --- /dev/null +++ b/spider_watcher.sh @@ -0,0 +1,39 @@ +#!/bin/sh +## spider watcher, work with start_cloud_in_container.sh + + +watcher() +{ + # you should change this directory path + script_root='~/Develop/machete_hero/' + + spider_num=`ps -ef | grep 'node spider.mjs' | grep -v grep | wc -l` + + if [ $spider_num -lt 1 ]; then + echo "Spider is down, try to restart it." + + echo "Checking ulixee cloud" + cloud_num=`docker ps | grep 'Up ' | grep -v grep | wc -l` + if [ $cloud_num -ge 1 ]; then + echo "ulixee cloud is alive, start spider" + cd $script_root + npm start -- myconfig.json + else + echo "ulixee cloud is down, try to restart it" + docker stop ulixee_cloud + docker rm ulixee_cloud + cd $script_root + ./start_cloud_in_container.sh + sleep 10 + fi + + else + echo "Spider is alive" + sleep 10 + fi + +} + +while true; do + watcher +done diff --git a/start_cloud_in_container.sh b/start_cloud_in_container.sh new file mode 100755 index 0000000..68804f0 --- /dev/null +++ b/start_cloud_in_container.sh @@ -0,0 +1,41 @@ +#!/bin/bash +## start ulixee cloud in docker container +## you should pull it's docker image before run this script +## docker pull ulixee/ulixee-cloud + +current_path=$(dirname $0) +cd $current_path + +## Configure the PORT ulixee will run on +port="${PORT:=1818}" +## Enable verbose logs +DEBUG=ulx* +## NOTE: these are unix oriented. adjust as needed for Windows +DATASTORES_MOUNT=$HOME/.cache/ulixee/datastores +DATADIR_MOUNT=/tmp/.ulixee + +mkdir -p $DATASTORES_MOUNT +mkdir $DATADIR_MOUNT + +chmod 777 $DATASTORES_MOUNT +chmod 777 $DATADIR_MOUNT + +# To add an environment configuration file: +# `--env-file ./.env` +# All environment configurations can be found at: `cloud/main/.env.defaults` +docker run -itd --init \ + --name ulixee_cloud \ + --ipc=host \ + --user ulixee \ + --restart unless-stopped \ + --sysctl net.ipv4.tcp_keepalive_intvl=10 \ + --sysctl net.ipv4.tcp_keepalive_probes=3 \ + --log-opt max-size=50m --log-opt max-file=3 \ + --log-driver local \ + -v $DATASTORES_MOUNT:/home/ulixee/.cache/ulixee/datastores \ + -v $DATADIR_MOUNT:/tmp/.ulixee \ + -p "$port:$port" \ + -e DEBUG=$DEBUG \ + -e DISPLAY=:99 \ + ulixee/ulixee-cloud:latest \ + xvfb-run npx @ulixee/cloud start --port=${port}