#!/bin/bash

#set -e

list="seg_XYZ.txt"

run_worker() {

   mkdir -p noblogs
   slug="$1"
   list="$2"
   today=`date +%Y-%m-%d`

   # Test if var is empty
   if [[ -z $slug ]]; then
      echo "No domain found. Exiting.."
      exit 0
   fi

   # Test if domain has already been archived
   if [[ -f "./noblogs/noblogs-$slug.wacz" ]]; then
      echo "Archive of $slug already exists. Exiting.."
      sed -i "/$slug/d" $list
      exit 0
   fi

   # Test if domain is currently archiving
   if docker ps | grep $slug; then
      echo "Archiving of $slug is already running. Skipping.."
      exit 0
   fi

   echo "Archiving $slug.."

   # Run browsertrix in Docker using n workers per container
   workers="6"

   docker run \
       --rm \
       --name "${slug}" \
       --shm-size=2g --log-opt max-size=100m --log-opt max-file=3 -e NODE_OPTIONS="--max-old-space-size=12288" \
       --volume ./crawls:/crawls \
       webrecorder/browsertrix-crawler:latest crawl \
       --workers $workers \
       --sitemap \
       --pageLoadTimeout 1440 --behaviorTimeout 600 --postLoadDelay 1 --maxPageRetries 3 \
       --saveState always --saveStateInterval 300 \
       --screenshot thumbnail \
       --text final-to-warc \
       --generateWACZ --generateCDX \
       --scopeType prefix \
       --exclude '.*[?]query.*' \
       --exclude '.*[?]action.*' \
       --exclude '.*[?]edit.*' \
       --url "https://${slug}.noblogs.org" \
       --collection "noblogs-${slug}" \
       --title "${slug} (noblogs)" \
       --description "crawl of read-only version from ${today}" > /dev/null

   mv "crawls/collections/noblogs-$slug/noblogs-$slug.wacz" noblogs/
   rm -rf "crawls/collections/noblogs-$slug"

   # Remove slug from list, after successful archiving
   sed -i 's/$slug//g' $list
}

# Create backup of the list
cp -n $list $list.bak

# Run the worker function in parallel
export -f run_worker
cat $list | parallel -j $(nproc --all) run_worker '{1}' "$list"