Add Scripts/browsertrix.sh

Scripts/browsertrix.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+#set -e
+
+list="seg_XYZ.txt"
+
+run_worker() {
+
+   mkdir -p noblogs
+   slug="$1"
+   list="$2"
+   today=`date +%Y-%m-%d`
+
+   # Test if var is empty
+   if [[ -z $slug ]]; then
+      echo "No domain found. Exiting.."
+      exit 0
+   fi
+
+   # Test if domain has already been archived
+   if [[ -f "./noblogs/noblogs-$slug.wacz" ]]; then
+      echo "Archive of $slug already exists. Exiting.."
+      sed -i "/$slug/d" $list
+      exit 0
+   fi
+
+   # Test if domain is currently archiving
+   if docker ps | grep $slug; then
+      echo "Archiving of $slug is already running. Skipping.."
+      exit 0
+   fi
+
+   echo "Archiving $slug.."
+
+   # Run browsertrix in Docker using n workers per container
+   workers="6"
+
+   docker run \
+       --rm \
+       --name "${slug}" \
+       --shm-size=2g --log-opt max-size=100m --log-opt max-file=3 -e NODE_OPTIONS="--max-old-space-size=12288" \
+       --volume ./crawls:/crawls \
+       webrecorder/browsertrix-crawler:latest crawl \
+       --workers $workers \
+       --sitemap \
+       --pageLoadTimeout 1440 --behaviorTimeout 600 --postLoadDelay 1 --maxPageRetries 3 \
+       --saveState always --saveStateInterval 300 \
+       --screenshot thumbnail \
+       --text final-to-warc \
+       --generateWACZ --generateCDX \
+       --scopeType prefix \
+       --exclude '.*[?]query.*' \
+       --exclude '.*[?]action.*' \
+       --exclude '.*[?]edit.*' \
+       --url "https://${slug}.noblogs.org" \
+       --collection "noblogs-${slug}" \
+       --title "${slug} (noblogs)" \
+       --description "crawl of read-only version from ${today}" > /dev/null
+
+   mv "crawls/collections/noblogs-$slug/noblogs-$slug.wacz" noblogs/
+   rm -rf "crawls/collections/noblogs-$slug"
+
+   # Remove slug from list, after successful archiving
+   sed -i 's/$slug//g' $list
+}
+
+# Create backup of the list
+cp -n $list $list.bak
+
+# Run the worker function in parallel
+export -f run_worker
+cat $list | parallel -j $(nproc --all) run_worker '{1}' "$list"