Delete Scripts/browsertrix.sh

Scripts/browsertrix.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-
-#set -e
-
-list="seg_XYZ.txt"
-
-run_worker() {
-
-   mkdir -p noblogs
-   slug="$1"
-   list="$2"
-   today=`date +%Y-%m-%d`
-
-   # Test if var is empty
-   if [[ -z $slug ]]; then
-      echo "No domain found. Exiting.."
-      exit 0
-   fi
-
-   # Test if domain has already been archived
-   if [[ -f "./noblogs/noblogs-$slug.wacz" ]]; then
-      echo "Archive of $slug already exists. Exiting.."
-      sed -i "/$slug/d" $list
-      exit 0
-   fi
-
-   # Test if domain is currently archiving
-   if docker ps | grep $slug; then
-      echo "Archiving of $slug is already running. Skipping.."
-      exit 0
-   fi
-
-   echo "Archiving $slug.."
-
-   # Run browsertrix in Docker using n workers per container
-   workers="6"
-
-   docker run \
-       --rm \
-       --name "${slug}" \
-       --shm-size=2g --log-opt max-size=100m --log-opt max-file=3 -e NODE_OPTIONS="--max-old-space-size=12288" \
-       --volume ./crawls:/crawls \
-       webrecorder/browsertrix-crawler:latest crawl \
-       --workers $workers \
-       --sitemap \
-       --pageLoadTimeout 1440 --behaviorTimeout 600 --postLoadDelay 1 --maxPageRetries 3 \
-       --saveState always --saveStateInterval 300 \
-       --screenshot thumbnail \
-       --text final-to-warc \
-       --generateWACZ --generateCDX \
-       --scopeType prefix \
-       --exclude '.*[?]query.*' \
-       --exclude '.*[?]action.*' \
-       --exclude '.*[?]edit.*' \
-       --url "https://${slug}.noblogs.org" \
-       --collection "noblogs-${slug}" \
-       --title "${slug} (noblogs)" \
-       --description "crawl of read-only version from ${today}" > /dev/null
-
-   mv "crawls/collections/noblogs-$slug/noblogs-$slug.wacz" noblogs/
-   rm -rf "crawls/collections/noblogs-$slug"
-
-   # Remove slug from list, after successful archiving
-   sed -i 's/$slug//g' $list
-}
-
-# Create backup of the list
-cp -n $list $list.bak
-
-# Run the worker function in parallel
-export -f run_worker
-cat $list | parallel -j $(nproc --all) run_worker '{1}' "$list"