From ac28f8e28f0f7bb2af679d096e2ba9fcab31ac0d Mon Sep 17 00:00:00 2001 From: DarkFeather Date: Tue, 22 Dec 2020 09:46:21 -0600 Subject: [PATCH] Modernizing WolfPack to use systemd.timer and systemd-cat --- .gitignore | 4 ++ Makefile | 53 ++++++++-------------- PKGBUILD | 46 +++++++++++++++++++ README => README.md | 7 +-- offload-wolfpack | 12 ++--- offload-wolfpack@.service | 11 +++++ offload-wolfpack@.timer | 10 +++++ test-torrent-engine | 2 +- update-transmission-blocklist | 18 -------- wolfpack | 85 +++++++++++++++++------------------ wolfpack.service | 11 +++++ wolfpack.timer | 10 +++++ 12 files changed, 164 insertions(+), 105 deletions(-) create mode 100644 .gitignore create mode 100644 PKGBUILD rename README => README.md (80%) create mode 100644 offload-wolfpack@.service create mode 100644 offload-wolfpack@.timer delete mode 100755 update-transmission-blocklist create mode 100644 wolfpack.service create mode 100644 wolfpack.timer diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e6bf8b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.tar.zst +pkg +src + diff --git a/Makefile b/Makefile index 958c262..5786fc2 100644 --- a/Makefile +++ b/Makefile @@ -1,45 +1,30 @@ -INSTALLER != curl -s https://aninix.net/foundation/installer-test.bash | /bin/bash +compile: + @echo Nothing to compile. -compile: ./tpb.torrentengine.sample - if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; fi - git -C /usr/local/src/SharedLibraries pull; - if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./tpb.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi - if [ ! -x /usr/sbin/deluged ] || [ ! -x /usr/sbin/locate ]; then ${INSTALLER} deluge mlocate; fi - -install: compile /usr/bin/wget /usr/bin/deluge-console /usr/bin/googler ./wolfpack /usr/local/bin/whatismyip /usr/bin/lynx - cp -p /usr/local/src/WolfPack/wolfpack /usr/local/bin/wolfpack +install: compile ./wolfpack ./offload-wolfpack ./1337x.torrentengine.sample + mkdir -p ${pkgdir}/usr/local/bin/ + for i in wolfpack offload-wolfpack; do install -o root -g root -m 0755 $$i ${pkgdir}/usr/local/bin/$$i; done + mkdir -p ${pkgdir}/usr/lib/systemd/system + for i in *.service *.timer; do install -o root -g root -m 0755 $$i ${pkgdir}/usr/lib/systemd/system/$$i; done + mkdir -p ${pkgdir}/usr/local/etc/WolfPack + chmod 0770 ${pkgdir}/usr/local/etc/WolfPack + cp ./*.pup ${pkgdir}/usr/local/etc/WolfPack; cp ./1337x.torrentengine.sample ${pkgdir}/usr/local/etc/WolfPack/torrentengine; make checkperm - systemctl enable deluged - systemctl start deluged - @echo - @echo "Add pups to /usr/local/etc/WolfPack/ to set up your searches." - @echo "Add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection." - @echo WolfPack installed\! clean: @echo Nothing to do test: - ./test-torrent-engine 'S01E01 Tech-TV' - ./wolfpack --member ./example-download.pup - ./wolfpack --member ./example-search.pup - ./wolfpack --member ./example-shows.pup - ./wolfpack --member ./example-torrent.pup + @./test-torrent-engine 'S01E01 Tech-TV' ./1337x.torrentengine.sample + ./wolfpack --help reverse: - cp /usr/local/bin/wolfpack* . - cp /root/bin/offload-wolfpack . + for i in wolfpack offload-wolfpack; do cp /usr/local/bin/$$i .; done + for i in *.service *.timer; do cp /usr/lib/systemd/system/$$i .; done checkperm: /usr/local/etc/WolfPack/ - chmod -R ug+rw /usr/local/etc/WolfPack/ - chmod 0770 /usr/local/etc/WolfPack/ - chmod 0755 /usr/local/bin/wolfpack - chown root:root /usr/local/bin/wolfpack* - -offload: offload-wolfpack /root/.ssh/id_rsa /root/.ssh/id_rsa.pub - cp offload-wolfpack /root/bin - chmod 0700 /root/bin/offload-wolfpack - chown root:root /root/bin/offload-wolfpack - -diff: /usr/local/bin/wolfpack - @echo wolfpack:; diff ./wolfpack /usr/local/bin/wolfpack; echo; if [ -f /root/bin/offload-wolfpack ]; then echo offload-wolfpack:; diff ./offload-wolfpack /root/bin/offload-wolfpack; echo; fi; + chmod -R ug+rw ${pkgdir}/usr/local/etc/WolfPack/ + chmod 0770 ${pkgdir}/usr/local/etc/WolfPack/ + chmod 0755 ${pkgdir}/usr/local/bin/*wolfpack + chown root:root ${pkgdir}/usr/local/bin/*wolfpack + chmod 0660 ${pkgdir}/usr/local/etc/WolfPack/*; diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..2f9e061 --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,46 @@ +depends=('bash>=4.4' 'wget>=1.20' 'deluge>=2.0.4' 'elinks>=0.13' 'Uniglot') +makedepends=('make>=4.2') +checkdepends=() +optdepends=() +pkgname="$(git config remote.origin.url | rev | cut -f 1 -d '/' | rev | sed 's/.git$//')" +pkgver="$(git describe --tag --abbrev=0)"."$(git rev-parse --short HEAD)" +pkgrel=1 +pkgrel() { + echo $(( `git log "$(git describe --tag --abbrev=0)"..HEAD | grep -c commit` + 1 )) +} +epoch="$(git log | grep -c commit)" +pkgdesc="$(head -n 1 README.md)" +arch=("x86_64") +url="$(git config remote.origin.url | sed 's/.git$//')" +license=('custom') +groups=() +provides=("${pkgname}") +conflicts=() +replaces=("${pkgname,,}", "aninix-${pkgname,,}") +backup=() +options=() +install= +changelog= +source=() +noextract=() +md5sums=() +validpgpkeys=() + +prepare() { + git pull +} + +build() { + make -C .. +} + +check() { + chmod -R u+r ../pkg + make -C .. test +} + +package() { + export pkgdir="${pkgdir}" + make -C .. install + install -D -m644 ../LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE" +} diff --git a/README b/README.md similarity index 80% rename from README rename to README.md index cc54939..68036a8 100644 --- a/README +++ b/README.md @@ -1,5 +1,4 @@ -=== AniNIX::WolfPack === -WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line +WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line. Please note that the torrent and show functionalities are provided for legitimate peer-to-peer downloading and should not be used for illicit practices. @@ -9,5 +8,7 @@ Some tools included are: ** Note: torrentproject.se is currently hard to scrape due to CloudFlare protections, and 1337x has a challenging reputation. * offload-wolfpack: This allows other machines to run the wolfpack and then copy the results to a known destination. -We have also included example "pups" and some helper scripts for updating torrent blocklists. +We have also included example "pups" in this source. + + diff --git a/offload-wolfpack b/offload-wolfpack index 196b6cb..5a3a1fb 100755 --- a/offload-wolfpack +++ b/offload-wolfpack @@ -1,7 +1,6 @@ #!/bin/bash set -x -logfile=/var/log/wolfpack.log target="$1" dir=`sudo -u deluge deluge-console config download_location 2>/dev/null | head -n 1 | sed 's/download_location: //'` @@ -10,22 +9,23 @@ if [ -z "$target" ]; then # || [ "$(ping -c 2 "$target" &>/dev/null; echo $?)" - exit 1 fi ssh $target mkdir -p /srv/yggdrasil/new_acquisition -sudo -u deluge deluge-console info 2>/dev/null | egrep '^\[S\]' | ( +timeout 60 sudo -u deluge /bin/bash -l -c 'deluge-console info' 2>/dev/null | egrep '^\[S\]' | ( while read line; do id="$(echo "$line" | awk '{ print $4; }')" name="$(echo "$line" | awk '{ print $3 }')" - sudo -u deluge deluge-console 2>/dev/null rm "$id"; # Clean up downloads before sending. rm -Rf "$dir"/"$name"/*.nfo "$dir"/"$name"/*.txt "$dir"/"$name"/*.png "$dir"/"$name"/*.jpg rsync -avzz "$dir"'/'"$name" $target:/srv/yggdrasil/new_acquisition ret="$?" if [ "$ret" -eq 0 ]; then + # Hack because `deluged rm -c $id` isn't working + sudo systemctl stop deluged + sudo rm -Rf /srv/deluge/.config/deluge/state"$id"; + sudo systemctl start deluged rm -Rf "$dir"'/'"$name" else echo "ERROR: Return was $ret" >> "$dir"/"$name"/error.log fi unset name; unset id; done -) -cat /var/log/wolfpack.log | ssh $target /bin/bash -c 'cat >> /var/log/wolfpack.log' -cat /dev/null > /var/log/wolfpack.log +) | systemd-cat -t "offload-wolfpack" diff --git a/offload-wolfpack@.service b/offload-wolfpack@.service new file mode 100644 index 0000000..5ff605c --- /dev/null +++ b/offload-wolfpack@.service @@ -0,0 +1,11 @@ +[Unit] +Description=AniNIX::WolfPack \\ Data Collection Botnet, Offload to %I + +[Service] +ExecStartPre=/usr/sbin/sudo /usr/bin/systemctl start wolfpack.service +ExecStart=/usr/local/bin/offload-wolfpack %I +KillMode=process +Type=oneshot +RemainAfterExit=no +User=depriv +Group=depriv diff --git a/offload-wolfpack@.timer b/offload-wolfpack@.timer new file mode 100644 index 0000000..336b2aa --- /dev/null +++ b/offload-wolfpack@.timer @@ -0,0 +1,10 @@ +[Unit] +Description=AniNIX::WolfPack \\ Data Collection Botnet, Offload to %I (timer) + +[Timer] +OnCalendar=00/4:00 +AccuracySec=12h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/test-torrent-engine b/test-torrent-engine index cd35c15..57f56dc 100755 --- a/test-torrent-engine +++ b/test-torrent-engine @@ -4,7 +4,7 @@ if [ -z "$1" ]; then exit 1; fi -source /usr/local/src/SharedLibraries/Bash/header +source /opt/aninix/Uniglot/Bash/header header Configuring... if [ -z "$2" ]; then configuretorrent /usr/local/etc/WolfPack/torrentengine diff --git a/update-transmission-blocklist b/update-transmission-blocklist deleted file mode 100755 index 5182c1e..0000000 --- a/update-transmission-blocklist +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -if [ $(pgrep -afc $0) -ne 0 ]; then - echo Already running! - exit 1; -fi - -cd /root/.config/transmission/blocklists/ -rm -Rf * -wget 'http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz' -O blocklist.gz -gunzip blocklist.gz; -while [ "$?" -ne 0 ]; do - echo Error $?: Couldn\'t get blocklist. - wget 'http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz' -O blocklist.gz - gunzip blocklist.gz; -done -/usr/bin/transmission-cli -b notorrent # Update the blocklist to a .bin format - diff --git a/wolfpack b/wolfpack index 582e49c..ebdd692 100755 --- a/wolfpack +++ b/wolfpack @@ -1,15 +1,10 @@ #!/bin/bash -source /usr/local/src/SharedLibraries/Bash/header +source /opt/aninix/Uniglot/Bash/header export downloadtorrent="sudo -u deluge deluge-console 2>/dev/null add" -if [ -z "$3" ]; then - configuretorrent /usr/local/etc/WolfPack/torrentengine -else - configuretorrent "$3" -fi export IFS=" " -export logfile='/var/log/wolfpack.log' +export servicename='wolfpack' function helptext { echo ' @@ -30,8 +25,6 @@ wolfpack --torrent-monitor ~~ Pull up the Deluge console for torrents. wolfpack --torrent-lookup ~~ Look up and add a torrent if user confirms. -wolfpack --reset-log ~~ Clear the wolfpack log. This - is in /var/log/wolfpack.log. wolfpack --help ~~ Show this helptext Available pup types (all pups are line-delimited with the type and @@ -57,7 +50,7 @@ search ~~ Uses the Google search function privacycheck { ip="$(/usr/local/bin/whatismyip)" if [ -z "$ip" ] || [ "$ip" == "$(nslookup aninix.net | grep Address | cut -f 2 -d ' ' | tail -n 1)" ]; then - echo "ERROR: Should not torrent directly from the AniNIX main IP for privacy reasons" 2>&1 | tee -a "$logfile" + echo "ERROR: Should not torrent directly from the AniNIX main IP for privacy reasons" 2>&1 | systemd-cat -t "$servicename" exit 1 fi } @@ -67,15 +60,25 @@ function nextairdate { echo "Need a show to search for" exit 1; fi - lynx -read_timeout=5 -connect_timeout=5 --dump "https://www.episodate.com/tv-show/$(echo $1 | tr '[:upper:]' '[:lower:]' | tr ' ' '-')?season=99" 2>&1 | grep 'and airs on' | sed 's# Season [0-9]\+ / Episode [0-9]\+ and airs on ##' + timeout 5 elinks --dump "https://www.episodate.com/tv-show/$(echo $1 | tr '[:upper:]' '[:lower:]' | tr ' ' '-')?season=99" 2>&1 | grep 'and airs on' | sed 's# Season [0-9]\+ / Episode [0-9]\+ and airs on ##' return } +function torrentConfigure { + if [ -z "$3" ]; then + configuretorrent /usr/local/etc/WolfPack/torrentengine + else + configuretorrent "$3" + fi +} + + function torrent { if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then echo "Need a valid argument." return; fi + torrentConfigure privacycheck; export downloaddir=$(head -n 2 "$1" | tail -n 1) sudo -u deluge deluge-console config -s autoadd_location "$downloaddir" 2>/dev/null @@ -85,11 +88,11 @@ function torrent { if [ "$magnetlink" != "" ]; then /bin/bash -c "$downloadtorrent $magnetlink" sed -i "/$i/d" "$1" - echo $magnetlink | tee -a "$logfile" - echo Added magnet to deluged and removed $i | tee -a "$logfile" + echo $magnetlink | systemd-cat -t "$servicename" + echo Added magnet to deluged and removed $i | systemd-cat -t "$servicename" fi done - if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | tee -a "$logfile"; fi + if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | systemd-cat -t "$servicename"; fi } function shows { @@ -116,12 +119,12 @@ function shows { fi if [ "$magnetlink" != "" ]; then # We found an episode. - echo $i | tee -a "$logfile" - echo $magnetlink | tee -a "$logfile" + echo $i | systemd-cat -t "$servicename" + echo $magnetlink | systemd-cat -t "$servicename" /bin/bash -c "$downloadtorrent $magnetlink" sed -i "s/$i/S${season}E${newep}#${showName}#${modifier}/" "$1" - echo Downloaded and pup updated. | tee -a "$logfile" - echo | tee -a "$logfile" + echo Downloaded and pup updated. | systemd-cat -t "$servicename" + echo | systemd-cat -t "$servicename" else # We didn't -- is there a new season? if [ -z "$modifier" ]; then @@ -130,24 +133,24 @@ function shows { getmagnetlink "$showName S${newseason}E01 $modifier" fi if [ "$magnetlink" != "" ]; then - echo "S${newseason}E01" $showName | tee -a "$logfile" - echo "$magnetlink" | tee -a "$logfile" + echo "S${newseason}E01" $showName | systemd-cat -t "$servicename" + echo "$magnetlink" | systemd-cat -t "$servicename" /bin/bash -c "$downloadtorrent $magnetlink" sed -i "s/$i/S${newseason}E02#$showName#$modifier/" "$1" else nextAirDate="$(nextairdate "$showName")" if [ ! -z "$nextAirDate" ]; then - echo "No new season found for $showName -- next airs $(nextairdate "$showName")" | tee -a "$logfile" + echo "No new season found for $showName -- next airs $(nextairdate "$showName")" | systemd-cat -t "$servicename" if [ "$(date -d "$nextAirDate" -u +%s)" -lt "$(date -u +%s)" ]; then - echo ERROR: Should have found a new episode for $searchterms and did not -- check your provider\! | tee -a "$logfile" + echo ERROR: Should have found a new episode for $searchterms and did not -- check your provider\! | systemd-cat -t "$servicename" fi else - echo "No new season found for $showName" | tee -a "$logfile" + echo "No new season found for $showName" | systemd-cat -t "$servicename" fi fi fi done - if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | tee -a "$logfile"; fi + if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | systemd-cat -t "$servicename"; fi } function download { @@ -170,15 +173,15 @@ function download { export filename="$downloaddir"/"$(echo "$nexturl" | rev | cut -f 1 -d \/ | rev)" #if file doesn't already exist.... if [ ! -f "$filename" ]; then - echo Downloading $nexturl to $filename ... | tee -a "$logfile" + echo Downloading $nexturl to $filename ... | systemd-cat -t "$servicename" for i in $(find "$downloaddir" -type f | grep -i "$dedupterm"); do - echo Deduped\! Removed $i from $downloaddir | tee -a "$logfile" - rm -f "$i" 2>&1 | tee -a "$logfile" + echo Deduped\! Removed $i from $downloaddir | systemd-cat -t "$servicename" + rm -f "$i" 2>&1 | systemd-cat -t "$servicename" done wget --timeout=5 -q -O "$filename" "$nexturl" - echo Downloaded | tee -a "$logfile" + echo Downloaded | systemd-cat -t "$servicename" fi - if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | tee -a "$logfile"; fi + if [ -x `which clamscan` ]; then clamscan -r -i "$downloaddir" | systemd-cat -t "$servicename"; fi } function search { @@ -191,7 +194,7 @@ function search { export searchstring=$(echo $i | sed 's/ /+/g') export file="$downloaddir"/$searchstring'.txt' echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file" - echo "Ran search for $searchstring to $file" | tee -a "$logfile" + echo "Ran search for $searchstring to $file" | systemd-cat -t "$servicename" done } @@ -200,25 +203,25 @@ if [ "$1" == "" ]; then exit 1; fi -echo '---' $(date) '---' | tee -a "$logfile" +echo '---' $(date) '---' | systemd-cat -t "$servicename" case "$1" in "--alpha") - echo Started alpha on `uname -n` | tee -a "$logfile" + echo Started alpha on `uname -n` | systemd-cat -t "$servicename" if [ "$2" == "" ]; then export basedir="/usr/local/etc/WolfPack" else export basedir="$2"; fi - echo "Starting members from pup in "$basedir | tee -a "$logfile" + echo "Starting members from pup in "$basedir | systemd-cat -t "$servicename" for i in $(find "$basedir" -type f | grep '.pup'); do /usr/local/bin/wolfpack --member "$i" "$3"; done ;; "--member") - echo "Spawning a $(head -n 1 "$2") member for $2 on `uname -n`" | tee -a "$logfile" - (printf "IP: "; /usr/local/bin/whatismyip) | tee -a /var/log/wolfpack.log + echo "Spawning a $(head -n 1 "$2") member for $2 on `uname -n`" | systemd-cat -t "$servicename" + (printf "IP: "; /usr/local/bin/whatismyip) | systemd-cat -t "$servicename" case "$(head -n 1 "$2")" in "shows") shows "$2" @@ -237,16 +240,12 @@ case "$1" in ;; esac ;; - "--reset-log") - cat /dev/null > "$logfile" - exit 0; - ;; "--torrent-monitor") exec sudo -u deluge deluge-console 2>/dev/null; exit 0; ;; "--torrent-lookup") - configuretorrent /usr/local/etc/WolfPack/torrentengine + torrentConfigure getmagnetlink "$(echo $2 | tr '#' "$searchunifier")" if [ -z "$magnetlink" ]; then echo Nothing found; @@ -269,6 +268,6 @@ case "$1" in ;; esac -echo "Exited "$1 | tee -a "$logfile" -echo '---' "$(date)" '---' | tee -a "$logfile" -echo | tee -a "$logfile" +echo "Exited "$1 | systemd-cat -t "$servicename" +echo '---' "$(date)" '---' | systemd-cat -t "$servicename" +echo | systemd-cat -t "$servicename" diff --git a/wolfpack.service b/wolfpack.service new file mode 100644 index 0000000..bae88ab --- /dev/null +++ b/wolfpack.service @@ -0,0 +1,11 @@ +[Unit] +Description=AniNIX::WolfPack \\ Data Collection Botnet + +[Service] +ExecStart=/usr/local/bin/wolfpack --alpha +KillMode=process +Type=oneshot +TimeoutStartSec=1800 +RemainAfterExit=no +User=depriv +Group=depriv diff --git a/wolfpack.timer b/wolfpack.timer new file mode 100644 index 0000000..57c5f90 --- /dev/null +++ b/wolfpack.timer @@ -0,0 +1,10 @@ +[Unit] +Description=AniNIX::WolfPack \\ Data Collection Botnet (timer) + +[Timer] +OnCalendar=00/4:00 +AccuracySec=12h +Persistent=true + +[Install] +WantedBy=timers.target