commit f2864631fbb448d80256950c0388334bdffb1147 Author: cxford Date: Thu Aug 4 12:08:16 2016 -0500 Converting from Bazaar. Old log: ------------------------------------------------------------ revno: 8 committer: root branch nick: WolfPack timestamp: Tue 2016-07-19 13:52:29 -0500 message: Clearing some read statements ------------------------------------------------------------ revno: 7 committer: root branch nick: WolfPack timestamp: Tue 2016-07-19 13:36:23 -0500 message: Fix for torrent protocol ------------------------------------------------------------ revno: 6 committer: dev branch nick: WolfPack timestamp: Mon 2016-07-18 10:23:17 -0500 message: Massive fixes; offloading updates; torrent function pup type updated ------------------------------------------------------------ revno: 5 committer: dev branch nick: WolfPack timestamp: Tue 2016-05-31 14:42:28 -0500 message: Removing ghost-wolfpack in favor of systemd vpn client. ------------------------------------------------------------ revno: 4 committer: dev branch nick: WolfPack timestamp: Sun 2016-05-15 21:35:02 -0500 message: Updated to include privacy VPN and offloading options. ------------------------------------------------------------ revno: 3 committer: dev branch nick: WolfPack timestamp: Thu 2016-05-12 20:51:23 -0500 message: Adding ghost-wolfpack script for sites using VPNs for privacy ------------------------------------------------------------ revno: 2 committer: dev branch nick: WolfPack timestamp: Thu 2016-05-12 15:58:05 -0500 message: Updated to use the googler package from the ArchLinux AUR for google searches and store as text files. Sufficient for now. ------------------------------------------------------------ revno: 1 committer: dev branch nick: WolfPack timestamp: Thu 2016-05-12 14:45:12 -0500 message: Adding all but search example -- search API is still in flux with no good command-line search option. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4bda53e --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +build: + Nothing to build -- this is a bash script. + +configure: + mkdir -p /usr/local/etc/WolfPack + chmod 0770 /usr/local/etc/WolfPack + cp ./*.pup /usr/local/etc/WolfPack + chmod 0660 /usr/local/etc/WolfPack/* + @echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups" + @echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection." + +install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip + cp -p /usr/local/src/WolfPack/wolfpack /usr/local/bin/wolfpack + cp -p /usr/local/src/WolfPack/wolfpack-stoptorrent /usr/local/bin/wolfpack-stoptorrent + [ ! -f /root/.bashrc ] || (grep 'transmission-cli' /root/.bashrc) || echo 'alias transmission-cli="transmission-cli -d 600 -u 1 -er -w /srv/yggdrasil/new_acquisition -b"' >> /root/.bashrc + chmod 0700 /usr/local/bin/wolfpack-stoptorrent + chmod 0755 /usr/local/bin/wolfpack + chown root:root /usr/local/bin/wolfpack* + +reverse: + cp /usr/local/bin/wolfpack* . + [ ! -f /root/bin/offload-wolfpack ] || cp /root/bin/offload-wolfpack . + +offload: offload-wolfpack /root/.ssh/id_rsa /root/.ssh/id_rsa.pub + cp offload-wolfpack /root/bin + chmod 0700 /root/bin/offload-wolfpack + chown root:root /root/bin/offload-wolfpack + + diff --git a/example-download.pup b/example-download.pup new file mode 100644 index 0000000..e6299fd --- /dev/null +++ b/example-download.pup @@ -0,0 +1,4 @@ +download +/srv/WolfPackResults +wget -q -O - 'https://www.archlinux.org/download/' | grep surlyjake | cut -f 2 -d \" | head -n 1 +echo PREVRESULT/$(wget -q -O - PREVRESULT | grep dual.iso | head -n 1 | cut -f 2 -d \") diff --git a/example-search.pup b/example-search.pup new file mode 100644 index 0000000..eb561da --- /dev/null +++ b/example-search.pup @@ -0,0 +1,3 @@ +search +/tmp +Disturbed 2015 concerts diff --git a/example-shows.pup b/example-shows.pup new file mode 100644 index 0000000..039e25d --- /dev/null +++ b/example-shows.pup @@ -0,0 +1,3 @@ +shows +/srv/yggdrasil/new_acquisition +S01E01#Tech-TV diff --git a/example-torrent.pup b/example-torrent.pup new file mode 100644 index 0000000..6943577 --- /dev/null +++ b/example-torrent.pup @@ -0,0 +1,3 @@ +torrent +/srv/yggdrasil/new_acquisition +ArchLinux ISO diff --git a/offload-wolfpack b/offload-wolfpack new file mode 100755 index 0000000..c33b3e1 --- /dev/null +++ b/offload-wolfpack @@ -0,0 +1,14 @@ +#!/bin/bash + +/usr/local/bin/wolfpack --alpha +sleep 10 +rsync -avz /srv/yggdrasil/new_acquisition/* cxford@10.0.1.3:/srv/yggdrasil/new_acquisition &>>/var/log/wolfpack.log +ls -l /srv/yggdrasil/new_acquisition >> /var/log/wolfpack.log +echo Files transferred from $(uname -n). >> /var/log/wolfpack.log +rm -Rf /srv/yggdrasil/new_acquisition/* +cat /var/log/wolfpack.log | ssh cxford@aninix.net "cat >> /var/log/wolfpack.log" +rm /var/log/wolfpack.log +for i in $(ls /usr/local/etc/WolfPack/*.pup); do + export cmdstring="$(echo $i | sed "s/pup/$(uname -n)/")" + scp "$i" cxford@aninix.net:"$cmdstring" +done diff --git a/update-transmission-blocklist b/update-transmission-blocklist new file mode 100755 index 0000000..5182c1e --- /dev/null +++ b/update-transmission-blocklist @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ $(pgrep -afc $0) -ne 0 ]; then + echo Already running! + exit 1; +fi + +cd /root/.config/transmission/blocklists/ +rm -Rf * +wget 'http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz' -O blocklist.gz +gunzip blocklist.gz; +while [ "$?" -ne 0 ]; do + echo Error $?: Couldn\'t get blocklist. + wget 'http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz' -O blocklist.gz + gunzip blocklist.gz; +done +/usr/bin/transmission-cli -b notorrent # Update the blocklist to a .bin format + diff --git a/wolfpack b/wolfpack new file mode 100755 index 0000000..9bef121 --- /dev/null +++ b/wolfpack @@ -0,0 +1,225 @@ +#!/bin/bash + +export IFS=" +" + +export logfile='/var/log/wolfpack.log' + +function helptext { + echo ' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + /\____ xxxxxxxxxxxxx AniNIX::WolfPack + // ~ / _\_____ Let loose the wolves to hunt, run, + / \ .. \/ and be free. They shall repay you + // /~_____/ with their prey. + /// \\/ / ~dev@aninix.net +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +wolfpack --alpha ~~ Tell the alpha to send the + pack members hunting. +wolfpack --member ~~ Send a member to train a pup + and bring back results. +wolfpack --stop-torrent ~~ Stop any transmission-cli + processes + +Available pup types (all pups are line-delimited with the type and +result location as the first two lines): +torrent ~~ Torrent the result of the + search term. + Will remove search line on + success. +shows ~~ Download the episode in a + show and increment. Shows + are listed in format + S01E01#Show terms +download ~~ Takes a list of commands to + run to arrive at a URL to + grab and downloads the last + URL. +search ~~ Uses the Google search + engine to return the top + five results for the search + terms on each line. +' +} + +function torrent { + if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then + echo "Need a valid argument." + return; + fi + export downloaddir=$(head -n 2 "$1" | tail -n 1) + if [ "$(pgrep -afc transmission-cli)" -ne 0 ]; then + echo "Transmission is still running. Aborting." | tee -a $logfile + return; + fi + for i in $(tail -n +3 "$1"); do + export searchstring="$(echo $i | sed 's/ /+/g')" + export searchlink="http://torrentproject.org/?t=""$searchstring" + export searchresult=$(wget -q -O - "$searchlink" | grep torrentproject.org | grep '
> $logfile + /usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" + export sepisode=$(echo $i | cut -f 1 -d '#') + export searchterms="$(echo $i | cut -f 2 -d '#')" + export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S') + export episode=$(echo $sepisode | cut -f 2 -d 'E') + export ep=$(printf "%02d\n" $((${episode#0} + 1))) + sed -i "s/$i/S"$season"E"$ep"#"$searchterms"/" "$1" + echo Downloaded and pup updated. >> $logfile + rm -Rf ~/.config/transmission/resume/* + rm -Rf ~/.config/transmission/torrents/* + rm -f "$downloaddir"/*/*.txt # Including this to avoid breaking Plex + rm -f "$downloaddir"/*/*.nfo + echo >> $logfile + else + # We didn't -- is there a new season? + export sepisode=$(echo $i | cut -f 1 -d '#') + export searchterms=$(echo $i | cut -f 2 -d '#') + export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S') + export episode=$(echo $sepisode | cut -f 2 -d 'E') + export newseason=$(printf "%02d\n" $((${season#0} + 1))) + export newsearch="$(echo $searchlink | sed 's/S'$season'E'$episode'/S'$newseason'E01/')" + export searchresult=$(wget -q -O - "$newsearch" | grep torrentproject.org | grep '
> $logfile + /usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" + sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1" + else + echo "No new season found for" $searchterms >> $logfile + fi + fi + done +} + +function download { + if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then + echo "Need a valid argument." + return; + fi + # Second line in the file is the directory to download to. + export downloaddir=$(head -n 2 "$1" | tail -n 1) + # The third line and any following lines should be + for i in $(tail -n +3 "$1"); do + #echo EVALUATING: $i + if [ "$i" == "" ]; then continue; fi + export comm=$(echo $i | sed "s#PREVRESULT#$nexturl#g" -) + export nexturl=$(bash -c "$comm") + #echo Found next url: $nexturl + done + export filename="$downloaddir"/"$(echo "$nexturl" | rev | cut -f 1 -d \/ | rev)" + #if file doesn't already exist.... + if [ ! -f "$filename" ]; then + echo Downloading $nexturl to $filename ... >> $logfile + export dedupterm="$(echo $1 | cut -f 1 -d '.' | rev | cut -f 1 -d '/' | rev)" + for i in $(find "$downloaddir" | grep -i "$dedupterm"); do + echo Deduped! Removed $i from $downloaddir >> $logfile + rm -Rf $i; + done + wget -q -O "$filename" "$nexturl" + echo Downloaded >> $logfile + fi +} + +function search { + if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then + echo "Need a valid argument." + return; + fi + export downloaddir=$(head -n 2 "$1" | tail -n 1) + for i in $(tail -n +3 "$1"); do + export searchstring=$(echo $i | sed 's/ /+/g') + export file="$downloaddir"/$searchstring'.txt' + echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file" + echo 'Ran search for '$searchstring' to '$file >> $logfile + done +} + +function stoptorrent { + # TODO This is destructive to all other transmission-CLI clients on the box. + killall transmission-cli +} + +if [ "$1" == "" ]; then + helptext; + exit; +fi + +echo '---' $(date) '---' >> $logfile +echo Started "$1" "$2" on $(uname -n) >> $logfile + +case "$1" in +"--alpha") + if [ "$2" == "" ]; then + export basedir="/usr/local/etc/WolfPack" + else + export basedir="$2"; + fi + echo "Starting members from pup in "$basedir >> $logfile + for i in $(find "$basedir" -type f | grep '.pup'); do + /usr/local/bin/wolfpack --member "$i"; + done + ;; + +"--member") + echo "Spawning a member for "$2 >> $logfile + whatismyip | tee -a /var/log/wolfpack.log + case "$(head -n 1 "$2")" in + "shows") + shows "$2" + ;; + "torrent") + torrent "$2" + ;; + "search") + search "$2" + ;; + "download") + download "$2" + ;; + *) + echo "Bad pup type." + ;; + esac + ;; + +"--stoptorrent") + stoptorrent + ;; + +*) + helptext + ;; +esac + +echo "Exited "$1 >> $logfile +echo '---' "$(date)" '---' >> $logfile +echo >> $logfile diff --git a/wolfpack-stoptorrent b/wolfpack-stoptorrent new file mode 100755 index 0000000..2f3c00b --- /dev/null +++ b/wolfpack-stoptorrent @@ -0,0 +1,2 @@ +#!/bin/bash +killall transmission-cli