Converting from Bazaar.

Old log:
------------------------------------------------------------
revno: 8
committer: root <root@aninix.net>
branch nick: WolfPack
timestamp: Tue 2016-07-19 13:52:29 -0500
message:
  Clearing some read statements
------------------------------------------------------------
revno: 7
committer: root <root@aninix.net>
branch nick: WolfPack
timestamp: Tue 2016-07-19 13:36:23 -0500
message:
  Fix for torrent protocol
------------------------------------------------------------
revno: 6
committer: dev <dev@aninix.net>
branch nick: WolfPack
timestamp: Mon 2016-07-18 10:23:17 -0500
message:
  Massive fixes; offloading updates; torrent function pup type updated
------------------------------------------------------------
revno: 5
committer: dev <dev@aninix.net>
branch nick: WolfPack
timestamp: Tue 2016-05-31 14:42:28 -0500
message:
  Removing ghost-wolfpack in favor of systemd vpn client.
------------------------------------------------------------
revno: 4
committer: dev <dev@aninix.net>
branch nick: WolfPack
timestamp: Sun 2016-05-15 21:35:02 -0500
message:
  Updated to include privacy VPN and offloading options.
------------------------------------------------------------
revno: 3
committer: dev <dev@aninix.net>
branch nick: WolfPack
timestamp: Thu 2016-05-12 20:51:23 -0500
message:
  Adding ghost-wolfpack script for sites using VPNs for privacy
------------------------------------------------------------
revno: 2
committer: dev <dev@aninix.net>
branch nick: WolfPack
timestamp: Thu 2016-05-12 15:58:05 -0500
message:
  Updated to use the googler package from the ArchLinux AUR for google searches and store as text files. Sufficient for now.
------------------------------------------------------------
revno: 1
committer: dev <dev@aninix.net>
branch nick: WolfPack
timestamp: Thu 2016-05-12 14:45:12 -0500
message:
  Adding all but search example -- search API is still in flux with no good command-line search option.
This commit is contained in:
cxford 2016-08-04 12:08:16 -05:00
commit f2864631fb
9 changed files with 301 additions and 0 deletions

29
Makefile Normal file
View File

@ -0,0 +1,29 @@
build:
Nothing to build -- this is a bash script.
configure:
mkdir -p /usr/local/etc/WolfPack
chmod 0770 /usr/local/etc/WolfPack
cp ./*.pup /usr/local/etc/WolfPack
chmod 0660 /usr/local/etc/WolfPack/*
@echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups"
@echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection."
install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip
cp -p /usr/local/src/WolfPack/wolfpack /usr/local/bin/wolfpack
cp -p /usr/local/src/WolfPack/wolfpack-stoptorrent /usr/local/bin/wolfpack-stoptorrent
[ ! -f /root/.bashrc ] || (grep 'transmission-cli' /root/.bashrc) || echo 'alias transmission-cli="transmission-cli -d 600 -u 1 -er -w /srv/yggdrasil/new_acquisition -b"' >> /root/.bashrc
chmod 0700 /usr/local/bin/wolfpack-stoptorrent
chmod 0755 /usr/local/bin/wolfpack
chown root:root /usr/local/bin/wolfpack*
reverse:
cp /usr/local/bin/wolfpack* .
[ ! -f /root/bin/offload-wolfpack ] || cp /root/bin/offload-wolfpack .
offload: offload-wolfpack /root/.ssh/id_rsa /root/.ssh/id_rsa.pub
cp offload-wolfpack /root/bin
chmod 0700 /root/bin/offload-wolfpack
chown root:root /root/bin/offload-wolfpack

4
example-download.pup Normal file
View File

@ -0,0 +1,4 @@
download
/srv/WolfPackResults
wget -q -O - 'https://www.archlinux.org/download/' | grep surlyjake | cut -f 2 -d \" | head -n 1
echo PREVRESULT/$(wget -q -O - PREVRESULT | grep dual.iso | head -n 1 | cut -f 2 -d \")

3
example-search.pup Normal file
View File

@ -0,0 +1,3 @@
search
/tmp
Disturbed 2015 concerts

3
example-shows.pup Normal file
View File

@ -0,0 +1,3 @@
shows
/srv/yggdrasil/new_acquisition
S01E01#Tech-TV

3
example-torrent.pup Normal file
View File

@ -0,0 +1,3 @@
torrent
/srv/yggdrasil/new_acquisition
ArchLinux ISO

14
offload-wolfpack Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash
/usr/local/bin/wolfpack --alpha
sleep 10
rsync -avz /srv/yggdrasil/new_acquisition/* cxford@10.0.1.3:/srv/yggdrasil/new_acquisition &>>/var/log/wolfpack.log
ls -l /srv/yggdrasil/new_acquisition >> /var/log/wolfpack.log
echo Files transferred from $(uname -n). >> /var/log/wolfpack.log
rm -Rf /srv/yggdrasil/new_acquisition/*
cat /var/log/wolfpack.log | ssh cxford@aninix.net "cat >> /var/log/wolfpack.log"
rm /var/log/wolfpack.log
for i in $(ls /usr/local/etc/WolfPack/*.pup); do
export cmdstring="$(echo $i | sed "s/pup/$(uname -n)/")"
scp "$i" cxford@aninix.net:"$cmdstring"
done

18
update-transmission-blocklist Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash
if [ $(pgrep -afc $0) -ne 0 ]; then
echo Already running!
exit 1;
fi
cd /root/.config/transmission/blocklists/
rm -Rf *
wget 'http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz' -O blocklist.gz
gunzip blocklist.gz;
while [ "$?" -ne 0 ]; do
echo Error $?: Couldn\'t get blocklist.
wget 'http://list.iblocklist.com/?list=bt_level1&fileformat=p2p&archiveformat=gz' -O blocklist.gz
gunzip blocklist.gz;
done
/usr/bin/transmission-cli -b notorrent # Update the blocklist to a .bin format

225
wolfpack Executable file
View File

@ -0,0 +1,225 @@
#!/bin/bash
export IFS="
"
export logfile='/var/log/wolfpack.log'
function helptext {
echo '
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/\____ xxxxxxxxxxxxx AniNIX::WolfPack
// ~ / _\_____ Let loose the wolves to hunt, run,
/ \ .. \/ and be free. They shall repay you
// /~_____/ with their prey.
/// \\/ / ~dev@aninix.net
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
wolfpack --alpha ~~ Tell the alpha to send the
pack members hunting.
wolfpack --member <pup> ~~ Send a member to train a pup
and bring back results.
wolfpack --stop-torrent ~~ Stop any transmission-cli
processes
Available pup types (all pups are line-delimited with the type and
result location as the first two lines):
torrent ~~ Torrent the result of the
search term.
Will remove search line on
success.
shows ~~ Download the episode in a
show and increment. Shows
are listed in format
S01E01#Show terms
download ~~ Takes a list of commands to
run to arrive at a URL to
grab and downloads the last
URL.
search ~~ Uses the Google search
engine to return the top
five results for the search
terms on each line.
'
}
function torrent {
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
echo "Need a valid argument."
return;
fi
export downloaddir=$(head -n 2 "$1" | tail -n 1)
if [ "$(pgrep -afc transmission-cli)" -ne 0 ]; then
echo "Transmission is still running. Aborting." | tee -a $logfile
return;
fi
for i in $(tail -n +3 "$1"); do
export searchstring="$(echo $i | sed 's/ /+/g')"
export searchlink="http://torrentproject.org/?t=""$searchstring"
export searchresult=$(wget -q -O - "$searchlink" | grep torrentproject.org | grep '<div><span><a href=' | cut -f 2 -d \')
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f 2 -d \')
if [ "$magnetlink" != "" ]; then
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "/$i/d" "$1"
echo Downloaded and removed $i | tee -a $logfile
fi
done
}
function shows {
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
echo "Need a valid argument."
return;
fi
if [ "$(pgrep -afc transmission-cli)" -ne 0 ]; then
echo "Transmission is still running. Aborting." | tee -a $logfile
return;
fi
export downloaddir=$(head -n 2 "$1" | tail -n 1)
echo "Would download to "$downloaddir
for i in $(tail -n +3 "$1"); do
export searchsegment="$(echo $i | tr '#' '+' | tr ' ' '+')"
export searchlink="http://torrentproject.org/?t=""$searchsegment"
export searchresult=$(wget -q -O - "$searchlink" | grep torrentproject.org | grep '<div><span><a href=' | cut -f 2 -d \')
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f 2 -d \')
if [ "$magnetlink" != "" ]; then
# We found an episode.
echo $i >> $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
export sepisode=$(echo $i | cut -f 1 -d '#')
export searchterms="$(echo $i | cut -f 2 -d '#')"
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E')
export ep=$(printf "%02d\n" $((${episode#0} + 1)))
sed -i "s/$i/S"$season"E"$ep"#"$searchterms"/" "$1"
echo Downloaded and pup updated. >> $logfile
rm -Rf ~/.config/transmission/resume/*
rm -Rf ~/.config/transmission/torrents/*
rm -f "$downloaddir"/*/*.txt # Including this to avoid breaking Plex
rm -f "$downloaddir"/*/*.nfo
echo >> $logfile
else
# We didn't -- is there a new season?
export sepisode=$(echo $i | cut -f 1 -d '#')
export searchterms=$(echo $i | cut -f 2 -d '#')
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E')
export newseason=$(printf "%02d\n" $((${season#0} + 1)))
export newsearch="$(echo $searchlink | sed 's/S'$season'E'$episode'/S'$newseason'E01/')"
export searchresult=$(wget -q -O - "$newsearch" | grep torrentproject.org | grep '<div><span><a href=' | cut -f 2 -d \')
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f 2 -d \')
if [ "$magnetlinki" != "" ]; then
echo "S"$newseason"E01" $searchterms >> $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1"
else
echo "No new season found for" $searchterms >> $logfile
fi
fi
done
}
function download {
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
echo "Need a valid argument."
return;
fi
# Second line in the file is the directory to download to.
export downloaddir=$(head -n 2 "$1" | tail -n 1)
# The third line and any following lines should be
for i in $(tail -n +3 "$1"); do
#echo EVALUATING: $i
if [ "$i" == "" ]; then continue; fi
export comm=$(echo $i | sed "s#PREVRESULT#$nexturl#g" -)
export nexturl=$(bash -c "$comm")
#echo Found next url: $nexturl
done
export filename="$downloaddir"/"$(echo "$nexturl" | rev | cut -f 1 -d \/ | rev)"
#if file doesn't already exist....
if [ ! -f "$filename" ]; then
echo Downloading $nexturl to $filename ... >> $logfile
export dedupterm="$(echo $1 | cut -f 1 -d '.' | rev | cut -f 1 -d '/' | rev)"
for i in $(find "$downloaddir" | grep -i "$dedupterm"); do
echo Deduped! Removed $i from $downloaddir >> $logfile
rm -Rf $i;
done
wget -q -O "$filename" "$nexturl"
echo Downloaded >> $logfile
fi
}
function search {
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
echo "Need a valid argument."
return;
fi
export downloaddir=$(head -n 2 "$1" | tail -n 1)
for i in $(tail -n +3 "$1"); do
export searchstring=$(echo $i | sed 's/ /+/g')
export file="$downloaddir"/$searchstring'.txt'
echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file"
echo 'Ran search for '$searchstring' to '$file >> $logfile
done
}
function stoptorrent {
# TODO This is destructive to all other transmission-CLI clients on the box.
killall transmission-cli
}
if [ "$1" == "" ]; then
helptext;
exit;
fi
echo '---' $(date) '---' >> $logfile
echo Started "$1" "$2" on $(uname -n) >> $logfile
case "$1" in
"--alpha")
if [ "$2" == "" ]; then
export basedir="/usr/local/etc/WolfPack"
else
export basedir="$2";
fi
echo "Starting members from pup in "$basedir >> $logfile
for i in $(find "$basedir" -type f | grep '.pup'); do
/usr/local/bin/wolfpack --member "$i";
done
;;
"--member")
echo "Spawning a member for "$2 >> $logfile
whatismyip | tee -a /var/log/wolfpack.log
case "$(head -n 1 "$2")" in
"shows")
shows "$2"
;;
"torrent")
torrent "$2"
;;
"search")
search "$2"
;;
"download")
download "$2"
;;
*)
echo "Bad pup type."
;;
esac
;;
"--stoptorrent")
stoptorrent
;;
*)
helptext
;;
esac
echo "Exited "$1 >> $logfile
echo '---' "$(date)" '---' >> $logfile
echo >> $logfile

2
wolfpack-stoptorrent Executable file
View File

@ -0,0 +1,2 @@
#!/bin/bash
killall transmission-cli