AniNIX/WolfPack
AniNIX
/
WolfPack
2
0
Fork 0
Code Issues 4 Pull Requests Releases Wiki Activity

Major fixes for using standard libraries, protecting variable use, and torrent engine specification.

This commit is contained in:
cxford 2016-11-02 20:31:21 -05:00
parent 7425722bed
commit 02ab50f585
7 changed files with 135 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
1337x.torrentengine.sample Normal file → Executable file
View File

@ -1,6 +1,6 @@
torrentengine="https://1337x.to/search/%s/1/" torrentengine=https://1337x.to/search/%s/1/
torrentterms='printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"' torrentterms=printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"
magnetposition='4' magnetposition=4
searchunifier='+' torrentdebugging=true
magnetseparator=\" searchunifier=+
torrentdebugging='true' magnetseparator="

7
Makefile
View File

@ -1,6 +1,7 @@
compile: ./1337x.torrentengine.sample compile: ./tpb.torrentengine.sample
if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; else git -C /usr/local/src/SharedLibraries pull; fi if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; fi
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./1337x.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi git -C /usr/local/src/SharedLibraries pull;
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./tpb.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi
@echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection." @echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection."
install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip

13
README Normal file
View File

@ -0,0 +1,13 @@
=== AniNIX::WolfPack ===
WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line
Please note that the torrent and show functionalities are provided for legitimate peer-to-peer downloading and should not be used for illicit practices.
Some tools included are:
* wolfpack: This is the core script, using .pup files to search for various things on the Internet. Pup types include shows, torrents, downloads, and searches.
* test-torrent-torrent-engine: A number of sample torrent engines have been provided to look at major sites.
** Note: torrentproject.se is currently hard to scrape due to CloudFlare protections, and 1337x has a challenging reputation.
* offload-wolfpack: This allows other machines to run the wolfpack and then copy the results to a known destination.
We have also included example "pups" and some helper scripts for updating torrent blocklists.

17
test-torrent-engine Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
if [ -z "$1" ]; then
echo "Need search terms"
exit 1;
fi
source /usr/local/src/SharedLibraries/Bash/header
header Configuring...
if [ -z "$2" ]; then
configuretorrent /usr/local/etc/WolfPack/torrentengine
else
configuretorrent "$2"
fi
header Getting magnet link.
getmagnetlink "$(echo $1 | tr '#' "$searchunifier")"

12
torrentproject.torrentengine.sample
View File

@ -1,6 +1,6 @@
torrentengine='http://torrentproject.se/?t=%s' torrentengine=http://torrentproject.se/?t=%s
torrentterms="printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1" torrentterms=printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1
magnetposition='2' magnetposition=2
torrentdebugging='true' torrentdebugging=true
searchunifier='+' searchunifier=+
magnetseparator=\' magnetseparator='

6
tpb.torrentengine.sample Normal file
View File

@ -0,0 +1,6 @@
torrentengine=https://thepiratebay.org/search/%s/0/99/0/
torrentterms=printf https://thepiratebay.org; grep "/torrent/" | head -n 1 | cut -f 4 -d \"
magnetposition=4
torrentdebugging=true
searchunifier=%20
magnetseparator="

208
wolfpack
View File

@ -1,46 +1,46 @@
#!/bin/bash #!/bin/bash
source /usr/local/etc/WolfPack/torrentengine source /usr/local/src/SharedLibraries/Bash/header
configuretorrent /usr/local/etc/WolfPack/torrentengine
export IFS=" export IFS="
" "
export logfile='/var/log/wolfpack.log' export logfile='/var/log/wolfpack.log'
function helptext { function helptext {
echo ' echo '
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/\____ xxxxxxxxxxxxx AniNIX::WolfPack /\____ xxxxxxxxxxxxx AniNIX::WolfPack
// ~ / _\_____ Let loose the wolves to hunt, run, // ~ / _\_____ Let loose the wolves to hunt, run,
/ \ .. \/ and be free. They shall repay you / \ .. \/ and be free. They shall repay you
// /~_____/ with their prey. // /~_____/ with their prey.
/// \\/ / ~dev@aninix.net /// \\/ / ~dev@aninix.net
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
wolfpack --alpha ~~ Tell the alpha to send the wolfpack --alpha ~~ Tell the alpha to send the
pack members hunting. pack members hunting.
wolfpack --member <pup> ~~ Send a member to train a pup wolfpack --member <pup> ~~ Send a member to train a pup
and bring back results. and bring back results.
wolfpack --stop-torrent ~~ Stop any transmission-cli wolfpack --stop-torrent ~~ Stop any transmission-cli
processes processes
Available pup types (all pups are line-delimited with the type and Available pup types (all pups are line-delimited with the type and
result location as the first two lines): result location as the first two lines):
torrent ~~ Torrent the result of the torrent ~~ Torrent the result of the
search term. search term.
Will remove search line on Will remove search line on
success. success.
shows ~~ Download the episode in a shows ~~ Download the episode in a
show and increment. Shows show and increment. Shows
are listed in format are listed in format
S01E01#Show terms S01E01#Show terms
download ~~ Takes a list of commands to download ~~ Takes a list of commands to
run to arrive at a URL to run to arrive at a URL to
grab and downloads the last grab and downloads the last
URL. URL.
search ~~ Uses the Google search search ~~ Uses the Google search
engine to return the top engine to return the top
five results for the search five results for the search
terms on each line. terms on each line.
' '
} }
function privacycheck { function privacycheck {
@ -57,7 +57,6 @@ function transmissioncheck {
fi fi
} }
function torrent { function torrent {
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
echo "Need a valid argument." echo "Need a valid argument."
@ -71,17 +70,7 @@ function torrent {
return; return;
fi fi
for i in $(tail -n +3 "$1"); do for i in $(tail -n +3 "$1"); do
export searchstring="$(echo $i | sed 's/ /+/g')" getmagnetlink "$i"
export searchlink="$(printf "$torrentengine" "$searchstring")"
export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms")
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | head -n 1 |cut -f $magnetposition -d $magnetseparator)
if [ $torrentdebugging == "true" ]; then
echo $searchstring
echo $searchlink
echo $searchresult
echo $magnetlink
read
fi
if [ "$magnetlink" != "" ]; then if [ "$magnetlink" != "" ]; then
/usr/bin/transmission-cli -d 1000 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" /usr/bin/transmission-cli -d 1000 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "/$i/d" "$1" sed -i "/$i/d" "$1"
@ -100,27 +89,18 @@ function shows {
export downloaddir=$(head -n 2 "$1" | tail -n 1) export downloaddir=$(head -n 2 "$1" | tail -n 1)
echo "Would download to "$downloaddir echo "Would download to "$downloaddir
for i in $(tail -n +3 "$1"); do for i in $(tail -n +3 "$1"); do
export searchsegment="$(echo $i | tr '#' '+' | tr ' ' '+')" getmagnetlink "$(echo $i | tr '#' "$searchunifier")"
export searchlink="$(printf "$torrentengine" "$searchsegment")" export sepisode=$(echo $i | cut -f 1 -d '#')
export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms") export searchterms="$(echo $i | cut -f 2 -d '#')"
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator) export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
if [ $torrentdebugging == "true" ]; then export episode=$(echo $sepisode | cut -f 2 -d 'E')
echo $searchstring export newep=$(printf "%02d\n" $((${episode#0} + 1)))
echo $searchlink export newseason="$(printf "%02d\n" $((${season#0} + 1)))"
echo $searchresult
echo $magnetlink
read
fi
if [ "$magnetlink" != "" ]; then if [ "$magnetlink" != "" ]; then
# We found an episode. # We found an episode.
echo $i | tee -a $logfile echo $i | tee -a $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" /usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
export sepisode=$(echo $i | cut -f 1 -d '#') sed -i "s/$i/S"$season"E"$newep"#"$searchterms"/" "$1"
export searchterms="$(echo $i | cut -f 2 -d '#')"
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E')
export ep=$(printf "%02d\n" $((${episode#0} + 1)))
sed -i "s/$i/S"$season"E"$ep"#"$searchterms"/" "$1"
echo Downloaded and pup updated. | tee -a $logfile echo Downloaded and pup updated. | tee -a $logfile
rm -Rf ~/.config/transmission/resume/* rm -Rf ~/.config/transmission/resume/*
rm -Rf ~/.config/transmission/torrents/* rm -Rf ~/.config/transmission/torrents/*
@ -129,30 +109,16 @@ function shows {
echo | tee -a $logfile echo | tee -a $logfile
else else
# We didn't -- is there a new season? # We didn't -- is there a new season?
export sepisode=$(echo $i | cut -f 1 -d '#') getmagnetlink "$(echo $i | sed 's/S'$season'E'$episode'/S'$newseason'E01/' | tr '#' "$searchunifier")"
export searchterms=$(echo $i | cut -f 2 -d '#')
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E')
export newseason=$(printf "%02d\n" $((${season#0} + 1)))
export newsearch="$(echo $searchlink | sed 's/S'$season'E'$episode'/S'$newseason'E01/')"
export searchresult=$(wget -q -O - "$newsearch" | bash -c "$torrentterms")
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator)
if [ $torrentdebugging == "true" ]; then
echo $newseason
echo $newsearch
echo $searchresult
echo $magnetlink
read
fi
if [ "$magnetlink" != "" ]; then if [ "$magnetlink" != "" ]; then
echo "S"$newseason"E01" $searchterms | tee -a $logfile echo "S"$newseason"E01" $searchterms | tee -a $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" /usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1" sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1"
else else
echo "No new season found for" $searchterms | tee -a $logfile echo "No new season found for $searchterms" | tee -a $logfile
fi fi
fi fi
done done
} }
function download { function download {
@ -194,65 +160,57 @@ function search {
export searchstring=$(echo $i | sed 's/ /+/g') export searchstring=$(echo $i | sed 's/ /+/g')
export file="$downloaddir"/$searchstring'.txt' export file="$downloaddir"/$searchstring'.txt'
echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file" echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file"
echo 'Ran search for '$searchstring' to '$file | tee -a $logfile echo "Ran search for $searchstring to $file" | tee -a $logfile
done done
} }
function stoptorrent {
# TODO This is destructive to all other transmission-CLI clients on the box.
killall transmission-cli
}
if [ "$1" == "" ]; then if [ "$1" == "" ]; then
helptext; helptext;
exit 1; exit 1;
fi fi
echo '---' $(date) '---' | tee -a $logfile echo '---' $(date) '---' | tee -a $logfile
echo Started "$1" "$2" on $(uname -n) | tee -a $logfile echo Started "$1" "$2" on $(uname -n) | tee -a $logfile
case "$1" in case "$1" in
"--alpha") "--alpha")
if [ "$2" == "" ]; then if [ "$2" == "" ]; then
export basedir="/usr/local/etc/WolfPack" export basedir="/usr/local/etc/WolfPack"
else else
export basedir="$2"; export basedir="$2";
fi fi
echo "Starting members from pup in "$basedir | tee -a $logfile echo "Starting members from pup in "$basedir | tee -a $logfile
for i in $(find "$basedir" -type f | grep '.pup'); do for i in $(find "$basedir" -type f | grep '.pup'); do
/usr/local/bin/wolfpack --member "$i"; /usr/local/bin/wolfpack --member "$i";
done done
;; ;;
"--member") "--member")
echo "Spawning a member for "$2 | tee -a $logfile echo "Spawning a member for "$2 | tee -a $logfile
whatismyip | tee -a /var/log/wolfpack.log whatismyip | tee -a /var/log/wolfpack.log
case "$(head -n 1 "$2")" in case "$(head -n 1 "$2")" in
"shows") "shows")
shows "$2" shows "$2"
;; ;;
"torrent") "torrent")
torrent "$2" torrent "$2"
;; ;;
"search") "search")
search "$2" search "$2"
;; ;;
"download") "download")
download "$2" download "$2"
;; ;;
*) *)
echo "Bad pup type." echo "Bad pup type."
;; ;;
esac esac
;; ;;
"--stoptorrent") *)
stoptorrent helptext
;; ;;
*)
helptext
;;
esac esac
echo "Exited "$1 | tee -a $logfile echo "Exited "$1 | tee -a $logfile