AniNIX/WolfPack
AniNIX
/
WolfPack
2
0
Fork 0
Code Issues 4 Pull Requests Releases Wiki Activity

Major fixes for using standard libraries, protecting variable use, and torrent engine specification.

This commit is contained in:
cxford 2016-11-02 20:31:21 -05:00
parent 7425722bed
commit 02ab50f585
7 changed files with 135 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
1337x.torrentengine.sample Normal file → Executable file
View File

@ -1,6 +1,6 @@
torrentengine="https://1337x.to/search/%s/1/"
torrentterms='printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"'
magnetposition='4'
searchunifier='+'
magnetseparator=\"
torrentdebugging='true'
torrentengine=https://1337x.to/search/%s/1/
torrentterms=printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"
magnetposition=4
torrentdebugging=true
searchunifier=+
magnetseparator="

7
Makefile
View File

@ -1,6 +1,7 @@
compile: ./1337x.torrentengine.sample
if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; else git -C /usr/local/src/SharedLibraries pull; fi
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./1337x.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi
compile: ./tpb.torrentengine.sample
if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; fi
git -C /usr/local/src/SharedLibraries pull;
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./tpb.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi
@echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection."
install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip

13
README Normal file
View File

@ -0,0 +1,13 @@
=== AniNIX::WolfPack ===
WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line
Please note that the torrent and show functionalities are provided for legitimate peer-to-peer downloading and should not be used for illicit practices.
Some tools included are:
* wolfpack: This is the core script, using .pup files to search for various things on the Internet. Pup types include shows, torrents, downloads, and searches.
* test-torrent-torrent-engine: A number of sample torrent engines have been provided to look at major sites.
** Note: torrentproject.se is currently hard to scrape due to CloudFlare protections, and 1337x has a challenging reputation.
* offload-wolfpack: This allows other machines to run the wolfpack and then copy the results to a known destination.
We have also included example "pups" and some helper scripts for updating torrent blocklists.

17
test-torrent-engine Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
if [ -z "$1" ]; then
echo "Need search terms"
exit 1;
fi
source /usr/local/src/SharedLibraries/Bash/header
header Configuring...
if [ -z "$2" ]; then
configuretorrent /usr/local/etc/WolfPack/torrentengine
else
configuretorrent "$2"
fi
header Getting magnet link.
getmagnetlink "$(echo $1 | tr '#' "$searchunifier")"

12
torrentproject.torrentengine.sample
View File

@ -1,6 +1,6 @@
torrentengine='http://torrentproject.se/?t=%s'
torrentterms="printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1"
magnetposition='2'
torrentdebugging='true'
searchunifier='+'
magnetseparator=\'
torrentengine=http://torrentproject.se/?t=%s
torrentterms=printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1
magnetposition=2
torrentdebugging=true
searchunifier=+
magnetseparator='

6
tpb.torrentengine.sample Normal file
View File

@ -0,0 +1,6 @@
torrentengine=https://thepiratebay.org/search/%s/0/99/0/
torrentterms=printf https://thepiratebay.org; grep "/torrent/" | head -n 1 | cut -f 4 -d \"
magnetposition=4
torrentdebugging=true
searchunifier=%20
magnetseparator="

208
wolfpack
View File

@ -1,46 +1,46 @@
#!/bin/bash
source /usr/local/etc/WolfPack/torrentengine
source /usr/local/src/SharedLibraries/Bash/header
configuretorrent /usr/local/etc/WolfPack/torrentengine
export IFS="
"
export logfile='/var/log/wolfpack.log'
function helptext {
echo '
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/\____ xxxxxxxxxxxxx AniNIX::WolfPack
// ~ / _\_____ Let loose the wolves to hunt, run,
/ \ .. \/ and be free. They shall repay you
// /~_____/ with their prey.
/// \\/ / ~dev@aninix.net
/\____ xxxxxxxxxxxxx AniNIX::WolfPack
// ~ / _\_____ Let loose the wolves to hunt, run,
/ \ .. \/ and be free. They shall repay you
// /~_____/ with their prey.
/// \\/ / ~dev@aninix.net
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
wolfpack --alpha ~~ Tell the alpha to send the
pack members hunting.
wolfpack --member <pup> ~~ Send a member to train a pup
and bring back results.
wolfpack --stop-torrent ~~ Stop any transmission-cli
processes
wolfpack --alpha ~~ Tell the alpha to send the
pack members hunting.
wolfpack --member <pup> ~~ Send a member to train a pup
and bring back results.
wolfpack --stop-torrent ~~ Stop any transmission-cli
processes
Available pup types (all pups are line-delimited with the type and
result location as the first two lines):
torrent ~~ Torrent the result of the
search term.
Will remove search line on
success.
shows ~~ Download the episode in a
show and increment. Shows
are listed in format
S01E01#Show terms
download ~~ Takes a list of commands to
run to arrive at a URL to
grab and downloads the last
URL.
search ~~ Uses the Google search
engine to return the top
five results for the search
terms on each line.
'
Available pup types (all pups are line-delimited with the type and
result location as the first two lines):
torrent ~~ Torrent the result of the
search term.
Will remove search line on
success.
shows ~~ Download the episode in a
show and increment. Shows
are listed in format
S01E01#Show terms
download ~~ Takes a list of commands to
run to arrive at a URL to
grab and downloads the last
URL.
search ~~ Uses the Google search
engine to return the top
five results for the search
terms on each line.
'
}
function privacycheck {
@ -57,7 +57,6 @@ function transmissioncheck {
fi
}
function torrent {
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
echo "Need a valid argument."
@ -71,17 +70,7 @@ function torrent {
return;
fi
for i in $(tail -n +3 "$1"); do
export searchstring="$(echo $i | sed 's/ /+/g')"
export searchlink="$(printf "$torrentengine" "$searchstring")"
export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms")
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | head -n 1 |cut -f $magnetposition -d $magnetseparator)
if [ $torrentdebugging == "true" ]; then
echo $searchstring
echo $searchlink
echo $searchresult
echo $magnetlink
read
fi
getmagnetlink "$i"
if [ "$magnetlink" != "" ]; then
/usr/bin/transmission-cli -d 1000 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "/$i/d" "$1"
@ -100,27 +89,18 @@ function shows {
export downloaddir=$(head -n 2 "$1" | tail -n 1)
echo "Would download to "$downloaddir
for i in $(tail -n +3 "$1"); do
export searchsegment="$(echo $i | tr '#' '+' | tr ' ' '+')"
export searchlink="$(printf "$torrentengine" "$searchsegment")"
export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms")
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator)
if [ $torrentdebugging == "true" ]; then
echo $searchstring
echo $searchlink
echo $searchresult
echo $magnetlink
read
fi
getmagnetlink "$(echo $i | tr '#' "$searchunifier")"
export sepisode=$(echo $i | cut -f 1 -d '#')
export searchterms="$(echo $i | cut -f 2 -d '#')"
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E')
export newep=$(printf "%02d\n" $((${episode#0} + 1)))
export newseason="$(printf "%02d\n" $((${season#0} + 1)))"
if [ "$magnetlink" != "" ]; then
# We found an episode.
echo $i | tee -a $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
export sepisode=$(echo $i | cut -f 1 -d '#')
export searchterms="$(echo $i | cut -f 2 -d '#')"
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E')
export ep=$(printf "%02d\n" $((${episode#0} + 1)))
sed -i "s/$i/S"$season"E"$ep"#"$searchterms"/" "$1"
sed -i "s/$i/S"$season"E"$newep"#"$searchterms"/" "$1"
echo Downloaded and pup updated. | tee -a $logfile
rm -Rf ~/.config/transmission/resume/*
rm -Rf ~/.config/transmission/torrents/*
@ -129,30 +109,16 @@ function shows {
echo | tee -a $logfile
else
# We didn't -- is there a new season?
export sepisode=$(echo $i | cut -f 1 -d '#')
export searchterms=$(echo $i | cut -f 2 -d '#')
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E')
export newseason=$(printf "%02d\n" $((${season#0} + 1)))
export newsearch="$(echo $searchlink | sed 's/S'$season'E'$episode'/S'$newseason'E01/')"
export searchresult=$(wget -q -O - "$newsearch" | bash -c "$torrentterms")
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator)
if [ $torrentdebugging == "true" ]; then
echo $newseason
echo $newsearch
echo $searchresult
echo $magnetlink
read
fi
getmagnetlink "$(echo $i | sed 's/S'$season'E'$episode'/S'$newseason'E01/' | tr '#' "$searchunifier")"
if [ "$magnetlink" != "" ]; then
echo "S"$newseason"E01" $searchterms | tee -a $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1"
else
echo "No new season found for" $searchterms | tee -a $logfile
echo "No new season found for $searchterms" | tee -a $logfile
fi
fi
done
fi
done
}
function download {
@ -194,65 +160,57 @@ function search {
export searchstring=$(echo $i | sed 's/ /+/g')
export file="$downloaddir"/$searchstring'.txt'
echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file"
echo 'Ran search for '$searchstring' to '$file | tee -a $logfile
echo "Ran search for $searchstring to $file" | tee -a $logfile
done
}
function stoptorrent {
# TODO This is destructive to all other transmission-CLI clients on the box.
killall transmission-cli
}
if [ "$1" == "" ]; then
helptext;
exit 1;
helptext;
exit 1;
fi
echo '---' $(date) '---' | tee -a $logfile
echo Started "$1" "$2" on $(uname -n) | tee -a $logfile
case "$1" in
"--alpha")
if [ "$2" == "" ]; then
export basedir="/usr/local/etc/WolfPack"
else
export basedir="$2";
fi
echo "Starting members from pup in "$basedir | tee -a $logfile
for i in $(find "$basedir" -type f | grep '.pup'); do
/usr/local/bin/wolfpack --member "$i";
done
;;
"--alpha")
if [ "$2" == "" ]; then
export basedir="/usr/local/etc/WolfPack"
else
export basedir="$2";
fi
echo "Starting members from pup in "$basedir | tee -a $logfile
for i in $(find "$basedir" -type f | grep '.pup'); do
/usr/local/bin/wolfpack --member "$i";
done
;;
"--member")
echo "Spawning a member for "$2 | tee -a $logfile
whatismyip | tee -a /var/log/wolfpack.log
case "$(head -n 1 "$2")" in
"shows")
shows "$2"
;;
"torrent")
torrent "$2"
;;
"search")
search "$2"
;;
"download")
download "$2"
;;
*)
echo "Bad pup type."
;;
esac
;;
"--member")
echo "Spawning a member for "$2 | tee -a $logfile
whatismyip | tee -a /var/log/wolfpack.log
case "$(head -n 1 "$2")" in
"shows")
shows "$2"
;;
"torrent")
torrent "$2"
;;
"search")
search "$2"
;;
"download")
download "$2"
;;
*)
echo "Bad pup type."
;;
esac
;;
"--stoptorrent")
stoptorrent
;;
*)
helptext
;;
*)
helptext
;;
esac
echo "Exited "$1 | tee -a $logfile