AniNIX/WolfPack
AniNIX
/
WolfPack
2
0
Fork 0
Code Issues 4 Pull Requests Releases Wiki Activity

Major fixes for using standard libraries, protecting variable use, and torrent engine specification.

This commit is contained in:
cxford 2016-11-02 20:31:21 -05:00
parent 7425722bed
commit 02ab50f585
7 changed files with 135 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
1337x.torrentengine.sample Normal file → Executable file
View File

@ -1,6 +1,6 @@
torrentengine="https://1337x.to/search/%s/1/" torrentengine=https://1337x.to/search/%s/1/
torrentterms='printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"' torrentterms=printf https://1337x.to; grep "/torrent/" | head -n 1 | cut -f 10 -d \"
magnetposition='4' magnetposition=4
searchunifier='+' torrentdebugging=true
magnetseparator=\" searchunifier=+
torrentdebugging='true' magnetseparator="

7
Makefile
View File

@ -1,6 +1,7 @@
compile: ./1337x.torrentengine.sample compile: ./tpb.torrentengine.sample
if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; else git -C /usr/local/src/SharedLibraries pull; fi if [ ! -d /usr/local/src/SharedLibraries ]; then git -C /usr/local/src/ clone https://aninix.net/foundation/SharedLibraries; fi
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./1337x.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi git -C /usr/local/src/SharedLibraries pull;
if [ ! -d /usr/local/etc/WolfPack ]; then mkdir -p /usr/local/etc/WolfPack; chmod 0770 /usr/local/etc/WolfPack; cp ./*.pup /usr/local/etc/WolfPack; cp ./tpb.torrentengine.sample /usr/local/etc/WolfPack; chmod 0660 /usr/local/etc/WolfPack/*; echo "Default configuration directory of /usr/local/etc/WolfPack created with example pups and torrent engine."; fi
@echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection." @echo "Run make install and add '/usr/local/bin/wolfpack --alpha' to root's crontab to schedule data collection."
install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip install: /usr/bin/wget /usr/bin/transmission-cli /usr/bin/googler wolfpack wolfpack-stoptorrent /usr/local/bin/whatismyip

13
README Normal file
View File

@ -0,0 +1,13 @@
=== AniNIX::WolfPack ===
WolfPack is a Internet search engine, bot crawler, and download utility designed to be scheduled and run on the command line
Please note that the torrent and show functionalities are provided for legitimate peer-to-peer downloading and should not be used for illicit practices.
Some tools included are:
* wolfpack: This is the core script, using .pup files to search for various things on the Internet. Pup types include shows, torrents, downloads, and searches.
* test-torrent-torrent-engine: A number of sample torrent engines have been provided to look at major sites.
** Note: torrentproject.se is currently hard to scrape due to CloudFlare protections, and 1337x has a challenging reputation.
* offload-wolfpack: This allows other machines to run the wolfpack and then copy the results to a known destination.
We have also included example "pups" and some helper scripts for updating torrent blocklists.

17
test-torrent-engine Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
if [ -z "$1" ]; then
echo "Need search terms"
exit 1;
fi
source /usr/local/src/SharedLibraries/Bash/header
header Configuring...
if [ -z "$2" ]; then
configuretorrent /usr/local/etc/WolfPack/torrentengine
else
configuretorrent "$2"
fi
header Getting magnet link.
getmagnetlink "$(echo $1 | tr '#' "$searchunifier")"

12
torrentproject.torrentengine.sample
View File

@ -1,6 +1,6 @@
torrentengine='http://torrentproject.se/?t=%s' torrentengine=http://torrentproject.se/?t=%s
torrentterms="printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1" torrentterms=printf http://torrentproject.se; grep "verified" | head -n 1 | cut -f 2 -d \' | head -n 1
magnetposition='2' magnetposition=2
torrentdebugging='true' torrentdebugging=true
searchunifier='+' searchunifier=+
magnetseparator=\' magnetseparator='

6
tpb.torrentengine.sample Normal file
View File

@ -0,0 +1,6 @@
torrentengine=https://thepiratebay.org/search/%s/0/99/0/
torrentterms=printf https://thepiratebay.org; grep "/torrent/" | head -n 1 | cut -f 4 -d \"
magnetposition=4
torrentdebugging=true
searchunifier=%20
magnetseparator="

70
wolfpack
View File

@ -1,11 +1,11 @@
#!/bin/bash #!/bin/bash
source /usr/local/etc/WolfPack/torrentengine source /usr/local/src/SharedLibraries/Bash/header
configuretorrent /usr/local/etc/WolfPack/torrentengine
export IFS=" export IFS="
" "
export logfile='/var/log/wolfpack.log' export logfile='/var/log/wolfpack.log'
function helptext { function helptext {
echo ' echo '
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -57,7 +57,6 @@ function transmissioncheck {
fi fi
} }
function torrent { function torrent {
if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then if [ "$1" == "" ] || [ $(wc -l "$1" | cut -f 1 -d ' ') -lt 3 ]; then
echo "Need a valid argument." echo "Need a valid argument."
@ -71,17 +70,7 @@ function torrent {
return; return;
fi fi
for i in $(tail -n +3 "$1"); do for i in $(tail -n +3 "$1"); do
export searchstring="$(echo $i | sed 's/ /+/g')" getmagnetlink "$i"
export searchlink="$(printf "$torrentengine" "$searchstring")"
export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms")
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | head -n 1 |cut -f $magnetposition -d $magnetseparator)
if [ $torrentdebugging == "true" ]; then
echo $searchstring
echo $searchlink
echo $searchresult
echo $magnetlink
read
fi
if [ "$magnetlink" != "" ]; then if [ "$magnetlink" != "" ]; then
/usr/bin/transmission-cli -d 1000 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" /usr/bin/transmission-cli -d 1000 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "/$i/d" "$1" sed -i "/$i/d" "$1"
@ -100,27 +89,18 @@ function shows {
export downloaddir=$(head -n 2 "$1" | tail -n 1) export downloaddir=$(head -n 2 "$1" | tail -n 1)
echo "Would download to "$downloaddir echo "Would download to "$downloaddir
for i in $(tail -n +3 "$1"); do for i in $(tail -n +3 "$1"); do
export searchsegment="$(echo $i | tr '#' '+' | tr ' ' '+')" getmagnetlink "$(echo $i | tr '#' "$searchunifier")"
export searchlink="$(printf "$torrentengine" "$searchsegment")"
export searchresult=$(wget -q -O - "$searchlink" | bash -c "$torrentterms")
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator)
if [ $torrentdebugging == "true" ]; then
echo $searchstring
echo $searchlink
echo $searchresult
echo $magnetlink
read
fi
if [ "$magnetlink" != "" ]; then
# We found an episode.
echo $i | tee -a $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
export sepisode=$(echo $i | cut -f 1 -d '#') export sepisode=$(echo $i | cut -f 1 -d '#')
export searchterms="$(echo $i | cut -f 2 -d '#')" export searchterms="$(echo $i | cut -f 2 -d '#')"
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S') export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E') export episode=$(echo $sepisode | cut -f 2 -d 'E')
export ep=$(printf "%02d\n" $((${episode#0} + 1))) export newep=$(printf "%02d\n" $((${episode#0} + 1)))
sed -i "s/$i/S"$season"E"$ep"#"$searchterms"/" "$1" export newseason="$(printf "%02d\n" $((${season#0} + 1)))"
if [ "$magnetlink" != "" ]; then
# We found an episode.
echo $i | tee -a $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "s/$i/S"$season"E"$newep"#"$searchterms"/" "$1"
echo Downloaded and pup updated. | tee -a $logfile echo Downloaded and pup updated. | tee -a $logfile
rm -Rf ~/.config/transmission/resume/* rm -Rf ~/.config/transmission/resume/*
rm -Rf ~/.config/transmission/torrents/* rm -Rf ~/.config/transmission/torrents/*
@ -129,27 +109,13 @@ function shows {
echo | tee -a $logfile echo | tee -a $logfile
else else
# We didn't -- is there a new season? # We didn't -- is there a new season?
export sepisode=$(echo $i | cut -f 1 -d '#') getmagnetlink "$(echo $i | sed 's/S'$season'E'$episode'/S'$newseason'E01/' | tr '#' "$searchunifier")"
export searchterms=$(echo $i | cut -f 2 -d '#')
export season=$(echo $sepisode | cut -f 1 -d 'E' | cut -f 2 -d 'S')
export episode=$(echo $sepisode | cut -f 2 -d 'E')
export newseason=$(printf "%02d\n" $((${season#0} + 1)))
export newsearch="$(echo $searchlink | sed 's/S'$season'E'$episode'/S'$newseason'E01/')"
export searchresult=$(wget -q -O - "$newsearch" | bash -c "$torrentterms")
export magnetlink=$(wget -q -O - "$searchresult" | grep 'magnet:?' | cut -f $magnetposition -d $magnetseparator)
if [ $torrentdebugging == "true" ]; then
echo $newseason
echo $newsearch
echo $searchresult
echo $magnetlink
read
fi
if [ "$magnetlink" != "" ]; then if [ "$magnetlink" != "" ]; then
echo "S"$newseason"E01" $searchterms | tee -a $logfile echo "S"$newseason"E01" $searchterms | tee -a $logfile
/usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink" /usr/bin/transmission-cli -d 600 -u 1 -er -w "$downloaddir" -b -f "/usr/local/bin/wolfpack-stoptorrent" "$magnetlink"
sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1" sed -i "s/$i/S"$newseason"E01#"$searchterms"/" "$1"
else else
echo "No new season found for" $searchterms | tee -a $logfile echo "No new season found for $searchterms" | tee -a $logfile
fi fi
fi fi
done done
@ -194,14 +160,10 @@ function search {
export searchstring=$(echo $i | sed 's/ /+/g') export searchstring=$(echo $i | sed 's/ /+/g')
export file="$downloaddir"/$searchstring'.txt' export file="$downloaddir"/$searchstring'.txt'
echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file" echo | googler --count=5 -C -x "$searchstring" | grep -v 'Enter n, p, result' > "$file"
echo 'Ran search for '$searchstring' to '$file | tee -a $logfile echo "Ran search for $searchstring to $file" | tee -a $logfile
done done
} }
function stoptorrent {
# TODO This is destructive to all other transmission-CLI clients on the box.
killall transmission-cli
}
if [ "$1" == "" ]; then if [ "$1" == "" ]; then
helptext; helptext;
@ -246,10 +208,6 @@ case "$1" in
esac esac
;; ;;
"--stoptorrent")
stoptorrent
;;
*) *)
helptext helptext
;; ;;