From 6e4fddb509f73bc3042499185f0f21f031356ce7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mon, 8 Dec 2014 18:00:45 +0000 Subject: [PATCH] Import the files for the torrent role from puppet --- roles/torrent/files/bittorrent.logrotate | 18 ++ roles/torrent/files/bittorrent.sysconfig | 8 + roles/torrent/files/do-torrent-stats.sh | 51 +++++ roles/torrent/files/opentracker-ipv4.conf | 100 +++++++++ roles/torrent/files/opentracker-ipv6.conf | 98 +++++++++ roles/torrent/files/pull_opentracker_data.sh | 38 ++++ roles/torrent/files/spins-generator/foot.html | 31 +++ roles/torrent/files/spins-generator/foot.rss | 3 + roles/torrent/files/spins-generator/head.html | 59 ++++++ roles/torrent/files/spins-generator/head.rss | 11 + .../files/spins-generator/torrent-generator | 200 ++++++++++++++++++ roles/torrent/files/spins-httpd.conf | 21 ++ roles/torrent/files/spins_generator.conf | 10 + roles/torrent/files/torrent-data.py | 86 ++++++++ .../torrent/files/torrent-generator/foot.html | 31 +++ .../torrent/files/torrent-generator/foot.rss | 3 + .../torrent/files/torrent-generator/head.html | 58 +++++ .../torrent/files/torrent-generator/head.rss | 11 + .../files/torrent-generator/torrent-generator | 190 +++++++++++++++++ roles/torrent/files/torrent-hash.cron | 5 + roles/torrent/files/torrent-hashes.py | 50 +++++ roles/torrent/files/torrent-httpd.conf | 54 +++++ roles/torrent/files/torrent-web-generate.cron | 3 + roles/torrent/files/torrent_generator.conf | 10 + roles/torrent/files/torrentjsonstats.py | 37 ++++ 25 files changed, 1186 insertions(+) create mode 100644 roles/torrent/files/bittorrent.logrotate create mode 100644 roles/torrent/files/bittorrent.sysconfig create mode 100644 roles/torrent/files/do-torrent-stats.sh create mode 100644 roles/torrent/files/opentracker-ipv4.conf create mode 100644 roles/torrent/files/opentracker-ipv6.conf create mode 100755 roles/torrent/files/pull_opentracker_data.sh create mode 100644 roles/torrent/files/spins-generator/foot.html create mode 100644 roles/torrent/files/spins-generator/foot.rss create mode 100644 roles/torrent/files/spins-generator/head.html create mode 100644 roles/torrent/files/spins-generator/head.rss create mode 100755 roles/torrent/files/spins-generator/torrent-generator create mode 100644 roles/torrent/files/spins-httpd.conf create mode 100644 roles/torrent/files/spins_generator.conf create mode 100755 roles/torrent/files/torrent-data.py create mode 100644 roles/torrent/files/torrent-generator/foot.html create mode 100644 roles/torrent/files/torrent-generator/foot.rss create mode 100644 roles/torrent/files/torrent-generator/head.html create mode 100644 roles/torrent/files/torrent-generator/head.rss create mode 100755 roles/torrent/files/torrent-generator/torrent-generator create mode 100644 roles/torrent/files/torrent-hash.cron create mode 100755 roles/torrent/files/torrent-hashes.py create mode 100644 roles/torrent/files/torrent-httpd.conf create mode 100644 roles/torrent/files/torrent-web-generate.cron create mode 100644 roles/torrent/files/torrent_generator.conf create mode 100755 roles/torrent/files/torrentjsonstats.py diff --git a/roles/torrent/files/bittorrent.logrotate b/roles/torrent/files/bittorrent.logrotate new file mode 100644 index 0000000000..83361d2fad --- /dev/null +++ b/roles/torrent/files/bittorrent.logrotate @@ -0,0 +1,18 @@ +/var/log/bittorrent/btseed.log { + notifempty + missingok + compress + postrotate + /sbin/service btseed condrestart 2>/dev/null >/dev/null || : + endscript +} + +/var/log/bittorrent/bttrack.log { + notifempty + missingok + compress + postrotate + /sbin/service bttrack condrestart 2>/dev/null >/dev/null || : + endscript +} + diff --git a/roles/torrent/files/bittorrent.sysconfig b/roles/torrent/files/bittorrent.sysconfig new file mode 100644 index 0000000000..d6dbaa20a7 --- /dev/null +++ b/roles/torrent/files/bittorrent.sysconfig @@ -0,0 +1,8 @@ +SEEDDIR=/srv/torrent/btholding +SEEDOPTS="--max_upload_rate 350 --display_interval 300" +SEEDLOG=/var/log/bittorrent/btseed.log +TRACKPORT=6969 +TRACKDIR=/srv/torrent/btholding +TRACKSTATEFILE=/srv/torrent/data/bttrack.dat +TRACKLOG=/var/log/bittorrent/bttrack.log +TRACKOPTS="--min_time_between_log_flushes 4.0 --show_names 1 --hupmonitor 1" diff --git a/roles/torrent/files/do-torrent-stats.sh b/roles/torrent/files/do-torrent-stats.sh new file mode 100644 index 0000000000..1e919e75e7 --- /dev/null +++ b/roles/torrent/files/do-torrent-stats.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +statscmd=/usr/local/bin/torrent-data.py +btdata=/srv/torrent/data/bttrack.dat +outputdir=/srv/torrent/www/stats/ +hourlydir=$outputdir/hourly +dailydir=$outputdir/daily +latestlink=$outputdir/current-stats.json + +if ! [ -f $btdata ]; then + echo "cannot find $btdata" + exit 1 +fi + +if ! [ -d $outputdir ]; then + mkdir -p $outputdir +fi + +if ! [ -d $hourlydir ]; then + mkdir -p $hourlydir +fi + +if ! [ -d $dailydir ]; then + mkdir -p $dailydir +fi + + +if ! [ -f $statscmd ]; then + echo "Cannot find stats generating command $statscmd" + exit 1 +fi + +now=`date +%Y-%m-%d-%H:%M` +today=`date +%Y-%m-%d` +outputfile="$hourlydir/torrent-stats-$now.json" +dataloc=`mktemp` + +\cp -f $btdata $dataloc + +$statscmd $dataloc $outputfile +rm -f $dataloc + +rm -f $latestlink +ln -s $outputfile $latestlink + +if [ ! -f "$dailydir/torrent-stats-$today.json" ]; then + \cp -f $outputfile $dailydir/torrent-stats-$today.json +fi + +/usr/sbin/tmpwatch -f 720 $hourlydir + diff --git a/roles/torrent/files/opentracker-ipv4.conf b/roles/torrent/files/opentracker-ipv4.conf new file mode 100644 index 0000000000..4fce278be9 --- /dev/null +++ b/roles/torrent/files/opentracker-ipv4.conf @@ -0,0 +1,100 @@ +# opentracker config file +# + +# I) Address opentracker will listen on, using both, tcp AND udp family +# (note, that port 6969 is implicite if ommitted). +# +# If no listen option is given (here or on the command line), opentracker +# listens on 0.0.0.0:6969 tcp and udp. +# +# listen.tcp_udp 0.0.0.0 +# listen.tcp_udp 192.168.0.1:80 +# listen.tcp_udp 10.0.0.5:6969 +# +# To only listen on tcp or udp family ports, list them this way: +# +# listen.tcp 0.0.0.0 +# listen.udp 192.168.0.1:6969 +# +# Note, that using 0.0.0.0 for udp sockets may yield surprising results. +# An answer packet sent on that socket will not necessarily have the +# source address that the requesting client may expect, but any address +# on that interface. +# + +# II) If opentracker runs in a non-open mode, point it to files containing +# all torrent hashes that it will serve (shell option -w) +# The path must be relative to the chroot directory (tracker.rootdir)! +# +access.whitelist ./whitelist +# +# or, if opentracker was compiled to allow blacklisting (shell option -b) +# +# access.blacklist ./blacklist +# +# It is pointless and hence not possible to compile black AND white +# listing, so choose one of those options at compile time. File format +# is straight forward: "\n\n..." +# +# If you do not want to grant anyone access to your stats, enable the +# WANT_RESTRICT_STATS option in Makefile and bless the ip addresses +# allowed to fetch stats here. +# +access.stats 98.122.179.19 +access.stats 127.0.0.1 +access.stats 152.19.134.148 +# +# There is another way of hiding your stats. You can obfuscate the path +# to them. Normally it is located at /stats but you can configure it to +# appear anywhere on your tracker. +# +# access.stats_path stats + +# III) Live sync uses udp multicast packets to keep a cluster of opentrackers +# synchronized. This option tells opentracker which port to listen for +# incoming live sync packets. The ip address tells opentracker, on which +# interface to join the multicast group, those packets will arrive. +# (shell option -i 192.168.0.1 -s 9696), port 9696 is default. +# +# livesync.cluster.listen 192.168.0.1:9696 +# +# Note that two udp sockets will be opened. One on ip address 0.0.0.0 +# port 9696, that will join the multicast group 224.0.42.23 for incoming +# udp packets and one on ip address 192.168.0.1 port 9696 for outgoing +# udp packets. +# +# As of now one and only one ip address must be given, if opentracker +# was built with the WANT_SYNC_LIVE feature. +# + +# IV) Sync between trackers running in a cluster is restricted to packets +# coming from trusted ip addresses. While source ip verification is far +# from perfect, the authors of opentracker trust in the correct +# application of tunnels, filters and LAN setups (shell option -A). +# +# livesync.cluster.node_ip 192.168.0.4 +# livesync.cluster.node_ip 192.168.0.5 +# livesync.cluster.node_ip 192.168.0.6 +# +# This is the admin ip address for old style (HTTP based) asynchronus +# tracker syncing. +# +# batchsync.cluster.admin_ip 10.1.1.1 +# + +# V) Control privilege drop behaviour. +# Put in the directory opentracker will chroot/chdir to. All black/white +# list files must be put in that directory (shell option -d). +# +# +tracker.rootdir /srv/torrent/btholding/ +# +# Tell opentracker which user to setuid to. +# +tracker.user opentracker +# + +# VI) opentracker can be told to answer to a "GET / HTTP"-request with a +# redirect to another location (shell option -r). +# +# tracker.redirect_url https://your.tracker.local/ diff --git a/roles/torrent/files/opentracker-ipv6.conf b/roles/torrent/files/opentracker-ipv6.conf new file mode 100644 index 0000000000..8008e50e81 --- /dev/null +++ b/roles/torrent/files/opentracker-ipv6.conf @@ -0,0 +1,98 @@ +# opentracker config file +# + +# I) Address opentracker will listen on, using both, tcp AND udp family +# (note, that port 6969 is implicite if ommitted). +# +# If no listen option is given (here or on the command line), opentracker +# listens on 0.0.0.0:6969 tcp and udp. +# +listen.tcp_udp [2610:28:3090:3001:dead:beef:cafe:fed7]:6969 +# listen.tcp_udp 192.168.0.1:80 +# listen.tcp_udp 10.0.0.5:6969 +# +# To only listen on tcp or udp family ports, list them this way: +# +# listen.tcp 0.0.0.0 +# listen.udp 192.168.0.1:6969 +# +# Note, that using 0.0.0.0 for udp sockets may yield surprising results. +# An answer packet sent on that socket will not necessarily have the +# source address that the requesting client may expect, but any address +# on that interface. +# + +# II) If opentracker runs in a non-open mode, point it to files containing +# all torrent hashes that it will serve (shell option -w) +# The path must be relative to the chroot directory (tracker.rootdir)! +# +access.whitelist ./whitelist +# +# or, if opentracker was compiled to allow blacklisting (shell option -b) +# +# access.blacklist ./blacklist +# +# It is pointless and hence not possible to compile black AND white +# listing, so choose one of those options at compile time. File format +# is straight forward: "\n\n..." +# +# If you do not want to grant anyone access to your stats, enable the +# WANT_RESTRICT_STATS option in Makefile and bless the ip addresses +# allowed to fetch stats here. +# +access.stats ::1 +# +# There is another way of hiding your stats. You can obfuscate the path +# to them. Normally it is located at /stats but you can configure it to +# appear anywhere on your tracker. +# +# access.stats_path stats + +# III) Live sync uses udp multicast packets to keep a cluster of opentrackers +# synchronized. This option tells opentracker which port to listen for +# incoming live sync packets. The ip address tells opentracker, on which +# interface to join the multicast group, those packets will arrive. +# (shell option -i 192.168.0.1 -s 9696), port 9696 is default. +# +# livesync.cluster.listen 192.168.0.1:9696 +# +# Note that two udp sockets will be opened. One on ip address 0.0.0.0 +# port 9696, that will join the multicast group 224.0.42.23 for incoming +# udp packets and one on ip address 192.168.0.1 port 9696 for outgoing +# udp packets. +# +# As of now one and only one ip address must be given, if opentracker +# was built with the WANT_SYNC_LIVE feature. +# + +# IV) Sync between trackers running in a cluster is restricted to packets +# coming from trusted ip addresses. While source ip verification is far +# from perfect, the authors of opentracker trust in the correct +# application of tunnels, filters and LAN setups (shell option -A). +# +# livesync.cluster.node_ip 192.168.0.4 +# livesync.cluster.node_ip 192.168.0.5 +# livesync.cluster.node_ip 192.168.0.6 +# +# This is the admin ip address for old style (HTTP based) asynchronus +# tracker syncing. +# +# batchsync.cluster.admin_ip 10.1.1.1 +# + +# V) Control privilege drop behaviour. +# Put in the directory opentracker will chroot/chdir to. All black/white +# list files must be put in that directory (shell option -d). +# +# +tracker.rootdir /srv/torrent/btholding/ +# +# Tell opentracker which user to setuid to. +# +tracker.user opentracker +# + +# VI) opentracker can be told to answer to a "GET / HTTP"-request with a +# redirect to another location (shell option -r). +# +# tracker.redirect_url https://your.tracker.local/ diff --git a/roles/torrent/files/pull_opentracker_data.sh b/roles/torrent/files/pull_opentracker_data.sh new file mode 100755 index 0000000000..924a858fd1 --- /dev/null +++ b/roles/torrent/files/pull_opentracker_data.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# fetch data files from the opentracker stats output +# store them away for later analysis +# skvidal +base=/srv/torrent/www/stats/raw +baseurl='http://torrent.fedoraproject.org:6969/stats?mode=' +date=`date +%Y-%m-%d` +hour=`date +%H` +wgetcmd="wget --timeout=20 -q" +#per-torrent stats +src="${baseurl}tpbs&format=txt" +dest=$base/torrent/$date +if [ ! -d $dest ]; then + mkdir -p $dest +fi +$wgetcmd -O $dest/$hour.txt $src +ln -sf $dest/$hour.txt $base/torrent/current.txt + + +#totalstats +src="${baseurl}everything" +dest=$base/everything/$date +if [ ! -d $dest ]; then + mkdir -p $dest +fi +$wgetcmd -O $dest/$hour.xml $src +ln -sf $dest/$hour.xml $base/everything/current.xml + + +# generate the json files +hourdate=`date +%Y-%m-%d-%H-%M` +/usr/local/bin/torrentjsonstats.py $base/torrent/current.txt > /srv/torrent/www/stats/hourly/torrent-stats-$hourdate.json +ln -sf /srv/torrent/www/stats/hourly/torrent-stats-$hourdate.json /srv/torrent/www/stats/current-stats.json + +if [ $hour == '12' ]; then + /usr/local/bin/torrentjsonstats.py $base/torrent/current.txt > /srv/torrent/www/stats/daily/torrent-stats-$date.json +fi + diff --git a/roles/torrent/files/spins-generator/foot.html b/roles/torrent/files/spins-generator/foot.html new file mode 100644 index 0000000000..f83f38dd5c --- /dev/null +++ b/roles/torrent/files/spins-generator/foot.html @@ -0,0 +1,31 @@ + +

+ See http://fedoraproject.org/wiki/Distribution/Download/BitTorrent for a guide to using BitTorrrent. +

+ + + +
+ +
+ + + diff --git a/roles/torrent/files/spins-generator/foot.rss b/roles/torrent/files/spins-generator/foot.rss new file mode 100644 index 0000000000..2a31cb0efa --- /dev/null +++ b/roles/torrent/files/spins-generator/foot.rss @@ -0,0 +1,3 @@ + + + diff --git a/roles/torrent/files/spins-generator/head.html b/roles/torrent/files/spins-generator/head.html new file mode 100644 index 0000000000..43228d3654 --- /dev/null +++ b/roles/torrent/files/spins-generator/head.html @@ -0,0 +1,59 @@ + + + + Spins for the Fedora Project + + + + + + +
+ +
+

Fedora Project Spins Tracker

+ + + + + + + + diff --git a/roles/torrent/files/spins-generator/head.rss b/roles/torrent/files/spins-generator/head.rss new file mode 100644 index 0000000000..3a152a5e1c --- /dev/null +++ b/roles/torrent/files/spins-generator/head.rss @@ -0,0 +1,11 @@ + + + + +Fedora Torrent +http://torrent.fedoraproject.org/ +RSS feed for Fedora +en-us +admin at fedoraproject.org +60 + diff --git a/roles/torrent/files/spins-generator/torrent-generator b/roles/torrent/files/spins-generator/torrent-generator new file mode 100755 index 0000000000..fdff3b75ae --- /dev/null +++ b/roles/torrent/files/spins-generator/torrent-generator @@ -0,0 +1,200 @@ +#!/usr/bin/python -tt +# generate html and rss output files based on ini of torrents. +# (c) 2007 Seth Vidal - skvidal @ fedoraproject.org + +# read in .ini files in a dir +# take each section from them and assemble an html file and an rss feed of the +# data contained w/i + +# ini files should be: +# [.torrent file path] +# description="my description of the torrent" +# group="Name of group it belongs to" +# releasedate=2007-10-06 +# size=629M +# map=http://url/to/map.png +# group is optional. if not listed group == description + +import ConfigParser +import sys +import glob +import time + +timeformat = "%Y-%m-%d" +rssformat = "%a, %d %b %Y %H:%M:%S" +globconf = '/etc/spins_generator.conf' +if len(sys.argv) > 1: + globconf = sys.argv[1] +groups = {} + +mapicon = 'http://mmcgrath.fedorapeople.org/map.png' +nomaptext = ' ' + +def outputtime(etime): + return time.strftime(timeformat, time.localtime(etime)) + +def rsstime(etime): + return time.strftime(rssformat, time.localtime(etime)) + +def do_html_output(config, groups): + myout = open(config.htmlout, 'w') + head = open('%s' % config.htmlheader).read() + foot = open('%s' % config.htmlfooter).read() + + myout.write(head) + + for group in groups: + msg = """ + + + + """ % group.name + myout.write(msg) + + t = group.torrents + t.sort() + t.reverse() + for torrent in t: + msg = """ + + + + + + + + """ % (config.torrent_url, torrent.torrent, torrent.torrent, torrent.description, torrent.size, torrent.map, outputtime(torrent.releasedate)) + myout.write(msg) + + myout.write(foot) + myout.close() + +def do_rss_output(config, groups): + + myout = open(config.rssout, 'w') + head = open('%s' % config.rssheader).read() + foot = open('%s' % config.rssfooter).read() + + myout.write(head) + + for group in groups: + for torrent in group.torrents: + msg = """ + %s + %s/%s + %s + """ % (torrent.description, config.torrent_url, torrent.torrent, rsstime(torrent.releasedate)) + myout.write(msg) + myout.write(foot) + myout.close() + + +class Group(object): + def __init__(self, name, date): + self.name = name + self.releasedate = date + self.torrents = [] + + def __cmp__(self, other): + if other.releasedate > self.releasedate: + return -1 + if self.releasedate > other.releasedate: + return 1 + + return 0 + + +class Torrent(object): + def __init__(self): + self.torrent = None + self.group = None + self.releasedate = '1969-01-01' + self.size = None + self.description = None + self.map = None + + def __cmp__(self, other): + if other.releasedate > self.releasedate: + return -1 + if self.releasedate > other.releasedate: + return 1 + + return 0 + + +class Config(object): + def __init__(self, cpobj): + if not cpobj.has_section('main'): + print >> sys.stderr, "no main section in config, exiting" + sys.exit(1) + try: + self.inidir = cpobj.get('main', 'inidir') + self.torrent_url = cpobj.get('main', 'torrent_url') + self.htmlheader = cpobj.get('main', 'htmlheader') + self.htmlfooter = cpobj.get('main', 'htmlfooter') + self.rssheader = cpobj.get('main', 'rssheader') + self.rssfooter = cpobj.get('main', 'rssfooter') + self.htmlout = cpobj.get('main', 'htmlout') + self.rssout = cpobj.get('main', 'rssout') + except ConfigParser.NoOptionError, e: + print >> sys.stderr, "Config file missing required option: %s" % e + sys.exit(1) + +def main(): + + conf = ConfigParser.ConfigParser() + conf.read(globconf) + config = Config(conf) + + fs = glob.glob(config.inidir + '/*.ini') + for fn in fs: + c = ConfigParser.ConfigParser() + c.read(fn) + for s in c.sections(): + if 'releasedate' not in c.options(s) or 'description' not in c.options(s): + print >> sys.stderr, "bad torrent config for %s" % s + continue + if c.has_option(s,'group'): + g = c.get(s, 'group') + else: + g = c.get(s, 'description') + + thisdate = time.mktime(time.strptime(c.get(s, 'releasedate'), timeformat)) + if groups.has_key(g): + thisgroup = groups[g] + if thisgroup.releasedate < thisdate: + thisgroup.releasedate = thisdate + else: + thisgroup = Group(g, thisdate) + groups[thisgroup.name] = thisgroup + + this = Torrent() + this.torrent = s + this.group = g + this.releasedate = thisdate + if c.has_option(s, 'size'): + this.size = c.get(s, 'size') + else: + this.size = 'Unknown' + + this.description = c.get(s, 'description') + this.map = nomaptext + if c.has_option(s, 'map'): + maploc = c.get(s, 'map') + this.map = 'map' % (maploc, mapicon) + else: + this.map = nomaptext + + thisgroup.torrents.append(this) + + sortgroups = groups.values() + sortgroups.sort() + sortgroups.reverse() + + do_html_output(config, sortgroups) + do_rss_output(config, sortgroups) + + +if __name__ == "__main__": + main() + diff --git a/roles/torrent/files/spins-httpd.conf b/roles/torrent/files/spins-httpd.conf new file mode 100644 index 0000000000..f86bb0c078 --- /dev/null +++ b/roles/torrent/files/spins-httpd.conf @@ -0,0 +1,21 @@ + + ## + # Domain: torrent.fedoraproject.org + # Owner: admin@fedoraproject.org + # + ServerAdmin admin@fedoraproject.org + + DocumentRoot "/srv/torrent/spins" + + ServerName spins.fedoraproject.org + + ErrorLog "| /usr/sbin/rotatelogs /var/log/httpd/spins.fedoraproject.org-error.log-%Y%m%d 86400 -300" + CustomLog "| /usr/sbin/rotatelogs /var/log/httpd/spins.fedoraproject.org-access.log-%Y%m%d 86400 -300" common + + RewriteEngine On + RewriteRule /favicon\.ico$ http://fedoraproject.org/static/images/favicon.ico [P] + + + Options Indexes FollowSymLinks + + diff --git a/roles/torrent/files/spins_generator.conf b/roles/torrent/files/spins_generator.conf new file mode 100644 index 0000000000..01afeaec87 --- /dev/null +++ b/roles/torrent/files/spins_generator.conf @@ -0,0 +1,10 @@ +[main] +inidir=/srv/torrent/spins-generator/ +torrent_url=http://spins.fedoraproject.org/torrents +htmlheader=/srv/torrent/spins-generator/head.html +htmlfooter=/srv/torrent/spins-generator/foot.html +rssheader=/srv/torrent/spins-generator/head.rss +rssfooter=/srv/torrent/spins-generator/foot.rss +htmlout=/srv/torrent/spins/index.html +rssout=/srv/torrent/spins/rss20.xml + diff --git a/roles/torrent/files/torrent-data.py b/roles/torrent/files/torrent-data.py new file mode 100755 index 0000000000..f6bce7707e --- /dev/null +++ b/roles/torrent/files/torrent-data.py @@ -0,0 +1,86 @@ +#!/usr/bin/python -tt + +import os +import sys +import BitTorrent +import BitTorrent.bencode +import simplejson + +# grab bttrack.dat +# dump out, per torrent stats: +# total completed downloads +# active number of downloaders +# active number of seeds + +class TorrentObj(object): + def __init__(self, fn, encode_name, name): + self.fn = fn + self.encode_name = encode_name + self.name = name + self.downloaders = 0 + self.seeds = 0 + self.size = 0L + self.completed = 0 + + def __cmp__(self, other): + return cmp(self.name, other.name) + + def json_obj(self): + obj = {'size':self.size, + 'completed':self.completed, + 'seeds':self.seeds, + 'downloaders':self.downloaders, + 'name':self.name} + return obj + + +def main(args): + bttrack = args[0] + outputfile = args[1] + + if not os.path.exists(bttrack): + print "Data file %s does not exist" % bttrack + sys.exit(1) + + bt = BitTorrent.bencode.bdecode(open(bttrack, 'rb').read()) + torrents = [] + + for encode_name in bt['allowed'].keys(): + tor = bt['allowed'][encode_name] + torobj = TorrentObj(tor['file'], encode_name, tor['name']) + torobj.size = tor['length'] + + if bt['completed'].has_key(torobj.encode_name): + torobj.completed = int(bt['completed'][torobj.encode_name]) + + if bt['peers'].has_key(torobj.encode_name): + peers = bt['peers'][torobj.encode_name] + for peer_dict in peers.values(): + if peer_dict['left'] == 0: + torobj.seeds += 1 + else: + torobj.downloaders += 1 + + torrents.append(torobj) + + # write out to outputfile + # torrent-name: size, finished, downloaders, seeds + + of = open(outputfile, 'w') + #of.write('#torrentname:size:completed:downloaders:seeders\n') + json_torrents = [ tor.json_obj() for tor in torrents ] + #for tor in sorted(torrents): + #of.write('%s:%s:%s:%s:%s\n' % (tor.name, tor.size, tor.completed, + # tor.downloaders, tor.seeds)) + simplejson.dump(json_torrents, of) + + + of.close() + +if __name__ == '__main__': + if len(sys.argv) < 3: + print "usage: torrent-data.py bttrack.dat output_filename" + sys.exit(1) + main(sys.argv[1:]) + + diff --git a/roles/torrent/files/torrent-generator/foot.html b/roles/torrent/files/torrent-generator/foot.html new file mode 100644 index 0000000000..a222345502 --- /dev/null +++ b/roles/torrent/files/torrent-generator/foot.html @@ -0,0 +1,31 @@ +
TorrentDescriptionSizeMapDate
%s
%s%s%s%s%s
+

+ See http://fedoraproject.org/wiki/Distribution/Download/BitTorrent for a guide to using BitTorrrent. +

+ +
+
+
+ +
+ + + diff --git a/roles/torrent/files/torrent-generator/foot.rss b/roles/torrent/files/torrent-generator/foot.rss new file mode 100644 index 0000000000..2a31cb0efa --- /dev/null +++ b/roles/torrent/files/torrent-generator/foot.rss @@ -0,0 +1,3 @@ + + + diff --git a/roles/torrent/files/torrent-generator/head.html b/roles/torrent/files/torrent-generator/head.html new file mode 100644 index 0000000000..31efca5a67 --- /dev/null +++ b/roles/torrent/files/torrent-generator/head.html @@ -0,0 +1,58 @@ + + + + Torrent Server for the Fedora Project + + + + + + +
+ +
+

Fedora Project Bittorrent Tracker

+ + + + + + + + diff --git a/roles/torrent/files/torrent-generator/head.rss b/roles/torrent/files/torrent-generator/head.rss new file mode 100644 index 0000000000..3a152a5e1c --- /dev/null +++ b/roles/torrent/files/torrent-generator/head.rss @@ -0,0 +1,11 @@ + + + + +Fedora Torrent +http://torrent.fedoraproject.org/ +RSS feed for Fedora +en-us +admin at fedoraproject.org +60 + diff --git a/roles/torrent/files/torrent-generator/torrent-generator b/roles/torrent/files/torrent-generator/torrent-generator new file mode 100755 index 0000000000..0ac65e7b6f --- /dev/null +++ b/roles/torrent/files/torrent-generator/torrent-generator @@ -0,0 +1,190 @@ +#!/usr/bin/python -tt +# generate html and rss output files based on ini of torrents. +# (c) 2007 Seth Vidal - skvidal @ fedoraproject.org + +# read in .ini files in a dir +# take each section from them and assemble an html file and an rss feed of the +# data contained w/i + +# ini files should be: +# [.torrent file path] +# description="my description of the torrent" +# group="Name of group it belongs to" +# releasedate=2007-10-06 +# size=629M +# map=http://url/to/map.png +# group is optional. if not listed group == description + +import ConfigParser +import sys +import glob +import time +import operator + +timeformat = "%Y-%m-%d" +rssformat = "%a, %d %b %Y %H:%M:%S" +globconf = '/etc/torrent_generator.conf' +if len(sys.argv) > 1: + globconf = sys.argv[1] +groups = {} + +mapicon = 'http://mmcgrath.fedorapeople.org/map.png' +nomaptext = ' ' + +def outputtime(etime): + return time.strftime(timeformat, time.localtime(etime)) + +def rsstime(etime): + return time.strftime(rssformat, time.localtime(etime)) + +def do_html_output(config, groups): + myout = open(config.htmlout, 'w') + head = open('%s' % config.htmlheader).read() + foot = open('%s' % config.htmlfooter).read() + + myout.write(head) + + for group in groups: + msg = """ + + + + """ % group.name + myout.write(msg) + + for torrent in sorted(group.torrents, key=operator.attrgetter('releasedate', 'torrent')): + msg = """ + + + + + + + + """ % (config.torrent_url, torrent.torrent, torrent.torrent, torrent.description, torrent.size, torrent.map, outputtime(torrent.releasedate)) + myout.write(msg) + + myout.write(foot) + myout.close() + +def do_rss_output(config, groups): + + myout = open(config.rssout, 'w') + head = open('%s' % config.rssheader).read() + foot = open('%s' % config.rssfooter).read() + + myout.write(head) + + for group in groups: + for torrent in sorted(group.torrents, key=operator.attrgetter('releasedate', 'torrent')): + msg = """ + %s + %s/%s + %s + """ % (torrent.description, config.torrent_url, torrent.torrent, rsstime(torrent.releasedate)) + myout.write(msg) + myout.write(foot) + myout.close() + + +class Group(object): + def __init__(self, name, date): + self.name = name + self.releasedate = date + self.torrents = [] + + def __cmp__(self, other): + if other.releasedate > self.releasedate: + return -1 + if self.releasedate > other.releasedate: + return 1 + + return 0 + + +class Torrent(object): + def __init__(self): + self.torrent = None + self.group = None + self.releasedate = '1969-01-01' + self.size = None + self.description = None + self.map = None + + +class Config(object): + def __init__(self, cpobj): + if not cpobj.has_section('main'): + print >> sys.stderr, "no main section in config, exiting" + sys.exit(1) + try: + self.inidir = cpobj.get('main', 'inidir') + self.torrent_url = cpobj.get('main', 'torrent_url') + self.htmlheader = cpobj.get('main', 'htmlheader') + self.htmlfooter = cpobj.get('main', 'htmlfooter') + self.rssheader = cpobj.get('main', 'rssheader') + self.rssfooter = cpobj.get('main', 'rssfooter') + self.htmlout = cpobj.get('main', 'htmlout') + self.rssout = cpobj.get('main', 'rssout') + except ConfigParser.NoOptionError, e: + print >> sys.stderr, "Config file missing required option: %s" % e + sys.exit(1) + +def main(): + + conf = ConfigParser.ConfigParser() + conf.read(globconf) + config = Config(conf) + + fs = glob.glob(config.inidir + '/*.ini') + for fn in fs: + c = ConfigParser.ConfigParser() + c.read(fn) + for s in c.sections(): + if 'releasedate' not in c.options(s) or 'description' not in c.options(s): + print >> sys.stderr, "bad torrent config for %s" % s + continue + if c.has_option(s,'group'): + g = c.get(s, 'group') + else: + g = c.get(s, 'description') + + thisdate = time.mktime(time.strptime(c.get(s, 'releasedate'), timeformat)) + if groups.has_key(g): + thisgroup = groups[g] + if thisgroup.releasedate < thisdate: + thisgroup.releasedate = thisdate + else: + thisgroup = Group(g, thisdate) + groups[thisgroup.name] = thisgroup + + this = Torrent() + this.torrent = s + this.group = g + this.releasedate = thisdate + if c.has_option(s, 'size'): + this.size = c.get(s, 'size') + else: + this.size = 'Unknown' + + this.description = c.get(s, 'description') + this.map = nomaptext + if c.has_option(s, 'map'): + maploc = c.get(s, 'map') + this.map = 'map' % (maploc, mapicon) + else: + this.map = nomaptext + + thisgroup.torrents.append(this) + + sortgroups = groups.values() + sortgroups.sort() + sortgroups.reverse() + + do_html_output(config, sortgroups) + do_rss_output(config, sortgroups) + + +if __name__ == "__main__": + main() + diff --git a/roles/torrent/files/torrent-hash.cron b/roles/torrent/files/torrent-hash.cron new file mode 100644 index 0000000000..d7a10254b0 --- /dev/null +++ b/roles/torrent/files/torrent-hash.cron @@ -0,0 +1,5 @@ +*/20 * * * * torrent /usr/local/bin/torrent-hashes.py -o /srv/torrent/btholding/whitelist /srv/torrent/btholding/*.torrent +2 * * * * opentracker /usr/local/bin/pull_opentracker_data.sh + + + diff --git a/roles/torrent/files/torrent-hashes.py b/roles/torrent/files/torrent-hashes.py new file mode 100755 index 0000000000..69ad1674f1 --- /dev/null +++ b/roles/torrent/files/torrent-hashes.py @@ -0,0 +1,50 @@ +#!/usr/bin/python +# by Matt Domsch +# License: BitTorrent +# +# This simply prints the bittorrent hashes for each file +# to stdout or an output file. +# To be used as the whitelist with opentracker + +import os +import sys +import hashlib +from optparse import OptionParser +from BitTorrent.bencode import bencode, bdecode +from BitTorrent.btformats import check_message + +def torrent_hash(fname): + f = open(fname, 'rb') + d = bdecode(f.read()) + f.close() + check_message(d) + hash = hashlib.sha1(bencode(d['info'])).hexdigest().upper() + fn = os.path.basename(fname) + return '%s - %s' % (hash,fn) + +def main(): + parser = OptionParser(usage=sys.argv[0] + " [options] [torrentfiles] ...") + parser.add_option("-o", "--output", type="string", metavar="FILE", + dest="output", default='-', + help="write hashes to FILE, default=stdout") + (options, args) = parser.parse_args() + + outfd = sys.stdout + if options.output != '-': + try: + outfd = open(options.output, 'w') + except: + sys.stderr.write("Error: unable to open output file %s\n" % options.output) + return 1 + + for a in args: + try: + hash = torrent_hash(a) + outfd.write(hash + '\n') + except: + sys.stderr.write("Error reading hash from %s\n" % a) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/roles/torrent/files/torrent-httpd.conf b/roles/torrent/files/torrent-httpd.conf new file mode 100644 index 0000000000..aa531a6c16 --- /dev/null +++ b/roles/torrent/files/torrent-httpd.conf @@ -0,0 +1,54 @@ +NameVirtualHost *:80 + +LoadModule deflate_module modules/mod_deflate.so + + + # Insert filter + SetOutputFilter DEFLATE + + # Netscape 4.x has some problems... + BrowserMatch ^Mozilla/4 gzip-only-text/html + + # Netscape 4.06-4.08 have some more problems + BrowserMatch ^Mozilla/4\.0[678] no-gzip + + # MSIE masquerades as Netscape, but it is fine + # BrowserMatch \bMSIE !no-gzip !gzip-only-text/html + + # NOTE: Due to a bug in mod_setenvif up to Apache 2.0.48 + # the above regex won't work. You can use the following + # workaround to get the desired effect: + BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html + + # Don't compress images + SetEnvIfNoCase Request_URI \ + \.(?:gif|jpe?g|png)$ no-gzip dont-vary + + # Make sure proxies don't deliver the wrong content + Header append Vary User-Agent env=!dont-vary + + +FileETag MTime Size + + + ## + # Domain: torrent.fedoraproject.org + # Owner: admin@fedoraproject.org + # + ServerAdmin admin@fedoraproject.org + + DocumentRoot "/srv/torrent/www" + + ServerName torrent.fedoraproject.org + ServerAlias torrents.fedoraproject.org + + ErrorLog "| /usr/sbin/rotatelogs /var/log/httpd/torrent.fedoraproject.org-error.log-%Y%m%d 86400 -300" + CustomLog "| /usr/sbin/rotatelogs /var/log/httpd/torrent.fedoraproject.org-access.log-%Y%m%d 86400 -300" common + + RewriteEngine On + RewriteRule /favicon\.ico$ http://fedoraproject.org/static/images/favicon.ico [P] + + + Options Indexes FollowSymLinks + + diff --git a/roles/torrent/files/torrent-web-generate.cron b/roles/torrent/files/torrent-web-generate.cron new file mode 100644 index 0000000000..181c117e3e --- /dev/null +++ b/roles/torrent/files/torrent-web-generate.cron @@ -0,0 +1,3 @@ +*/20 * * * * torrent /srv/torrent/torrent-generator/torrent-generator +*/20 * * * * torrent /srv/torrent/spins-generator/torrent-generator + diff --git a/roles/torrent/files/torrent_generator.conf b/roles/torrent/files/torrent_generator.conf new file mode 100644 index 0000000000..035478ffbf --- /dev/null +++ b/roles/torrent/files/torrent_generator.conf @@ -0,0 +1,10 @@ +[main] +inidir=/srv/torrent/torrent-generator/ +torrent_url=http://torrent.fedoraproject.org/torrents +htmlheader=/srv/torrent/torrent-generator/head.html +htmlfooter=/srv/torrent/torrent-generator/foot.html +rssheader=/srv/torrent/torrent-generator/head.rss +rssfooter=/srv/torrent/torrent-generator/foot.rss +htmlout=/srv/torrent/www/index.html +rssout=/srv/torrent/www/rss20.xml + diff --git a/roles/torrent/files/torrentjsonstats.py b/roles/torrent/files/torrentjsonstats.py new file mode 100755 index 0000000000..f8da8b4921 --- /dev/null +++ b/roles/torrent/files/torrentjsonstats.py @@ -0,0 +1,37 @@ +#!/usr/bin/python -tt + +import sys +import simplejson + + +whitelist='/srv/torrent/btholding/whitelist' + + +if len(sys.argv) < 2: + print "Usage: torrentstats.py /path/to/torrent/stats/file" + sys.exit(1) + +torrents = {} +for line in open(whitelist,'r').readlines(): + line = line.strip() + csum, torrent = line.split('-', 1) + torrents[csum.strip()] = torrent.replace('.torrent','') + + +tlist = [] +for line in open(sys.argv[1],'r').readlines(): + line = line.strip() + if not line: + continue + tdict = {} + csum,total,active = line.split(':') + if csum not in torrents: + continue + tdict['name'] = torrents[csum].strip() + tdict['completed'] = int(total) + tdict['downloaders'] = int(active) + tdict['size'] = 0 + tdict['seeds'] = 1 + tlist.append(tdict) + +print simplejson.dumps(tlist)
TorrentDescriptionSizeMapDate
%s
%s%s%s%s%s