b3d5bd9d8e
The script checks for the file at the new location. As a result, it will report that the file is missing if it had only been uploaded to the old location, which will prompt the client to reupload. With this change, the script will check at the new location, and if it doesn't find the file it will try checking for it at the old location as well. If the file is found at the old location, we hardlink it to the new location, and report the file is available.
241 lines
7.6 KiB
Python
241 lines
7.6 KiB
Python
#!/usr/bin/python
|
|
#
|
|
# CGI script to handle file updates for the rpms git repository. There
|
|
# is nothing really complex here other than tedious checking of our
|
|
# every step along the way...
|
|
#
|
|
# License: GPL
|
|
|
|
import os
|
|
import sys
|
|
import cgi
|
|
import tempfile
|
|
import grp
|
|
import errno
|
|
|
|
import fedmsg
|
|
import fedmsg.config
|
|
|
|
import hashlib
|
|
|
|
# Reading buffer size
|
|
BUFFER_SIZE = 4096
|
|
|
|
# We check modules exist from this dircetory
|
|
GITREPO = '/srv/git/rpms'
|
|
|
|
# Lookaside cache directory
|
|
CACHE_DIR = '/srv/cache/lookaside/pkgs'
|
|
|
|
# Fedora Packager Group
|
|
PACKAGER_GROUP = 'packager'
|
|
|
|
def send_error(text, status='500 Internal Server Error'):
|
|
"""Send an error back to the client
|
|
|
|
This ensures that the client will get a proper error, including the HTTP
|
|
status code, so that it can handle problems appropriately.
|
|
|
|
Args:
|
|
text (str): The error message to send the client
|
|
status (str, optional): The HTTP status code to return to the client.
|
|
"""
|
|
print 'Status: %s' % status
|
|
print 'Content-type: text/plain'
|
|
print
|
|
print text
|
|
sys.exit(0)
|
|
|
|
def check_form(form, var):
|
|
ret = form.getvalue(var, None)
|
|
if ret is None:
|
|
send_error('Required field "%s" is not present.' % var,
|
|
status='400 Bad Request')
|
|
if isinstance(ret, list):
|
|
send_error('Multiple values given for "%s". Aborting.' % var,
|
|
status='400 Bad Request')
|
|
return ret
|
|
|
|
def check_auth(username):
|
|
authenticated = False
|
|
try:
|
|
if username in grp.getgrnam(PACKAGER_GROUP)[3]:
|
|
authenticated = True
|
|
except KeyError:
|
|
pass
|
|
return authenticated
|
|
|
|
|
|
def hardlink(src, dest):
|
|
try:
|
|
os.makedirs(os.path.dirname(dest))
|
|
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
send_error(str(e))
|
|
|
|
try:
|
|
os.link(src, dest)
|
|
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
send_error(str(e))
|
|
|
|
# The file already existed at the dest path, hardlink over it
|
|
os.unlink(dest)
|
|
os.link(src, dest)
|
|
|
|
|
|
def main():
|
|
os.umask(002)
|
|
|
|
username = os.environ.get('SSL_CLIENT_S_DN_CN', None)
|
|
if not check_auth(username):
|
|
send_error('You must connect with a valid certificate and be in the %s group to upload.' % PACKAGER_GROUP,
|
|
status='403 Forbidden')
|
|
|
|
print 'Content-Type: text/plain'
|
|
print
|
|
|
|
assert os.environ['REQUEST_URI'].split('/')[1] == 'repo'
|
|
|
|
form = cgi.FieldStorage()
|
|
name = check_form(form, 'name')
|
|
|
|
# Search for the file hash, start with stronger hash functions
|
|
if form.has_key('sha512sum'):
|
|
checksum = check_form(form, 'sha512sum')
|
|
hash_type = "sha512"
|
|
|
|
elif form.has_key('md5sum'):
|
|
# Fallback on md5, as it's what we currently use
|
|
checksum = check_form(form, 'md5sum')
|
|
hash_type = "md5"
|
|
|
|
else:
|
|
send_error('Required checksum is not present.',
|
|
status='400 Bad Request')
|
|
|
|
action = None
|
|
upload_file = None
|
|
filename = None
|
|
|
|
# Is this a submission or a test?
|
|
# in a test, we don't get a file, just a filename.
|
|
# In a submission, we don;t get a filename, just the file.
|
|
if form.has_key('filename'):
|
|
action = 'check'
|
|
filename = check_form(form, 'filename')
|
|
filename = os.path.basename(filename)
|
|
print >> sys.stderr, '[username=%s] Checking file status: NAME=%s FILENAME=%s %sSUM=%s' % (username, name, filename, hash_type.upper(), checksum)
|
|
else:
|
|
action = 'upload'
|
|
if form.has_key('file'):
|
|
upload_file = form['file']
|
|
if not upload_file.file:
|
|
send_error('No file given for upload. Aborting.',
|
|
status='400 Bad Request')
|
|
filename = os.path.basename(upload_file.filename)
|
|
else:
|
|
send_error('Required field "file" is not present.',
|
|
status='400 Bad Request')
|
|
print >> sys.stderr, '[username=%s] Processing upload request: NAME=%s FILENAME=%s %sSUM=%s' % (username, name, filename, hash_type.upper(), checksum)
|
|
|
|
module_dir = os.path.join(CACHE_DIR, name)
|
|
hash_dir = os.path.join(module_dir, filename, hash_type, checksum)
|
|
msgpath = os.path.join(name, module_dir, filename, hash_type, checksum, filename)
|
|
|
|
unwanted_prefix = '/srv/cache/lookaside/pkgs/'
|
|
if msgpath.startswith(unwanted_prefix):
|
|
msgpath = msgpath[len(unwanted_prefix):]
|
|
|
|
# first test if the module really exists
|
|
git_dir = os.path.join(GITREPO, '%s.git' % name)
|
|
if not os.path.isdir(git_dir):
|
|
print >> sys.stderr, '[username=%s] Unknown module: %s' % (username, name)
|
|
send_error('Module "%s" does not exist!' % name,
|
|
status='404 Not Found')
|
|
|
|
# try to see if we already have this file...
|
|
dest_file = os.path.join(hash_dir, filename)
|
|
old_dir = os.path.join(module_dir, filename, checksum)
|
|
old_path = os.path.join(old_dir, filename)
|
|
|
|
if os.path.exists(dest_file):
|
|
if action == 'check':
|
|
print 'Available'
|
|
else:
|
|
upload_file.file.close()
|
|
dest_file_stat = os.stat(dest_file)
|
|
print 'File %s already exists' % filename
|
|
print 'File: %s Size: %d' % (dest_file, dest_file_stat.st_size)
|
|
sys.exit(0)
|
|
elif action == 'check':
|
|
if os.path.exists(old_path):
|
|
# The file had been uploaded at the old path
|
|
hardlink(old_path, dest_file)
|
|
print 'Available'
|
|
else:
|
|
print 'Missing'
|
|
|
|
sys.exit(0)
|
|
|
|
# check that all directories are in place
|
|
if not os.path.isdir(module_dir):
|
|
os.makedirs(module_dir, 02775)
|
|
|
|
# grab a temporary filename and dump our file in there
|
|
tempfile.tempdir = module_dir
|
|
tmpfile = tempfile.mkstemp(checksum)[1]
|
|
tmpfd = open(tmpfile, 'w')
|
|
|
|
# now read the whole file in
|
|
m = getattr(hashlib, hash_type)()
|
|
filesize = 0
|
|
while True:
|
|
data = upload_file.file.read(BUFFER_SIZE)
|
|
if not data:
|
|
break
|
|
tmpfd.write(data)
|
|
m.update(data)
|
|
filesize += len(data)
|
|
|
|
# now we're done reading, check the checksum of what we got
|
|
tmpfd.close()
|
|
check_checksum = m.hexdigest()
|
|
if checksum != check_checksum:
|
|
os.unlink(tmpfile)
|
|
send_error("%s check failed. Received %s instead of %s." % (hash_type.upper(), check_checksum, checksum),
|
|
status='400 Bad Request')
|
|
|
|
# wow, even the checksum matches. make sure full path is valid now
|
|
if not os.path.isdir(hash_dir):
|
|
os.makedirs(hash_dir, 02775)
|
|
print >> sys.stderr, '[username=%s] mkdir %s' % (username, hash_dir)
|
|
|
|
os.rename(tmpfile, dest_file)
|
|
os.chmod(dest_file, 0644)
|
|
|
|
print >> sys.stderr, '[username=%s] Stored %s (%d bytes)' % (username, dest_file, filesize)
|
|
print 'File %s size %d %s %s stored OK' % (filename, filesize, hash_type.upper(), checksum)
|
|
|
|
# Add the file to the old path, where fedpkg is currently looking for it
|
|
if hash_type == "md5":
|
|
hardlink(dest_file, old_path)
|
|
|
|
# Emit a fedmsg message. Load the config to talk to the fedmsg-relay.
|
|
try:
|
|
config = fedmsg.config.load_config([], None)
|
|
config['active'] = True
|
|
config['endpoints']['relay_inbound'] = config['relay_inbound']
|
|
fedmsg.init(name="relay_inbound", cert_prefix="lookaside", **config)
|
|
|
|
topic = "lookaside.new"
|
|
msg = dict(name=name, md5sum=checksum, filename=filename.split('/')[-1],
|
|
agent=username, path=msgpath)
|
|
fedmsg.publish(modname="git", topic=topic, msg=msg)
|
|
except Exception as e:
|
|
print "Error with fedmsg", str(e)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|