Infrastructure/distgit-bugzilla-sync
3
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Run black over the entire project

Browse source 
Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
This commit is contained in:
Pierre-Yves Chibon 2020-04-29 11:40:42 +02:00
parent d37f60b474
commit 392627ecab
4 changed files with 462 additions and 335 deletions
Download patch file Download diff file Expand all files Collapse all files

34
distgit_bugzilla_sync/config.py
View file

@ -23,17 +23,17 @@ import toml
_here = os.path.dirname(__file__) _here = os.path.dirname(__file__)
_default_conf_root = os.path.join(_here, 'default-config-files') _default_conf_root = os.path.join(_here, "default-config-files")
_system_conf_root = os.path.join(os.path.sep, 'etc', 'distgit-bugzilla-sync') _system_conf_root = os.path.join(os.path.sep, "etc", "distgit-bugzilla-sync")
config_files = { config_files = {
'default': { "default": {
'configuration': os.path.join(_default_conf_root, 'configuration.toml'), "configuration": os.path.join(_default_conf_root, "configuration.toml"),
'email_overrides': os.path.join(_default_conf_root, 'email_overrides.toml'), "email_overrides": os.path.join(_default_conf_root, "email_overrides.toml"),
}, },
'system': { "system": {
'configuration': os.path.join(_system_conf_root, 'configuration.toml'), "configuration": os.path.join(_system_conf_root, "configuration.toml"),
'email_overrides': os.path.join(_system_conf_root, 'email_overrides.toml'), "email_overrides": os.path.join(_system_conf_root, "email_overrides.toml"),
}, },
} }
@ -85,8 +85,10 @@ class ConfigDict(dict):
self[k] = v self[k] = v
def load_configuration(addl_config_files: Optional[Sequence[str]] = None, def load_configuration(
addl_email_overrides_files: Optional[Sequence[str]] = None): addl_config_files: Optional[Sequence[str]] = None,
addl_email_overrides_files: Optional[Sequence[str]] = None,
):
"""Load stored configuration. """Load stored configuration.
This function loads default, system-wide, and if specified, additional This function loads default, system-wide, and if specified, additional
@ -99,14 +101,14 @@ def load_configuration(addl_config_files: Optional[Sequence[str]] = None,
# Load default files. # Load default files.
try: try:
default_config = toml.load(config_files['default']['configuration']) default_config = toml.load(config_files["default"]["configuration"])
except FileNotFoundError as e: except FileNotFoundError as e:
raise RuntimeError( raise RuntimeError(
f"Default configuration file {config_files['default']['configuration']} not found." f"Default configuration file {config_files['default']['configuration']} not found."
) from e ) from e
try: try:
default_email_overrides = toml.load(config_files['default']['email_overrides']) default_email_overrides = toml.load(config_files["default"]["email_overrides"])
except FileNotFoundError as e: except FileNotFoundError as e:
raise RuntimeError( raise RuntimeError(
f"Default email overrides file {config_files['default']['email_overrides']} not found." f"Default email overrides file {config_files['default']['email_overrides']} not found."
@ -114,12 +116,12 @@ def load_configuration(addl_config_files: Optional[Sequence[str]] = None,
# Attempt to load system-wide files. # Attempt to load system-wide files.
try: try:
system_config = toml.load(config_files['system']['configuration']) system_config = toml.load(config_files["system"]["configuration"])
except FileNotFoundError: except FileNotFoundError:
system_config = {} system_config = {}
try: try:
system_email_overrides = toml.load(config_files['system']['email_overrides']) system_email_overrides = toml.load(config_files["system"]["email_overrides"])
except FileNotFoundError: except FileNotFoundError:
system_email_overrides = {} system_email_overrides = {}
@ -152,10 +154,10 @@ def load_configuration(addl_config_files: Optional[Sequence[str]] = None,
email_overrides.update(system_email_overrides) email_overrides.update(system_email_overrides)
email_overrides.update(addl_email_overrides) email_overrides.update(addl_email_overrides)
for env in config['environments'].values(): for env in config["environments"].values():
# Fill environments with default data. # Fill environments with default data.
env_values = copy.deepcopy(config) env_values = copy.deepcopy(config)
del env_values['environments'] del env_values["environments"]
# Values specified in the environments should take precedence. # Values specified in the environments should take precedence.
env_values.update(env) env_values.update(env)

92
distgit_bugzilla_sync/package_summaries.py
View file

@ -22,44 +22,48 @@ from defusedxml import cElementTree as etree
import requests import requests
KOJI_REPO = 'https://kojipkgs.fedoraproject.org/repos/' KOJI_REPO = "https://kojipkgs.fedoraproject.org/repos/"
repomd_xml_namespace = { repomd_xml_namespace = {
'repo': 'http://linux.duke.edu/metadata/repo', "repo": "http://linux.duke.edu/metadata/repo",
'rpm': 'http://linux.duke.edu/metadata/rpm', "rpm": "http://linux.duke.edu/metadata/rpm",
} }
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def download_db(name, repomd_url, archive): def download_db(name, repomd_url, archive):
log.info('%-12s Downloading file: %s to %s', name, repomd_url, archive) log.info("%-12s Downloading file: %s to %s", name, repomd_url, archive)
response = requests.get(repomd_url, verify=True) response = requests.get(repomd_url, verify=True)
with open(archive, 'wb') as stream: with open(archive, "wb") as stream:
stream.write(response.content) stream.write(response.content)
def decompress_db(name, archive, location): def decompress_db(name, archive, location):
''' Decompress the given archive at the specified location. ''' """ Decompress the given archive at the specified location. """
log.info('%-12s Extracting %s to %s', name, archive, location) log.info("%-12s Extracting %s to %s", name, archive, location)
if archive.endswith('.xz'): if archive.endswith(".xz"):
import lzma import lzma
with contextlib.closing(lzma.LZMAFile(archive)) as stream_xz: with contextlib.closing(lzma.LZMAFile(archive)) as stream_xz:
data = stream_xz.read() data = stream_xz.read()
with open(location, 'wb') as stream: with open(location, "wb") as stream:
stream.write(data) stream.write(data)
elif archive.endswith('.tar.gz'): elif archive.endswith(".tar.gz"):
import tarfile import tarfile
with tarfile.open(archive) as tar: with tarfile.open(archive) as tar:
tar.extractall(path=location) tar.extractall(path=location)
elif archive.endswith('.gz'): elif archive.endswith(".gz"):
import gzip import gzip
with open(location, 'wb') as out:
with gzip.open(archive, 'rb') as inp: with open(location, "wb") as out:
with gzip.open(archive, "rb") as inp:
out.write(inp.read()) out.write(inp.read())
elif archive.endswith('.bz2'): elif archive.endswith(".bz2"):
import bz2 import bz2
with open(location, 'wb') as out:
with open(location, "wb") as out:
bzar = bz2.BZ2File(archive) bzar = bz2.BZ2File(archive)
out.write(bzar.read()) out.write(bzar.read())
bzar.close() bzar.close()
@ -68,9 +72,9 @@ def decompress_db(name, archive, location):
def needs_update(local_file, remote_sha, sha_type): def needs_update(local_file, remote_sha, sha_type):
''' Compare hash of a local and remote file. """ Compare hash of a local and remote file.
Return True if our local file needs to be updated. Return True if our local file needs to be updated.
''' """
if not os.path.isfile(local_file): if not os.path.isfile(local_file):
# If we have never downloaded this before, then obviously it has # If we have never downloaded this before, then obviously it has
@ -78,11 +82,11 @@ def needs_update(local_file, remote_sha, sha_type):
return True return True
# Old epel5 doesn't even know which sha it is using... # Old epel5 doesn't even know which sha it is using...
if sha_type == 'sha': if sha_type == "sha":
sha_type = 'sha1' sha_type = "sha1"
hashobj = getattr(hashlib, sha_type)() hashobj = getattr(hashlib, sha_type)()
with open(local_file, 'rb') as f: with open(local_file, "rb") as f:
hashobj.update(f.read()) hashobj.update(f.read())
local_sha = hashobj.hexdigest() local_sha = hashobj.hexdigest()
@ -93,24 +97,24 @@ def needs_update(local_file, remote_sha, sha_type):
def get_primary_xml(destfolder, url, name): def get_primary_xml(destfolder, url, name):
''' Retrieve the repo metadata at the given url and store them using """ Retrieve the repo metadata at the given url and store them using
the provided name. the provided name.
''' """
repomd_url = url + '/repomd.xml' repomd_url = url + "/repomd.xml"
response = requests.get(repomd_url, verify=True) response = requests.get(repomd_url, verify=True)
if not bool(response): if not bool(response):
log.warning('%-12s !! Failed to get %s %s', name, repomd_url, response) log.warning("%-12s !! Failed to get %s %s", name, repomd_url, response)
return return
try: try:
root = etree.fromstring(response.text) root = etree.fromstring(response.text)
except ParseError: except ParseError:
log.warning('%-12s !! Failed to parse %s %s', name, repomd_url, response) log.warning("%-12s !! Failed to parse %s %s", name, repomd_url, response)
return return
data_nodes = list(root.findall('repo:data[@type="primary"]', repomd_xml_namespace)) data_nodes = list(root.findall('repo:data[@type="primary"]', repomd_xml_namespace))
if not data_nodes: if not data_nodes:
log.debug('No primary.xml could be found in %s', url) log.debug("No primary.xml could be found in %s", url)
return return
elif len(data_nodes) > 1: elif len(data_nodes) > 1:
log.debug("More than one primary.xml could be found in %s", url) log.debug("More than one primary.xml could be found in %s", url)
@ -118,21 +122,21 @@ def get_primary_xml(destfolder, url, name):
primary_node = data_nodes[0] primary_node = data_nodes[0]
location_node = primary_node.find('repo:location', repomd_xml_namespace) location_node = primary_node.find("repo:location", repomd_xml_namespace)
if location_node is None or 'href' not in location_node.attrib: if location_node is None or "href" not in location_node.attrib:
log.debug('No valid location found for primary.xml in %s', url) log.debug("No valid location found for primary.xml in %s", url)
return return
cksuminfo_node = primary_node.find('repo:open-checksum', repomd_xml_namespace) cksuminfo_node = primary_node.find("repo:open-checksum", repomd_xml_namespace)
if cksuminfo_node is None or 'type' not in cksuminfo_node.attrib: if cksuminfo_node is None or "type" not in cksuminfo_node.attrib:
log.debug('No valid checksum information found for primary.xml in %s', url) log.debug("No valid checksum information found for primary.xml in %s", url)
return return
filename = location_node.attrib['href'].replace('repodata/', '') filename = location_node.attrib["href"].replace("repodata/", "")
hash_digest = cksuminfo_node.text hash_digest = cksuminfo_node.text
hash_type = cksuminfo_node.attrib['type'] hash_type = cksuminfo_node.attrib["type"]
repomd_url = url + '/' + filename repomd_url = url + "/" + filename
# First, determine if the file has changed by comparing hash # First, determine if the file has changed by comparing hash
db = "distgit-bugzilla-sync-primary.xml" db = "distgit-bugzilla-sync-primary.xml"
@ -140,7 +144,7 @@ def get_primary_xml(destfolder, url, name):
# Have we downloaded this before? Did it change? # Have we downloaded this before? Did it change?
destfile = os.path.join(destfolder, db) destfile = os.path.join(destfolder, db)
if not needs_update(destfile, hash_digest, hash_type): if not needs_update(destfile, hash_digest, hash_type):
log.debug('%s No change of %s', name.ljust(12), repomd_url) log.debug("%s No change of %s", name.ljust(12), repomd_url)
else: else:
# If it has changed, then download it and move it into place. # If it has changed, then download it and move it into place.
archive = os.path.join(destfolder, filename) archive = os.path.join(destfolder, filename)
@ -157,12 +161,10 @@ def get_package_summaries():
start = time.time() start = time.time()
primary_xml = get_primary_xml( primary_xml = get_primary_xml(
"/var/tmp", "/var/tmp", KOJI_REPO + "rawhide/latest/x86_64/repodata", "koji",
KOJI_REPO + 'rawhide/latest/x86_64/repodata',
"koji",
) )
context = etree.iterparse(primary_xml, events=('start', 'end')) context = etree.iterparse(primary_xml, events=("start", "end"))
root = None root = None
@ -172,9 +174,13 @@ def get_package_summaries():
root = elem root = elem
continue continue
if event == 'end' and elem.tag == 'package' and elem.get('type', 'rpm') == 'rpm': if (
name = elem.findtext('name') event == "end"
summary = elem.findtext('summary') and elem.tag == "package"
and elem.get("type", "rpm") == "rpm"
):
name = elem.findtext("name")
summary = elem.findtext("summary")
if name is not None and summary is not None: if name is not None and summary is not None:
summaries[name] = summary summaries[name] = summary
# remove package child from root element to keep memory consumption low # remove package child from root element to keep memory consumption low

617
distgit_bugzilla_sync/script.py
View file

File diff suppressed because it is too large Load diff

54
setup.py
View file

@ -4,49 +4,49 @@ from setuptools import setup
HERE = os.path.dirname(__file__) HERE = os.path.dirname(__file__)
with open(os.path.join(HERE, 'requirements.txt'), 'r') as f: with open(os.path.join(HERE, "requirements.txt"), "r") as f:
INSTALL_REQUIRES = [x.strip() for x in f.readlines()] INSTALL_REQUIRES = [x.strip() for x in f.readlines()]
with open(os.path.join(HERE, 'test_requirements.txt'), 'r') as f: with open(os.path.join(HERE, "test_requirements.txt"), "r") as f:
TESTS_REQUIRE = [x.strip() for x in f.readlines()] TESTS_REQUIRE = [x.strip() for x in f.readlines()]
setup( setup(
name='distgit-bugzilla-sync', name="distgit-bugzilla-sync",
version='0.1', version="0.1",
description='script to set default assignee, CC list from component owners', description="script to set default assignee, CC list from component owners",
# Possible options are at https://pypi.python.org/pypi?%3Aaction=list_classifiers # Possible options are at https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers=[ classifiers=[
'Development Status :: 3 - Alpha', "Development Status :: 3 - Alpha",
'Intended Audience :: Developers', "Intended Audience :: Developers",
'Intended Audience :: System Administrators', "Intended Audience :: System Administrators",
'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)",
'Operating System :: POSIX :: Linux', "Operating System :: POSIX :: Linux",
'Programming Language :: Python :: 3', "Programming Language :: Python :: 3",
'Programming Language :: Python :: 3.6', "Programming Language :: Python :: 3.6",
'Programming Language :: Python :: 3.7', "Programming Language :: Python :: 3.7",
'Programming Language :: Python :: 3.8', "Programming Language :: Python :: 3.8",
'Topic :: Software Development :: Bug Tracking', "Topic :: Software Development :: Bug Tracking",
], ],
license='GPLv2+', license="GPLv2+",
maintainer='Fedora Infrastructure Team', maintainer="Fedora Infrastructure Team",
maintainer_email='infrastructure@lists.fedoraproject.org', maintainer_email="infrastructure@lists.fedoraproject.org",
platforms=['Fedora', 'GNU/Linux'], platforms=["Fedora", "GNU/Linux"],
url='https://pagure.io/Fedora-Infra/distgit-bugzilla-sync', url="https://pagure.io/Fedora-Infra/distgit-bugzilla-sync",
keywords='fedora', keywords="fedora",
packages=['distgit_bugzilla_sync'], packages=["distgit_bugzilla_sync"],
include_package_data=False, include_package_data=False,
package_data={ package_data={
'distgit_bugzilla_sync': [ "distgit_bugzilla_sync": [
'default-config-files/configuration.toml', "default-config-files/configuration.toml",
'default-config-files/email_overrides.toml', "default-config-files/email_overrides.toml",
], ],
}, },
zip_safe=False, zip_safe=False,
install_requires=INSTALL_REQUIRES, install_requires=INSTALL_REQUIRES,
tests_require=TESTS_REQUIRE, tests_require=TESTS_REQUIRE,
entry_points={ entry_points={
'console_scripts': [ "console_scripts": [
'distgit-bugzilla-sync = distgit_bugzilla_sync.script:DistgitBugzillaSync.main', "distgit-bugzilla-sync = distgit_bugzilla_sync.script:DistgitBugzillaSync.main",
], ],
}, },
) )