Infrastructure/distgit-bugzilla-sync
3
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Run black over the entire project

Browse source 
Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
This commit is contained in:
Pierre-Yves Chibon 2020-04-29 11:40:42 +02:00
parent d37f60b474
commit 392627ecab
4 changed files with 462 additions and 335 deletions
Download patch file Download diff file Expand all files Collapse all files

34
distgit_bugzilla_sync/config.py
View file

@ -23,17 +23,17 @@ import toml
_here = os.path.dirname(__file__)
_default_conf_root = os.path.join(_here, 'default-config-files')
_system_conf_root = os.path.join(os.path.sep, 'etc', 'distgit-bugzilla-sync')
_default_conf_root = os.path.join(_here, "default-config-files")
_system_conf_root = os.path.join(os.path.sep, "etc", "distgit-bugzilla-sync")
config_files = {
'default': {
'configuration': os.path.join(_default_conf_root, 'configuration.toml'),
'email_overrides': os.path.join(_default_conf_root, 'email_overrides.toml'),
"default": {
"configuration": os.path.join(_default_conf_root, "configuration.toml"),
"email_overrides": os.path.join(_default_conf_root, "email_overrides.toml"),
},
'system': {
'configuration': os.path.join(_system_conf_root, 'configuration.toml'),
'email_overrides': os.path.join(_system_conf_root, 'email_overrides.toml'),
"system": {
"configuration": os.path.join(_system_conf_root, "configuration.toml"),
"email_overrides": os.path.join(_system_conf_root, "email_overrides.toml"),
},
}
@ -85,8 +85,10 @@ class ConfigDict(dict):
self[k] = v
def load_configuration(addl_config_files: Optional[Sequence[str]] = None,
addl_email_overrides_files: Optional[Sequence[str]] = None):
def load_configuration(
addl_config_files: Optional[Sequence[str]] = None,
addl_email_overrides_files: Optional[Sequence[str]] = None,
):
"""Load stored configuration.
This function loads default, system-wide, and if specified, additional
@ -99,14 +101,14 @@ def load_configuration(addl_config_files: Optional[Sequence[str]] = None,
# Load default files.
try:
default_config = toml.load(config_files['default']['configuration'])
default_config = toml.load(config_files["default"]["configuration"])
except FileNotFoundError as e:
raise RuntimeError(
f"Default configuration file {config_files['default']['configuration']} not found."
) from e
try:
default_email_overrides = toml.load(config_files['default']['email_overrides'])
default_email_overrides = toml.load(config_files["default"]["email_overrides"])
except FileNotFoundError as e:
raise RuntimeError(
f"Default email overrides file {config_files['default']['email_overrides']} not found."
@ -114,12 +116,12 @@ def load_configuration(addl_config_files: Optional[Sequence[str]] = None,
# Attempt to load system-wide files.
try:
system_config = toml.load(config_files['system']['configuration'])
system_config = toml.load(config_files["system"]["configuration"])
except FileNotFoundError:
system_config = {}
try:
system_email_overrides = toml.load(config_files['system']['email_overrides'])
system_email_overrides = toml.load(config_files["system"]["email_overrides"])
except FileNotFoundError:
system_email_overrides = {}
@ -152,10 +154,10 @@ def load_configuration(addl_config_files: Optional[Sequence[str]] = None,
email_overrides.update(system_email_overrides)
email_overrides.update(addl_email_overrides)
for env in config['environments'].values():
for env in config["environments"].values():
# Fill environments with default data.
env_values = copy.deepcopy(config)
del env_values['environments']
del env_values["environments"]
# Values specified in the environments should take precedence.
env_values.update(env)

92
distgit_bugzilla_sync/package_summaries.py
View file

@ -22,44 +22,48 @@ from defusedxml import cElementTree as etree
import requests
KOJI_REPO = 'https://kojipkgs.fedoraproject.org/repos/'
KOJI_REPO = "https://kojipkgs.fedoraproject.org/repos/"
repomd_xml_namespace = {
'repo': 'http://linux.duke.edu/metadata/repo',
'rpm': 'http://linux.duke.edu/metadata/rpm',
"repo": "http://linux.duke.edu/metadata/repo",
"rpm": "http://linux.duke.edu/metadata/rpm",
}
log = logging.getLogger(__name__)
def download_db(name, repomd_url, archive):
log.info('%-12s Downloading file: %s to %s', name, repomd_url, archive)
log.info("%-12s Downloading file: %s to %s", name, repomd_url, archive)
response = requests.get(repomd_url, verify=True)
with open(archive, 'wb') as stream:
with open(archive, "wb") as stream:
stream.write(response.content)
def decompress_db(name, archive, location):
''' Decompress the given archive at the specified location. '''
log.info('%-12s Extracting %s to %s', name, archive, location)
if archive.endswith('.xz'):
""" Decompress the given archive at the specified location. """
log.info("%-12s Extracting %s to %s", name, archive, location)
if archive.endswith(".xz"):
import lzma
with contextlib.closing(lzma.LZMAFile(archive)) as stream_xz:
data = stream_xz.read()
with open(location, 'wb') as stream:
with open(location, "wb") as stream:
stream.write(data)
elif archive.endswith('.tar.gz'):
elif archive.endswith(".tar.gz"):
import tarfile
with tarfile.open(archive) as tar:
tar.extractall(path=location)
elif archive.endswith('.gz'):
elif archive.endswith(".gz"):
import gzip
with open(location, 'wb') as out:
with gzip.open(archive, 'rb') as inp:
with open(location, "wb") as out:
with gzip.open(archive, "rb") as inp:
out.write(inp.read())
elif archive.endswith('.bz2'):
elif archive.endswith(".bz2"):
import bz2
with open(location, 'wb') as out:
with open(location, "wb") as out:
bzar = bz2.BZ2File(archive)
out.write(bzar.read())
bzar.close()
@ -68,9 +72,9 @@ def decompress_db(name, archive, location):
def needs_update(local_file, remote_sha, sha_type):
''' Compare hash of a local and remote file.
""" Compare hash of a local and remote file.
Return True if our local file needs to be updated.
'''
"""
if not os.path.isfile(local_file):
# If we have never downloaded this before, then obviously it has
@ -78,11 +82,11 @@ def needs_update(local_file, remote_sha, sha_type):
return True
# Old epel5 doesn't even know which sha it is using...
if sha_type == 'sha':
sha_type = 'sha1'
if sha_type == "sha":
sha_type = "sha1"
hashobj = getattr(hashlib, sha_type)()
with open(local_file, 'rb') as f:
with open(local_file, "rb") as f:
hashobj.update(f.read())
local_sha = hashobj.hexdigest()
@ -93,24 +97,24 @@ def needs_update(local_file, remote_sha, sha_type):
def get_primary_xml(destfolder, url, name):
''' Retrieve the repo metadata at the given url and store them using
""" Retrieve the repo metadata at the given url and store them using
the provided name.
'''
repomd_url = url + '/repomd.xml'
"""
repomd_url = url + "/repomd.xml"
response = requests.get(repomd_url, verify=True)
if not bool(response):
log.warning('%-12s !! Failed to get %s %s', name, repomd_url, response)
log.warning("%-12s !! Failed to get %s %s", name, repomd_url, response)
return
try:
root = etree.fromstring(response.text)
except ParseError:
log.warning('%-12s !! Failed to parse %s %s', name, repomd_url, response)
log.warning("%-12s !! Failed to parse %s %s", name, repomd_url, response)
return
data_nodes = list(root.findall('repo:data[@type="primary"]', repomd_xml_namespace))
if not data_nodes:
log.debug('No primary.xml could be found in %s', url)
log.debug("No primary.xml could be found in %s", url)
return
elif len(data_nodes) > 1:
log.debug("More than one primary.xml could be found in %s", url)
@ -118,21 +122,21 @@ def get_primary_xml(destfolder, url, name):
primary_node = data_nodes[0]
location_node = primary_node.find('repo:location', repomd_xml_namespace)
if location_node is None or 'href' not in location_node.attrib:
log.debug('No valid location found for primary.xml in %s', url)
location_node = primary_node.find("repo:location", repomd_xml_namespace)
if location_node is None or "href" not in location_node.attrib:
log.debug("No valid location found for primary.xml in %s", url)
return
cksuminfo_node = primary_node.find('repo:open-checksum', repomd_xml_namespace)
if cksuminfo_node is None or 'type' not in cksuminfo_node.attrib:
log.debug('No valid checksum information found for primary.xml in %s', url)
cksuminfo_node = primary_node.find("repo:open-checksum", repomd_xml_namespace)
if cksuminfo_node is None or "type" not in cksuminfo_node.attrib:
log.debug("No valid checksum information found for primary.xml in %s", url)
return
filename = location_node.attrib['href'].replace('repodata/', '')
filename = location_node.attrib["href"].replace("repodata/", "")
hash_digest = cksuminfo_node.text
hash_type = cksuminfo_node.attrib['type']
hash_type = cksuminfo_node.attrib["type"]
repomd_url = url + '/' + filename
repomd_url = url + "/" + filename
# First, determine if the file has changed by comparing hash
db = "distgit-bugzilla-sync-primary.xml"
@ -140,7 +144,7 @@ def get_primary_xml(destfolder, url, name):
# Have we downloaded this before? Did it change?
destfile = os.path.join(destfolder, db)
if not needs_update(destfile, hash_digest, hash_type):
log.debug('%s No change of %s', name.ljust(12), repomd_url)
log.debug("%s No change of %s", name.ljust(12), repomd_url)
else:
# If it has changed, then download it and move it into place.
archive = os.path.join(destfolder, filename)
@ -157,12 +161,10 @@ def get_package_summaries():
start = time.time()
primary_xml = get_primary_xml(
"/var/tmp",
KOJI_REPO + 'rawhide/latest/x86_64/repodata',
"koji",
"/var/tmp", KOJI_REPO + "rawhide/latest/x86_64/repodata", "koji",
)
context = etree.iterparse(primary_xml, events=('start', 'end'))
context = etree.iterparse(primary_xml, events=("start", "end"))
root = None
@ -172,9 +174,13 @@ def get_package_summaries():
root = elem
continue
if event == 'end' and elem.tag == 'package' and elem.get('type', 'rpm') == 'rpm':
name = elem.findtext('name')
summary = elem.findtext('summary')
if (
event == "end"
and elem.tag == "package"
and elem.get("type", "rpm") == "rpm"
):
name = elem.findtext("name")
summary = elem.findtext("summary")
if name is not None and summary is not None:
summaries[name] = summary
# remove package child from root element to keep memory consumption low

617
distgit_bugzilla_sync/script.py
View file

File diff suppressed because it is too large Load diff

54
setup.py
View file

@ -4,49 +4,49 @@ from setuptools import setup
HERE = os.path.dirname(__file__)
with open(os.path.join(HERE, 'requirements.txt'), 'r') as f:
with open(os.path.join(HERE, "requirements.txt"), "r") as f:
INSTALL_REQUIRES = [x.strip() for x in f.readlines()]
with open(os.path.join(HERE, 'test_requirements.txt'), 'r') as f:
with open(os.path.join(HERE, "test_requirements.txt"), "r") as f:
TESTS_REQUIRE = [x.strip() for x in f.readlines()]
setup(
name='distgit-bugzilla-sync',
version='0.1',
description='script to set default assignee, CC list from component owners',
name="distgit-bugzilla-sync",
version="0.1",
description="script to set default assignee, CC list from component owners",
# Possible options are at https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers=[
'Development Status :: 3 - Alpha',
'Intended Audience :: Developers',
'Intended Audience :: System Administrators',
'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Topic :: Software Development :: Bug Tracking',
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"Intended Audience :: System Administrators",
"License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)",
"Operating System :: POSIX :: Linux",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Topic :: Software Development :: Bug Tracking",
],
license='GPLv2+',
maintainer='Fedora Infrastructure Team',
maintainer_email='infrastructure@lists.fedoraproject.org',
platforms=['Fedora', 'GNU/Linux'],
url='https://pagure.io/Fedora-Infra/distgit-bugzilla-sync',
keywords='fedora',
packages=['distgit_bugzilla_sync'],
license="GPLv2+",
maintainer="Fedora Infrastructure Team",
maintainer_email="infrastructure@lists.fedoraproject.org",
platforms=["Fedora", "GNU/Linux"],
url="https://pagure.io/Fedora-Infra/distgit-bugzilla-sync",
keywords="fedora",
packages=["distgit_bugzilla_sync"],
include_package_data=False,
package_data={
'distgit_bugzilla_sync': [
'default-config-files/configuration.toml',
'default-config-files/email_overrides.toml',
"distgit_bugzilla_sync": [
"default-config-files/configuration.toml",
"default-config-files/email_overrides.toml",
],
},
zip_safe=False,
install_requires=INSTALL_REQUIRES,
tests_require=TESTS_REQUIRE,
entry_points={
'console_scripts': [
'distgit-bugzilla-sync = distgit_bugzilla_sync.script:DistgitBugzillaSync.main',
"console_scripts": [
"distgit-bugzilla-sync = distgit_bugzilla_sync.script:DistgitBugzillaSync.main",
],
},
)