diff --git a/.zuul.yaml b/.zuul.yaml index bc47a30..c0c34be 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -12,6 +12,7 @@ - python3.12 - python3.12-devel - poetry + - libmemcached-devel - fi-tox-format: vars: tox_envlist: black @@ -23,6 +24,7 @@ - python3.12 - python3.12-devel - poetry + - libmemcached-devel - fi-tox-lint: vars: tox_envlist: flake8 @@ -34,6 +36,7 @@ - python3.12 - python3.12-devel - poetry + - libmemcached-devel - fi-tox-python39: vars: dependencies: @@ -42,6 +45,7 @@ - gobject-introspection-devel - libmodulemd - poetry + - libmemcached-devel - fi-tox-python310: vars: dependencies: @@ -56,6 +60,7 @@ - python3-py - python3-toml - poetry + - libmemcached-devel - fi-tox-python311: vars: dependencies: @@ -70,6 +75,7 @@ - python3-py - python3-toml - poetry + - libmemcached-devel - fi-tox-python312: vars: dependencies: @@ -84,4 +90,5 @@ - python3-py - python3-toml - poetry + - libmemcached-devel ... diff --git a/poetry.lock b/poetry.lock index 748f179..e63bb8a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "arrow" @@ -608,6 +608,25 @@ Pygments = ">=2.9.0,<3.0.0" [package.extras] toml = ["tomli (>=1.2.1)"] +[[package]] +name = "dogpile-cache" +version = "1.3.3" +description = "A caching front-end based on the Dogpile lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "dogpile.cache-1.3.3-py3-none-any.whl", hash = "sha256:5e211c4902ebdf88c678d268e22454b41e68071632daa9402d8ee24e825ed8ca"}, + {file = "dogpile.cache-1.3.3.tar.gz", hash = "sha256:f84b8ed0b0fb297d151055447fa8dcaf7bae566d4dbdefecdcc1f37662ab588b"}, +] + +[package.dependencies] +decorator = ">=4.0.0" +stevedore = ">=3.0.0" +typing_extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +pifpaf = ["pifpaf (>=2.5.0)", "setuptools"] + [[package]] name = "exceptiongroup" version = "1.2.1" @@ -1310,6 +1329,17 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] +[[package]] +name = "pbr" +version = "6.1.0" +description = "Python Build Reasonableness" +optional = false +python-versions = ">=2.6" +files = [ + {file = "pbr-6.1.0-py2.py3-none-any.whl", hash = "sha256:a776ae228892d8013649c0aeccbb3d5f99ee15e005a4cbb7e61d55a067b28a2a"}, + {file = "pbr-6.1.0.tar.gz", hash = "sha256:788183e382e3d1d7707db08978239965e8b9e4e5ed42669bf4758186734d5f24"}, +] + [[package]] name = "pdc-client" version = "1.8.0" @@ -1484,6 +1514,38 @@ pycairo = ">=1.16" dev = ["flake8", "pytest", "pytest-cov"] docs = ["sphinx (>=4.0,<5.0)", "sphinx-rtd-theme (>=0.5,<2.0)"] +[[package]] +name = "pylibmc" +version = "1.6.3" +description = "Quick and small memcached client for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pylibmc-1.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a7baf4b78720f8b72839b3642b374754cb77cf57dab465a70ed1764d943e19d5"}, + {file = "pylibmc-1.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dd98054a571bd450200a61a12b9ada3424678d17a25456bbf9a6100470401e52"}, + {file = "pylibmc-1.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e847cdc78d82964236599ff5b312bc97fde3d10f4b93c9ee17dc33b7cf3c032a"}, + {file = "pylibmc-1.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f46b5aa0364bca5e02000f5d62eb408d834a20722ffaf7dae20f75e7d009e6c"}, + {file = "pylibmc-1.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c35816082848723455071670770d89b5a531d40e9063fe4e942ea456f86da49"}, + {file = "pylibmc-1.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9d2ff6d702eb5ae502e29d97772dc85c749d596c6cb8c82a5d18f175fd4eabcc"}, + {file = "pylibmc-1.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e589b7e70dec4daf0da1216789713c753d85611d70cfcd32574161cc75b1527e"}, + {file = "pylibmc-1.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b93e381dec1520a3fec922765e04679ac553d2f3fda830a5faa7cdc527280a2"}, + {file = "pylibmc-1.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f2574f390a2ec89b52a84bccca3ae57c21a4bb4d0e72df210d0d66783eee7f98"}, + {file = "pylibmc-1.6.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db8c0f0467182a2a3e8d625b5c60c296f971dd2ee179e865b0262bd44528d676"}, + {file = "pylibmc-1.6.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b34c1e4021b9a395950be19ea9d98f02bea0e3a88be26dbaa7e8ac4416e1232b"}, + {file = "pylibmc-1.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6cce1d7705952eb30a3aca9ea3f054040cbee53c668d4e1e29144110da113bc"}, + {file = "pylibmc-1.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6c4bdd8790aede67a464a32df842cacb562f77b0415a8c7823421f5c07524c6"}, + {file = "pylibmc-1.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:218125aca214d62e6f69e4f8022bd795fbcb3643ad783f5f5ff33c23a1731c73"}, + {file = "pylibmc-1.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f536d73632007358796654ab088d65c55a1a4368a85cfd7c956d2100e2cd8d89"}, + {file = "pylibmc-1.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f2aeff000de7d918806876dfa4880d21b72089f9809ad0b8e7dff26501367ec6"}, + {file = "pylibmc-1.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9ef3dc70ee2dfd0981bdf3a383a044bc591de7e445296a64a24f10a560e8b4f"}, + {file = "pylibmc-1.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b649eb7fdd774290b2da73334456eb01e0d66e3d3685acd88ae6bf456a227dc6"}, + {file = "pylibmc-1.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5251df82535411d8dc08c01141b8e6e61004f0a3ee50db3aa48ffa00e928cebb"}, + {file = "pylibmc-1.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cd61f1ff46aa1ca6b0b3dac17a727cd29ac019e85db868c5523c491eef4459d7"}, + {file = "pylibmc-1.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7660c561e5415f4be01ff4791c1b035359c1d76fed012e18eee907c2d3249deb"}, + {file = "pylibmc-1.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f4516a14b2beff6062d1d240c00098227ca5478c00afba7e8b329415b0d4d67"}, + {file = "pylibmc-1.6.3.tar.gz", hash = "sha256:eefa46115537abad65fbe2e032acd1b3463d9bf9e335af4b0916df4e4d3206e0"}, +] + [[package]] name = "pyopenssl" version = "24.1.0" @@ -2105,6 +2167,20 @@ files = [ {file = "sspilib-0.1.0.tar.gz", hash = "sha256:58b5291553cf6220549c0f855e0e6973f4977375d8236ce47bb581efb3e9b1cf"}, ] +[[package]] +name = "stevedore" +version = "5.4.0" +description = "Manage dynamic plugins for Python applications" +optional = false +python-versions = ">=3.9" +files = [ + {file = "stevedore-5.4.0-py3-none-any.whl", hash = "sha256:b0be3c4748b3ea7b854b265dcb4caa891015e442416422be16f8b31756107857"}, + {file = "stevedore-5.4.0.tar.gz", hash = "sha256:79e92235ecb828fe952b6b8b0c6c87863248631922c8e8e0fa5b17b232c4514d"}, +] + +[package.dependencies] +pbr = ">=2.0.0" + [[package]] name = "swagger-spec-validator" version = "3.0.4" @@ -2563,4 +2639,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "01ee8d5a7e87f9c7475416ba933304d0279d9db723177d1a149d6d361e9b1773" +content-hash = "a49a574d1aad08af2f31b7af7979b211e7bb31d662027e16f59bfd381fa96c38" diff --git a/pyproject.toml b/pyproject.toml index 7100cf8..3cabf6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ python-fedora = "^1.1.1" python-bugzilla = ">=3.2.0" pdc-client = "^1.8.0" zstandard = "^0.23.0" - +dogpile-cache = "^1.3.3" +pylibmc = "^1.6.3" [tool.poetry.group.dev.dependencies] pytest = "^8.2.2" diff --git a/tests/conftest.py b/tests/conftest.py index 928d187..2ff0a51 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,9 +5,15 @@ from unittest.mock import MagicMock import pytest +from toddlers.utils.cache import cache from toddlers.utils.requests import make_session +@pytest.fixture(autouse=True) +def disable_cache(): + cache.configure(backend="dogpile.cache.null", replace_existing_backend=True) + + @pytest.fixture def toddler(request, monkeypatch): """Fixture creating a toddler for a class testing it @@ -30,6 +36,8 @@ def toddler(request, monkeypatch): monkeypatch.setattr(toddler_module, name, MagicMock()) toddler_obj = toddler_cls() + # disable the cache + cache.configure(backend="dogpile.cache.null", replace_existing_backend=True) return toddler_obj diff --git a/tests/utils/test_pagure.py b/tests/utils/test_pagure.py index 5f733f5..17e3f33 100644 --- a/tests/utils/test_pagure.py +++ b/tests/utils/test_pagure.py @@ -8,6 +8,7 @@ from unittest.mock import call, MagicMock, Mock, patch import pytest from toddlers.exceptions import PagureError +from toddlers.utils.cache import cache import toddlers.utils.pagure as pagure @@ -1456,6 +1457,39 @@ class TestPagureAssignMaintainerToProject: headers=self.pagure.get_auth_header(), ) + def test_assign_orphan_to_project(self, monkeypatch): + """ + Assigning the orphan user as a maintainer should invalidate the cache + """ + response_mock = Mock() + response_mock.status_code = 200 + + self.pagure._requests_session.patch.return_value = response_mock + monkeypatch.setattr(cache, "delete", Mock(), raising=True) + + namespace = "namespace" + repo = "repo" + + self.pagure.assign_maintainer_to_project(namespace, repo, "orphan") + self.pagure._requests_session.patch.assert_called() + cache.delete.assert_called_once_with( + "toddlers.utils.pagure:is_project_orphaned|namespace repo" + ) + + def test_assign_orphan_to_project_failure(self, monkeypatch): + """ + Failure to assign the orphan user as a maintainer should not invalidate the cache + """ + response_mock = Mock() + response_mock.status_code = 500 + + self.pagure._requests_session.patch.side_effect = response_mock + monkeypatch.setattr(cache, "delete", Mock(), raising=True) + + with pytest.raises(PagureError): + self.pagure.assign_maintainer_to_project("namespace", "repo", "orphan") + cache.delete.assert_not_called() + class TestPagureAddMemberToGroup: """ @@ -1937,7 +1971,7 @@ class TestPagureOrphanPackage: self.pagure = pagure.set_pagure(config) self.pagure._requests_session = Mock() - def test_orphan_package(self): + def test_orphan_package(self, monkeypatch): """ Assert that orphaning package is processed correctly. """ @@ -1946,6 +1980,8 @@ class TestPagureOrphanPackage: self.pagure._requests_session.post.return_value = response_mock + monkeypatch.setattr(cache, "delete", Mock(), raising=True) + namespace = "rpms" package = "foo" reason = "reason" @@ -1958,8 +1994,11 @@ class TestPagureOrphanPackage: data=json.dumps({"orphan_reason": reason, "orphan_reason_info": info}), headers=self.pagure.get_auth_header(), ) + cache.delete.assert_called_once_with( + "toddlers.utils.pagure:is_project_orphaned|rpms foo" + ) - def test_orphan_package_failure(self): + def test_orphan_package_failure(self, monkeypatch): """ Assert that failing to orphan package is handled correctly. """ @@ -1968,6 +2007,8 @@ class TestPagureOrphanPackage: self.pagure._requests_session.post.return_value = response_mock + monkeypatch.setattr(cache, "delete", Mock(), raising=True) + namespace = "rpms" package = "foo" reason = "reason" @@ -1983,6 +2024,7 @@ class TestPagureOrphanPackage: data=json.dumps({"orphan_reason": reason, "orphan_reason_info": info}), headers=self.pagure.get_auth_header(), ) + cache.delete.assert_not_called() class TestSetBugzillaOverrides: @@ -2058,6 +2100,29 @@ class TestSetBugzillaOverrides: headers=self.pagure.get_auth_header(), ) + def test_set_bugzilla_overrides_orphan(self, monkeypatch): + """ + Setting the orphan user as bugzilla maintainer should invalidate the cache. + """ + response_mock = MagicMock() + response_mock.ok = True + + self.pagure._requests_session.post.return_value = response_mock + monkeypatch.setattr(cache, "delete", Mock(), raising=True) + + namespace = "rpms" + package = "foo" + + for distro in ("fedora", "epel"): + kwargs = {f"{distro}_assignee": "orphan"} + self.pagure.set_bugzilla_overrides(namespace, package, **kwargs) + + self.pagure._requests_session.post.call_count == 2 + assert cache.delete.call_count == 2 + cache.delete.assert_called_with( + "toddlers.utils.pagure:is_project_orphaned|rpms foo" + ) + class TestModifyACLs: """ diff --git a/toddlers.toml.example b/toddlers.toml.example index baab58b..f66ef9d 100644 --- a/toddlers.toml.example +++ b/toddlers.toml.example @@ -45,6 +45,15 @@ routing_keys = ["#"] # This is dynamically generated in the code # more of them. blocked_toddlers = ["debug"] +# Caching. The default is in-memory caching, here is an example for memcached. +# [consumer_config.cache] +# backend = "dogpile.cache.pylibmc" +# expiration_time = 3600 +# [consumer_config.cache.arguments] +# url = ["127.0.0.1"] +# binary = true +# behaviors = {"tcp_nodelay": true, "ketama": true} + [consumer_config.default] # Configuration common to all toddlers. # diff --git a/toddlers/base.py b/toddlers/base.py index adab4fd..a221707 100644 --- a/toddlers/base.py +++ b/toddlers/base.py @@ -4,6 +4,7 @@ import logging import fedora_messaging.config import fedora_messaging.exceptions +from .utils.cache import set_cache from .utils.misc import merge_dicts @@ -25,6 +26,7 @@ class ToddlerBase(metaclass=abc.ABCMeta): def __init__(self): self.reset_routing_keys() self._config = _get_toddler_config(self.name) + set_cache(fedora_messaging.config.conf["consumer_config"].get("cache", {})) def reset_routing_keys(self): # Only for dev/debug: in Fedora infra all bindings are set in ansible diff --git a/toddlers/utils/cache.py b/toddlers/utils/cache.py new file mode 100644 index 0000000..6b3908e --- /dev/null +++ b/toddlers/utils/cache.py @@ -0,0 +1,18 @@ +""" +This module contains the caching system. +""" + +from dogpile.cache import exception, make_region +from dogpile.cache.util import kwarg_function_key_generator + +cache = make_region(function_key_generator=kwarg_function_key_generator) + + +def set_cache(conf: dict): + # We use memory_pickle instead of memory to catch attempts at caching + # non-serializable values. + conf.setdefault("backend", "dogpile.cache.memory_pickle") + try: + cache.configure(**conf) + except exception.RegionAlreadyConfigured: + pass diff --git a/toddlers/utils/pagure.py b/toddlers/utils/pagure.py index 72369e2..31f4323 100644 --- a/toddlers/utils/pagure.py +++ b/toddlers/utils/pagure.py @@ -22,6 +22,7 @@ from typing import Any, List, Optional from toddlers.exceptions import PagureError from toddlers.utils import requests +from toddlers.utils.cache import cache log = logging.getLogger(__name__) @@ -36,6 +37,9 @@ def set_pagure(conf: dict) -> Any: return Pagure(conf) +ORPHAN_USERNAME = "orphan" + + class Pagure: """ Object that is a wrapper above pagure API. @@ -685,6 +689,7 @@ class Pagure: return result + @cache.cache_on_arguments(expiration_time=3600) def get_branches(self, namespace: str, repo: str) -> Any: """ Return all branches for the specified repository. @@ -740,6 +745,7 @@ class Pagure: return result + @cache.cache_on_arguments(expiration_time=3600) def get_default_branch(self, namespace: str, repo: str) -> Any: """ Return the default branch for the specified repository. @@ -850,6 +856,7 @@ class Pagure: return result + @cache.cache_on_arguments(expiration_time=3600) def is_project_orphaned(self, namespace: str, repo: str) -> bool: """ Check if project is orphaned. @@ -865,8 +872,6 @@ class Pagure: Raises: `toddlers.utils.exceptions.PagureError``: When getting project maintainers fails. """ - keyword = "orphan" - endpoint_url = ( "https://src.fedoraproject.org/_dg/bzoverrides/" + namespace + "/" + repo ) @@ -878,8 +883,8 @@ class Pagure: if response.ok: result = response.json() if ( - result["epel_assignee"] == keyword - or result["fedora_assignee"] == keyword + result["epel_assignee"] == ORPHAN_USERNAME + or result["fedora_assignee"] == ORPHAN_USERNAME ): return True else: @@ -936,6 +941,8 @@ class Pagure: namespace, repo ) ) + if maintainer_fas == ORPHAN_USERNAME: + self.is_project_orphaned.invalidate(self, namespace, repo) # type: ignore[attr-defined] def add_member_to_group(self, user: str, group: str) -> None: """ @@ -1103,6 +1110,7 @@ class Pagure: return result + @cache.cache_on_arguments(expiration_time=7200) def get_retired_packages(self, branch: str) -> list: """ Retrieve list of retired packages in specific dist git branch. @@ -1282,13 +1290,14 @@ class Pagure: ) log.debug("Package '%s/%s' successfully orphaned.", namespace, package) + self.is_project_orphaned.invalidate(self, namespace, package) # type: ignore[attr-defined] def set_bugzilla_overrides( self, namespace: str, package: str, - fedora_assignee: Optional[str], - epel_assignee: Optional[str], + fedora_assignee: Optional[str] = None, + epel_assignee: Optional[str] = None, ) -> None: """ Set the bugzilla overrides for package to specific value. @@ -1358,6 +1367,12 @@ class Pagure: fedora_assignee, epel_assignee, ) + if fedora_assignee == ORPHAN_USERNAME or epel_assignee == ORPHAN_USERNAME: + self.is_project_orphaned.invalidate( # type: ignore[attr-defined] + self, + namespace, + package, + ) def modify_acls( self,