Add a caching system based on dogpile

Cache some calls to Pagure/DistGit

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

.zuul.yaml

@@ -12,6 +12,7 @@
                 - python3.12
                 - python3.12-devel
                 - poetry
+                - libmemcached-devel
         - fi-tox-format:
             vars:
               tox_envlist: black
@@ -23,6 +24,7 @@
                 - python3.12
                 - python3.12-devel
                 - poetry
+                - libmemcached-devel
         - fi-tox-lint:
             vars:
               tox_envlist: flake8
@@ -34,6 +36,7 @@
                 - python3.12
                 - python3.12-devel
                 - poetry
+                - libmemcached-devel
         - fi-tox-python39:
             vars:
               dependencies:
@@ -42,6 +45,7 @@
                 - gobject-introspection-devel
                 - libmodulemd
                 - poetry
+                - libmemcached-devel
         - fi-tox-python310:
             vars:
               dependencies:
@@ -56,6 +60,7 @@
                 - python3-py
                 - python3-toml
                 - poetry
+                - libmemcached-devel
         - fi-tox-python311:
             vars:
               dependencies:
@@ -70,6 +75,7 @@
                 - python3-py
                 - python3-toml
                 - poetry
+                - libmemcached-devel
         - fi-tox-python312:
             vars:
               dependencies:
@@ -84,4 +90,5 @@
                 - python3-py
                 - python3-toml
                 - poetry
+                - libmemcached-devel
 ...

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
 
 [[package]]
 name = "arrow"
@@ -608,6 +608,25 @@ Pygments = ">=2.9.0,<3.0.0"
 [package.extras]
 toml = ["tomli (>=1.2.1)"]
 
+[[package]]
+name = "dogpile-cache"
+version = "1.3.3"
+description = "A caching front-end based on the Dogpile lock."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "dogpile.cache-1.3.3-py3-none-any.whl", hash = "sha256:5e211c4902ebdf88c678d268e22454b41e68071632daa9402d8ee24e825ed8ca"},
+    {file = "dogpile.cache-1.3.3.tar.gz", hash = "sha256:f84b8ed0b0fb297d151055447fa8dcaf7bae566d4dbdefecdcc1f37662ab588b"},
+]
+
+[package.dependencies]
+decorator = ">=4.0.0"
+stevedore = ">=3.0.0"
+typing_extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""}
+
+[package.extras]
+pifpaf = ["pifpaf (>=2.5.0)", "setuptools"]
+
 [[package]]
 name = "exceptiongroup"
 version = "1.2.1"
@@ -1310,6 +1329,17 @@ files = [
     {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"},
 ]
 
+[[package]]
+name = "pbr"
+version = "6.1.0"
+description = "Python Build Reasonableness"
+optional = false
+python-versions = ">=2.6"
+files = [
+    {file = "pbr-6.1.0-py2.py3-none-any.whl", hash = "sha256:a776ae228892d8013649c0aeccbb3d5f99ee15e005a4cbb7e61d55a067b28a2a"},
+    {file = "pbr-6.1.0.tar.gz", hash = "sha256:788183e382e3d1d7707db08978239965e8b9e4e5ed42669bf4758186734d5f24"},
+]
+
 [[package]]
 name = "pdc-client"
 version = "1.8.0"
@@ -1484,6 +1514,38 @@ pycairo = ">=1.16"
 dev = ["flake8", "pytest", "pytest-cov"]
 docs = ["sphinx (>=4.0,<5.0)", "sphinx-rtd-theme (>=0.5,<2.0)"]
 
+[[package]]
+name = "pylibmc"
+version = "1.6.3"
+description = "Quick and small memcached client for Python"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "pylibmc-1.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a7baf4b78720f8b72839b3642b374754cb77cf57dab465a70ed1764d943e19d5"},
+    {file = "pylibmc-1.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dd98054a571bd450200a61a12b9ada3424678d17a25456bbf9a6100470401e52"},
+    {file = "pylibmc-1.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e847cdc78d82964236599ff5b312bc97fde3d10f4b93c9ee17dc33b7cf3c032a"},
+    {file = "pylibmc-1.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f46b5aa0364bca5e02000f5d62eb408d834a20722ffaf7dae20f75e7d009e6c"},
+    {file = "pylibmc-1.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c35816082848723455071670770d89b5a531d40e9063fe4e942ea456f86da49"},
+    {file = "pylibmc-1.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9d2ff6d702eb5ae502e29d97772dc85c749d596c6cb8c82a5d18f175fd4eabcc"},
+    {file = "pylibmc-1.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e589b7e70dec4daf0da1216789713c753d85611d70cfcd32574161cc75b1527e"},
+    {file = "pylibmc-1.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b93e381dec1520a3fec922765e04679ac553d2f3fda830a5faa7cdc527280a2"},
+    {file = "pylibmc-1.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f2574f390a2ec89b52a84bccca3ae57c21a4bb4d0e72df210d0d66783eee7f98"},
+    {file = "pylibmc-1.6.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db8c0f0467182a2a3e8d625b5c60c296f971dd2ee179e865b0262bd44528d676"},
+    {file = "pylibmc-1.6.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b34c1e4021b9a395950be19ea9d98f02bea0e3a88be26dbaa7e8ac4416e1232b"},
+    {file = "pylibmc-1.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6cce1d7705952eb30a3aca9ea3f054040cbee53c668d4e1e29144110da113bc"},
+    {file = "pylibmc-1.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6c4bdd8790aede67a464a32df842cacb562f77b0415a8c7823421f5c07524c6"},
+    {file = "pylibmc-1.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:218125aca214d62e6f69e4f8022bd795fbcb3643ad783f5f5ff33c23a1731c73"},
+    {file = "pylibmc-1.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f536d73632007358796654ab088d65c55a1a4368a85cfd7c956d2100e2cd8d89"},
+    {file = "pylibmc-1.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f2aeff000de7d918806876dfa4880d21b72089f9809ad0b8e7dff26501367ec6"},
+    {file = "pylibmc-1.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9ef3dc70ee2dfd0981bdf3a383a044bc591de7e445296a64a24f10a560e8b4f"},
+    {file = "pylibmc-1.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b649eb7fdd774290b2da73334456eb01e0d66e3d3685acd88ae6bf456a227dc6"},
+    {file = "pylibmc-1.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5251df82535411d8dc08c01141b8e6e61004f0a3ee50db3aa48ffa00e928cebb"},
+    {file = "pylibmc-1.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cd61f1ff46aa1ca6b0b3dac17a727cd29ac019e85db868c5523c491eef4459d7"},
+    {file = "pylibmc-1.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7660c561e5415f4be01ff4791c1b035359c1d76fed012e18eee907c2d3249deb"},
+    {file = "pylibmc-1.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f4516a14b2beff6062d1d240c00098227ca5478c00afba7e8b329415b0d4d67"},
+    {file = "pylibmc-1.6.3.tar.gz", hash = "sha256:eefa46115537abad65fbe2e032acd1b3463d9bf9e335af4b0916df4e4d3206e0"},
+]
+
 [[package]]
 name = "pyopenssl"
 version = "24.1.0"
@@ -2105,6 +2167,20 @@ files = [
     {file = "sspilib-0.1.0.tar.gz", hash = "sha256:58b5291553cf6220549c0f855e0e6973f4977375d8236ce47bb581efb3e9b1cf"},
 ]
 
+[[package]]
+name = "stevedore"
+version = "5.4.0"
+description = "Manage dynamic plugins for Python applications"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "stevedore-5.4.0-py3-none-any.whl", hash = "sha256:b0be3c4748b3ea7b854b265dcb4caa891015e442416422be16f8b31756107857"},
+    {file = "stevedore-5.4.0.tar.gz", hash = "sha256:79e92235ecb828fe952b6b8b0c6c87863248631922c8e8e0fa5b17b232c4514d"},
+]
+
+[package.dependencies]
+pbr = ">=2.0.0"
+
 [[package]]
 name = "swagger-spec-validator"
 version = "3.0.4"
@@ -2563,4 +2639,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "01ee8d5a7e87f9c7475416ba933304d0279d9db723177d1a149d6d361e9b1773"
+content-hash = "a49a574d1aad08af2f31b7af7979b211e7bb31d662027e16f59bfd381fa96c38"

pyproject.toml

@@ -25,7 +25,8 @@ python-fedora = "^1.1.1"
 python-bugzilla = ">=3.2.0"
 pdc-client = "^1.8.0"
 zstandard = "^0.23.0"
-
+dogpile-cache = "^1.3.3"
+pylibmc = "^1.6.3"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.2"

tests/conftest.py

@@ -5,9 +5,15 @@ from unittest.mock import MagicMock
 
 import pytest
 
+from toddlers.utils.cache import cache
 from toddlers.utils.requests import make_session
 
 
+@pytest.fixture(autouse=True)
+def disable_cache():
+    cache.configure(backend="dogpile.cache.null", replace_existing_backend=True)
+
+
 @pytest.fixture
 def toddler(request, monkeypatch):
     """Fixture creating a toddler for a class testing it
@@ -30,6 +36,8 @@ def toddler(request, monkeypatch):
             monkeypatch.setattr(toddler_module, name, MagicMock())
 
     toddler_obj = toddler_cls()
+    # disable the cache
+    cache.configure(backend="dogpile.cache.null", replace_existing_backend=True)
     return toddler_obj
 
 

tests/utils/test_pagure.py

@@ -8,6 +8,7 @@ from unittest.mock import call, MagicMock, Mock, patch
 import pytest
 
 from toddlers.exceptions import PagureError
+from toddlers.utils.cache import cache
 import toddlers.utils.pagure as pagure
 
 
@@ -1456,6 +1457,39 @@ class TestPagureAssignMaintainerToProject:
             headers=self.pagure.get_auth_header(),
         )
 
+    def test_assign_orphan_to_project(self, monkeypatch):
+        """
+        Assigning the orphan user as a maintainer should invalidate the cache
+        """
+        response_mock = Mock()
+        response_mock.status_code = 200
+
+        self.pagure._requests_session.patch.return_value = response_mock
+        monkeypatch.setattr(cache, "delete", Mock(), raising=True)
+
+        namespace = "namespace"
+        repo = "repo"
+
+        self.pagure.assign_maintainer_to_project(namespace, repo, "orphan")
+        self.pagure._requests_session.patch.assert_called()
+        cache.delete.assert_called_once_with(
+            "toddlers.utils.pagure:is_project_orphaned|namespace repo"
+        )
+
+    def test_assign_orphan_to_project_failure(self, monkeypatch):
+        """
+        Failure to assign the orphan user as a maintainer should not invalidate the cache
+        """
+        response_mock = Mock()
+        response_mock.status_code = 500
+
+        self.pagure._requests_session.patch.side_effect = response_mock
+        monkeypatch.setattr(cache, "delete", Mock(), raising=True)
+
+        with pytest.raises(PagureError):
+            self.pagure.assign_maintainer_to_project("namespace", "repo", "orphan")
+        cache.delete.assert_not_called()
+
 
 class TestPagureAddMemberToGroup:
     """
@@ -1937,7 +1971,7 @@ class TestPagureOrphanPackage:
         self.pagure = pagure.set_pagure(config)
         self.pagure._requests_session = Mock()
 
-    def test_orphan_package(self):
+    def test_orphan_package(self, monkeypatch):
         """
         Assert that orphaning package is processed correctly.
         """
@@ -1946,6 +1980,8 @@ class TestPagureOrphanPackage:
 
         self.pagure._requests_session.post.return_value = response_mock
 
+        monkeypatch.setattr(cache, "delete", Mock(), raising=True)
+
         namespace = "rpms"
         package = "foo"
         reason = "reason"
@@ -1958,8 +1994,11 @@ class TestPagureOrphanPackage:
             data=json.dumps({"orphan_reason": reason, "orphan_reason_info": info}),
             headers=self.pagure.get_auth_header(),
         )
+        cache.delete.assert_called_once_with(
+            "toddlers.utils.pagure:is_project_orphaned|rpms foo"
+        )
 
-    def test_orphan_package_failure(self):
+    def test_orphan_package_failure(self, monkeypatch):
         """
         Assert that failing to orphan package is handled correctly.
         """
@@ -1968,6 +2007,8 @@ class TestPagureOrphanPackage:
 
         self.pagure._requests_session.post.return_value = response_mock
 
+        monkeypatch.setattr(cache, "delete", Mock(), raising=True)
+
         namespace = "rpms"
         package = "foo"
         reason = "reason"
@@ -1983,6 +2024,7 @@ class TestPagureOrphanPackage:
             data=json.dumps({"orphan_reason": reason, "orphan_reason_info": info}),
             headers=self.pagure.get_auth_header(),
         )
+        cache.delete.assert_not_called()
 
 
 class TestSetBugzillaOverrides:
@@ -2058,6 +2100,29 @@ class TestSetBugzillaOverrides:
             headers=self.pagure.get_auth_header(),
         )
 
+    def test_set_bugzilla_overrides_orphan(self, monkeypatch):
+        """
+        Setting the orphan user as bugzilla maintainer should invalidate the cache.
+        """
+        response_mock = MagicMock()
+        response_mock.ok = True
+
+        self.pagure._requests_session.post.return_value = response_mock
+        monkeypatch.setattr(cache, "delete", Mock(), raising=True)
+
+        namespace = "rpms"
+        package = "foo"
+
+        for distro in ("fedora", "epel"):
+            kwargs = {f"{distro}_assignee": "orphan"}
+            self.pagure.set_bugzilla_overrides(namespace, package, **kwargs)
+
+        self.pagure._requests_session.post.call_count == 2
+        assert cache.delete.call_count == 2
+        cache.delete.assert_called_with(
+            "toddlers.utils.pagure:is_project_orphaned|rpms foo"
+        )
+
 
 class TestModifyACLs:
     """

toddlers.toml.example

@@ -45,6 +45,15 @@ routing_keys = ["#"]  # This is dynamically generated in the code
 # more of them.
 blocked_toddlers = ["debug"]
 
+# Caching. The default is in-memory caching, here is an example for memcached.
+# [consumer_config.cache]
+# backend = "dogpile.cache.pylibmc"
+# expiration_time = 3600
+# [consumer_config.cache.arguments]
+# url = ["127.0.0.1"]
+# binary = true
+# behaviors = {"tcp_nodelay": true, "ketama": true}
+
 [consumer_config.default]
 # Configuration common to all toddlers.
 #

toddlers/base.py

@@ -4,6 +4,7 @@ import logging
 import fedora_messaging.config
 import fedora_messaging.exceptions
 
+from .utils.cache import set_cache
 from .utils.misc import merge_dicts
 
 
@@ -25,6 +26,7 @@ class ToddlerBase(metaclass=abc.ABCMeta):
     def __init__(self):
         self.reset_routing_keys()
         self._config = _get_toddler_config(self.name)
+        set_cache(fedora_messaging.config.conf["consumer_config"].get("cache", {}))
 
     def reset_routing_keys(self):
         # Only for dev/debug: in Fedora infra all bindings are set in ansible

toddlers/utils/cache.py

@@ -0,0 +1,18 @@
+"""
+This module contains the caching system.
+"""
+
+from dogpile.cache import exception, make_region
+from dogpile.cache.util import kwarg_function_key_generator
+
+cache = make_region(function_key_generator=kwarg_function_key_generator)
+
+
+def set_cache(conf: dict):
+    # We use memory_pickle instead of memory to catch attempts at caching
+    # non-serializable values.
+    conf.setdefault("backend", "dogpile.cache.memory_pickle")
+    try:
+        cache.configure(**conf)
+    except exception.RegionAlreadyConfigured:
+        pass

toddlers/utils/pagure.py

@@ -22,6 +22,7 @@ from typing import Any, List, Optional
 
 from toddlers.exceptions import PagureError
 from toddlers.utils import requests
+from toddlers.utils.cache import cache
 
 log = logging.getLogger(__name__)
 
@@ -36,6 +37,9 @@ def set_pagure(conf: dict) -> Any:
     return Pagure(conf)
 
 
+ORPHAN_USERNAME = "orphan"
+
+
 class Pagure:
     """
     Object that is a wrapper above pagure API.
@@ -685,6 +689,7 @@ class Pagure:
 
         return result
 
+    @cache.cache_on_arguments(expiration_time=3600)
     def get_branches(self, namespace: str, repo: str) -> Any:
         """
         Return all branches for the specified repository.
@@ -740,6 +745,7 @@ class Pagure:
 
         return result
 
+    @cache.cache_on_arguments(expiration_time=3600)
     def get_default_branch(self, namespace: str, repo: str) -> Any:
         """
         Return the default branch for the specified repository.
@@ -850,6 +856,7 @@ class Pagure:
 
         return result
 
+    @cache.cache_on_arguments(expiration_time=3600)
     def is_project_orphaned(self, namespace: str, repo: str) -> bool:
         """
         Check if project is orphaned.
@@ -865,8 +872,6 @@ class Pagure:
         Raises:
             `toddlers.utils.exceptions.PagureError``: When getting project maintainers fails.
         """
-        keyword = "orphan"
-
         endpoint_url = (
             "https://src.fedoraproject.org/_dg/bzoverrides/" + namespace + "/" + repo
         )
@@ -878,8 +883,8 @@ class Pagure:
         if response.ok:
             result = response.json()
             if (
-                result["epel_assignee"] == keyword
+                result["epel_assignee"] == ORPHAN_USERNAME
-                or result["fedora_assignee"] == keyword
+                or result["fedora_assignee"] == ORPHAN_USERNAME
             ):
                 return True
             else:
@@ -936,6 +941,8 @@ class Pagure:
                     namespace, repo
                 )
             )
+        if maintainer_fas == ORPHAN_USERNAME:
+            self.is_project_orphaned.invalidate(self, namespace, repo)  # type: ignore[attr-defined]
 
     def add_member_to_group(self, user: str, group: str) -> None:
         """
@@ -1103,6 +1110,7 @@ class Pagure:
 
         return result
 
+    @cache.cache_on_arguments(expiration_time=7200)
     def get_retired_packages(self, branch: str) -> list:
         """
         Retrieve list of retired packages in specific dist git branch.
@@ -1282,13 +1290,14 @@ class Pagure:
             )
 
         log.debug("Package '%s/%s' successfully orphaned.", namespace, package)
+        self.is_project_orphaned.invalidate(self, namespace, package)  # type: ignore[attr-defined]
 
     def set_bugzilla_overrides(
         self,
         namespace: str,
         package: str,
-        fedora_assignee: Optional[str],
+        fedora_assignee: Optional[str] = None,
-        epel_assignee: Optional[str],
+        epel_assignee: Optional[str] = None,
     ) -> None:
         """
         Set the bugzilla overrides for package to specific value.
@@ -1358,6 +1367,12 @@ class Pagure:
             fedora_assignee,
             epel_assignee,
         )
+        if fedora_assignee == ORPHAN_USERNAME or epel_assignee == ORPHAN_USERNAME:
+            self.is_project_orphaned.invalidate(  # type: ignore[attr-defined]
+                self,
+                namespace,
+                package,
+            )
 
     def modify_acls(
         self,