Infrastructure/toddlers
3
0
Fork 0
Code Issues 10 Pull requests 1 Projects Releases Packages Wiki Activity Actions

Add a caching system based on dogpile

Browse source 
Cache some calls to Pagure/DistGit

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>
This commit is contained in:
Aurélien Bompard 2024-12-17 10:11:11 +01:00
parent 048ce44bbc
commit 1bfdebd297
No known key found for this signature in database
GPG key ID: 31584CFEB9BF64AD
9 changed files with 212 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

7
.zuul.yaml
View file

@ -12,6 +12,7 @@
- python3.12
- python3.12-devel
- poetry
- libmemcached-devel
- fi-tox-format:
vars:
tox_envlist: black
@ -23,6 +24,7 @@
- python3.12
- python3.12-devel
- poetry
- libmemcached-devel
- fi-tox-lint:
vars:
tox_envlist: flake8
@ -34,6 +36,7 @@
- python3.12
- python3.12-devel
- poetry
- libmemcached-devel
- fi-tox-python39:
vars:
dependencies:
@ -42,6 +45,7 @@
- gobject-introspection-devel
- libmodulemd
- poetry
- libmemcached-devel
- fi-tox-python310:
vars:
dependencies:
@ -56,6 +60,7 @@
- python3-py
- python3-toml
- poetry
- libmemcached-devel
- fi-tox-python311:
vars:
dependencies:
@ -70,6 +75,7 @@
- python3-py
- python3-toml
- poetry
- libmemcached-devel
- fi-tox-python312:
vars:
dependencies:
@ -84,4 +90,5 @@
- python3-py
- python3-toml
- poetry
- libmemcached-devel
...

80
poetry.lock generated
View file

@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
[[package]]
name = "arrow"
@ -608,6 +608,25 @@ Pygments = ">=2.9.0,<3.0.0"
[package.extras]
toml = ["tomli (>=1.2.1)"]
[[package]]
name = "dogpile-cache"
version = "1.3.3"
description = "A caching front-end based on the Dogpile lock."
optional = false
python-versions = ">=3.8"
files = [
{file = "dogpile.cache-1.3.3-py3-none-any.whl", hash = "sha256:5e211c4902ebdf88c678d268e22454b41e68071632daa9402d8ee24e825ed8ca"},
{file = "dogpile.cache-1.3.3.tar.gz", hash = "sha256:f84b8ed0b0fb297d151055447fa8dcaf7bae566d4dbdefecdcc1f37662ab588b"},
]
[package.dependencies]
decorator = ">=4.0.0"
stevedore = ">=3.0.0"
typing_extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""}
[package.extras]
pifpaf = ["pifpaf (>=2.5.0)", "setuptools"]
[[package]]
name = "exceptiongroup"
version = "1.2.1"
@ -1310,6 +1329,17 @@ files = [
{file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"},
]
[[package]]
name = "pbr"
version = "6.1.0"
description = "Python Build Reasonableness"
optional = false
python-versions = ">=2.6"
files = [
{file = "pbr-6.1.0-py2.py3-none-any.whl", hash = "sha256:a776ae228892d8013649c0aeccbb3d5f99ee15e005a4cbb7e61d55a067b28a2a"},
{file = "pbr-6.1.0.tar.gz", hash = "sha256:788183e382e3d1d7707db08978239965e8b9e4e5ed42669bf4758186734d5f24"},
]
[[package]]
name = "pdc-client"
version = "1.8.0"
@ -1484,6 +1514,38 @@ pycairo = ">=1.16"
dev = ["flake8", "pytest", "pytest-cov"]
docs = ["sphinx (>=4.0,<5.0)", "sphinx-rtd-theme (>=0.5,<2.0)"]
[[package]]
name = "pylibmc"
version = "1.6.3"
description = "Quick and small memcached client for Python"
optional = false
python-versions = ">=3.6"
files = [
{file = "pylibmc-1.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a7baf4b78720f8b72839b3642b374754cb77cf57dab465a70ed1764d943e19d5"},
{file = "pylibmc-1.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dd98054a571bd450200a61a12b9ada3424678d17a25456bbf9a6100470401e52"},
{file = "pylibmc-1.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e847cdc78d82964236599ff5b312bc97fde3d10f4b93c9ee17dc33b7cf3c032a"},
{file = "pylibmc-1.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f46b5aa0364bca5e02000f5d62eb408d834a20722ffaf7dae20f75e7d009e6c"},
{file = "pylibmc-1.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c35816082848723455071670770d89b5a531d40e9063fe4e942ea456f86da49"},
{file = "pylibmc-1.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9d2ff6d702eb5ae502e29d97772dc85c749d596c6cb8c82a5d18f175fd4eabcc"},
{file = "pylibmc-1.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e589b7e70dec4daf0da1216789713c753d85611d70cfcd32574161cc75b1527e"},
{file = "pylibmc-1.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b93e381dec1520a3fec922765e04679ac553d2f3fda830a5faa7cdc527280a2"},
{file = "pylibmc-1.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f2574f390a2ec89b52a84bccca3ae57c21a4bb4d0e72df210d0d66783eee7f98"},
{file = "pylibmc-1.6.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db8c0f0467182a2a3e8d625b5c60c296f971dd2ee179e865b0262bd44528d676"},
{file = "pylibmc-1.6.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b34c1e4021b9a395950be19ea9d98f02bea0e3a88be26dbaa7e8ac4416e1232b"},
{file = "pylibmc-1.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6cce1d7705952eb30a3aca9ea3f054040cbee53c668d4e1e29144110da113bc"},
{file = "pylibmc-1.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6c4bdd8790aede67a464a32df842cacb562f77b0415a8c7823421f5c07524c6"},
{file = "pylibmc-1.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:218125aca214d62e6f69e4f8022bd795fbcb3643ad783f5f5ff33c23a1731c73"},
{file = "pylibmc-1.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f536d73632007358796654ab088d65c55a1a4368a85cfd7c956d2100e2cd8d89"},
{file = "pylibmc-1.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f2aeff000de7d918806876dfa4880d21b72089f9809ad0b8e7dff26501367ec6"},
{file = "pylibmc-1.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9ef3dc70ee2dfd0981bdf3a383a044bc591de7e445296a64a24f10a560e8b4f"},
{file = "pylibmc-1.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b649eb7fdd774290b2da73334456eb01e0d66e3d3685acd88ae6bf456a227dc6"},
{file = "pylibmc-1.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5251df82535411d8dc08c01141b8e6e61004f0a3ee50db3aa48ffa00e928cebb"},
{file = "pylibmc-1.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cd61f1ff46aa1ca6b0b3dac17a727cd29ac019e85db868c5523c491eef4459d7"},
{file = "pylibmc-1.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7660c561e5415f4be01ff4791c1b035359c1d76fed012e18eee907c2d3249deb"},
{file = "pylibmc-1.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f4516a14b2beff6062d1d240c00098227ca5478c00afba7e8b329415b0d4d67"},
{file = "pylibmc-1.6.3.tar.gz", hash = "sha256:eefa46115537abad65fbe2e032acd1b3463d9bf9e335af4b0916df4e4d3206e0"},
]
[[package]]
name = "pyopenssl"
version = "24.1.0"
@ -2105,6 +2167,20 @@ files = [
{file = "sspilib-0.1.0.tar.gz", hash = "sha256:58b5291553cf6220549c0f855e0e6973f4977375d8236ce47bb581efb3e9b1cf"},
]
[[package]]
name = "stevedore"
version = "5.4.0"
description = "Manage dynamic plugins for Python applications"
optional = false
python-versions = ">=3.9"
files = [
{file = "stevedore-5.4.0-py3-none-any.whl", hash = "sha256:b0be3c4748b3ea7b854b265dcb4caa891015e442416422be16f8b31756107857"},
{file = "stevedore-5.4.0.tar.gz", hash = "sha256:79e92235ecb828fe952b6b8b0c6c87863248631922c8e8e0fa5b17b232c4514d"},
]
[package.dependencies]
pbr = ">=2.0.0"
[[package]]
name = "swagger-spec-validator"
version = "3.0.4"
@ -2563,4 +2639,4 @@ cffi = ["cffi (>=1.11)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.9"
content-hash = "01ee8d5a7e87f9c7475416ba933304d0279d9db723177d1a149d6d361e9b1773"
content-hash = "a49a574d1aad08af2f31b7af7979b211e7bb31d662027e16f59bfd381fa96c38"

3
pyproject.toml
View file

@ -25,7 +25,8 @@ python-fedora = "^1.1.1"
python-bugzilla = ">=3.2.0"
pdc-client = "^1.8.0"
zstandard = "^0.23.0"
dogpile-cache = "^1.3.3"
pylibmc = "^1.6.3"
[tool.poetry.group.dev.dependencies]
pytest = "^8.2.2"

8
tests/conftest.py
View file

@ -5,9 +5,15 @@ from unittest.mock import MagicMock
import pytest
from toddlers.utils.cache import cache
from toddlers.utils.requests import make_session
@pytest.fixture(autouse=True)
def disable_cache():
cache.configure(backend="dogpile.cache.null", replace_existing_backend=True)
@pytest.fixture
def toddler(request, monkeypatch):
"""Fixture creating a toddler for a class testing it
@ -30,6 +36,8 @@ def toddler(request, monkeypatch):
monkeypatch.setattr(toddler_module, name, MagicMock())
toddler_obj = toddler_cls()
# disable the cache
cache.configure(backend="dogpile.cache.null", replace_existing_backend=True)
return toddler_obj

69
tests/utils/test_pagure.py
View file

@ -8,6 +8,7 @@ from unittest.mock import call, MagicMock, Mock, patch
import pytest
from toddlers.exceptions import PagureError
from toddlers.utils.cache import cache
import toddlers.utils.pagure as pagure
@ -1456,6 +1457,39 @@ class TestPagureAssignMaintainerToProject:
headers=self.pagure.get_auth_header(),
)
def test_assign_orphan_to_project(self, monkeypatch):
"""
Assigning the orphan user as a maintainer should invalidate the cache
"""
response_mock = Mock()
response_mock.status_code = 200
self.pagure._requests_session.patch.return_value = response_mock
monkeypatch.setattr(cache, "delete", Mock(), raising=True)
namespace = "namespace"
repo = "repo"
self.pagure.assign_maintainer_to_project(namespace, repo, "orphan")
self.pagure._requests_session.patch.assert_called()
cache.delete.assert_called_once_with(
"toddlers.utils.pagure:is_project_orphaned|namespace repo"
)
def test_assign_orphan_to_project_failure(self, monkeypatch):
"""
Failure to assign the orphan user as a maintainer should not invalidate the cache
"""
response_mock = Mock()
response_mock.status_code = 500
self.pagure._requests_session.patch.side_effect = response_mock
monkeypatch.setattr(cache, "delete", Mock(), raising=True)
with pytest.raises(PagureError):
self.pagure.assign_maintainer_to_project("namespace", "repo", "orphan")
cache.delete.assert_not_called()
class TestPagureAddMemberToGroup:
"""
@ -1937,7 +1971,7 @@ class TestPagureOrphanPackage:
self.pagure = pagure.set_pagure(config)
self.pagure._requests_session = Mock()
def test_orphan_package(self):
def test_orphan_package(self, monkeypatch):
"""
Assert that orphaning package is processed correctly.
"""
@ -1946,6 +1980,8 @@ class TestPagureOrphanPackage:
self.pagure._requests_session.post.return_value = response_mock
monkeypatch.setattr(cache, "delete", Mock(), raising=True)
namespace = "rpms"
package = "foo"
reason = "reason"
@ -1958,8 +1994,11 @@ class TestPagureOrphanPackage:
data=json.dumps({"orphan_reason": reason, "orphan_reason_info": info}),
headers=self.pagure.get_auth_header(),
)
cache.delete.assert_called_once_with(
"toddlers.utils.pagure:is_project_orphaned|rpms foo"
)
def test_orphan_package_failure(self):
def test_orphan_package_failure(self, monkeypatch):
"""
Assert that failing to orphan package is handled correctly.
"""
@ -1968,6 +2007,8 @@ class TestPagureOrphanPackage:
self.pagure._requests_session.post.return_value = response_mock
monkeypatch.setattr(cache, "delete", Mock(), raising=True)
namespace = "rpms"
package = "foo"
reason = "reason"
@ -1983,6 +2024,7 @@ class TestPagureOrphanPackage:
data=json.dumps({"orphan_reason": reason, "orphan_reason_info": info}),
headers=self.pagure.get_auth_header(),
)
cache.delete.assert_not_called()
class TestSetBugzillaOverrides:
@ -2058,6 +2100,29 @@ class TestSetBugzillaOverrides:
headers=self.pagure.get_auth_header(),
)
def test_set_bugzilla_overrides_orphan(self, monkeypatch):
"""
Setting the orphan user as bugzilla maintainer should invalidate the cache.
"""
response_mock = MagicMock()
response_mock.ok = True
self.pagure._requests_session.post.return_value = response_mock
monkeypatch.setattr(cache, "delete", Mock(), raising=True)
namespace = "rpms"
package = "foo"
for distro in ("fedora", "epel"):
kwargs = {f"{distro}_assignee": "orphan"}
self.pagure.set_bugzilla_overrides(namespace, package, **kwargs)
self.pagure._requests_session.post.call_count == 2
assert cache.delete.call_count == 2
cache.delete.assert_called_with(
"toddlers.utils.pagure:is_project_orphaned|rpms foo"
)
class TestModifyACLs:
"""

9
toddlers.toml.example
View file

@ -45,6 +45,15 @@ routing_keys = ["#"] # This is dynamically generated in the code
# more of them.
blocked_toddlers = ["debug"]
# Caching. The default is in-memory caching, here is an example for memcached.
# [consumer_config.cache]
# backend = "dogpile.cache.pylibmc"
# expiration_time = 3600
# [consumer_config.cache.arguments]
# url = ["127.0.0.1"]
# binary = true
# behaviors = {"tcp_nodelay": true, "ketama": true}
[consumer_config.default]
# Configuration common to all toddlers.
#

2
toddlers/base.py
View file

@ -4,6 +4,7 @@ import logging
import fedora_messaging.config
import fedora_messaging.exceptions
from .utils.cache import set_cache
from .utils.misc import merge_dicts
@ -25,6 +26,7 @@ class ToddlerBase(metaclass=abc.ABCMeta):
def __init__(self):
self.reset_routing_keys()
self._config = _get_toddler_config(self.name)
set_cache(fedora_messaging.config.conf["consumer_config"].get("cache", {}))
def reset_routing_keys(self):
# Only for dev/debug: in Fedora infra all bindings are set in ansible

18
toddlers/utils/cache.py Normal file
View file

@ -0,0 +1,18 @@
"""
This module contains the caching system.
"""
from dogpile.cache import exception, make_region
from dogpile.cache.util import kwarg_function_key_generator
cache = make_region(function_key_generator=kwarg_function_key_generator)
def set_cache(conf: dict):
# We use memory_pickle instead of memory to catch attempts at caching
# non-serializable values.
conf.setdefault("backend", "dogpile.cache.memory_pickle")
try:
cache.configure(**conf)
except exception.RegionAlreadyConfigured:
pass

27
toddlers/utils/pagure.py
View file

@ -22,6 +22,7 @@ from typing import Any, List, Optional
from toddlers.exceptions import PagureError
from toddlers.utils import requests
from toddlers.utils.cache import cache
log = logging.getLogger(__name__)
@ -36,6 +37,9 @@ def set_pagure(conf: dict) -> Any:
return Pagure(conf)
ORPHAN_USERNAME = "orphan"
class Pagure:
"""
Object that is a wrapper above pagure API.
@ -685,6 +689,7 @@ class Pagure:
return result
@cache.cache_on_arguments(expiration_time=3600)
def get_branches(self, namespace: str, repo: str) -> Any:
"""
Return all branches for the specified repository.
@ -740,6 +745,7 @@ class Pagure:
return result
@cache.cache_on_arguments(expiration_time=3600)
def get_default_branch(self, namespace: str, repo: str) -> Any:
"""
Return the default branch for the specified repository.
@ -850,6 +856,7 @@ class Pagure:
return result
@cache.cache_on_arguments(expiration_time=3600)
def is_project_orphaned(self, namespace: str, repo: str) -> bool:
"""
Check if project is orphaned.
@ -865,8 +872,6 @@ class Pagure:
Raises:
`toddlers.utils.exceptions.PagureError``: When getting project maintainers fails.
"""
keyword = "orphan"
endpoint_url = (
"https://src.fedoraproject.org/_dg/bzoverrides/" + namespace + "/" + repo
)
@ -878,8 +883,8 @@ class Pagure:
if response.ok:
result = response.json()
if (
result["epel_assignee"] == keyword
or result["fedora_assignee"] == keyword
result["epel_assignee"] == ORPHAN_USERNAME
or result["fedora_assignee"] == ORPHAN_USERNAME
):
return True
else:
@ -936,6 +941,8 @@ class Pagure:
namespace, repo
)
)
if maintainer_fas == ORPHAN_USERNAME:
self.is_project_orphaned.invalidate(self, namespace, repo) # type: ignore[attr-defined]
def add_member_to_group(self, user: str, group: str) -> None:
"""
@ -1103,6 +1110,7 @@ class Pagure:
return result
@cache.cache_on_arguments(expiration_time=7200)
def get_retired_packages(self, branch: str) -> list:
"""
Retrieve list of retired packages in specific dist git branch.
@ -1282,13 +1290,14 @@ class Pagure:
)
log.debug("Package '%s/%s' successfully orphaned.", namespace, package)
self.is_project_orphaned.invalidate(self, namespace, package) # type: ignore[attr-defined]
def set_bugzilla_overrides(
self,
namespace: str,
package: str,
fedora_assignee: Optional[str],
epel_assignee: Optional[str],
fedora_assignee: Optional[str] = None,
epel_assignee: Optional[str] = None,
) -> None:
"""
Set the bugzilla overrides for package to specific value.
@ -1358,6 +1367,12 @@ class Pagure:
fedora_assignee,
epel_assignee,
)
if fedora_assignee == ORPHAN_USERNAME or epel_assignee == ORPHAN_USERNAME:
self.is_project_orphaned.invalidate( # type: ignore[attr-defined]
self,
namespace,
package,
)
def modify_acls(
self,