bodhi: Upgrade production to Bodhi 4.0.0.
Signed-off-by: Randy Barlow <randy@electronsweatshop.com>
This commit is contained in:
parent
73a51e7fa4
commit
4cf1624c76
19 changed files with 9 additions and 478 deletions
|
@ -15,15 +15,6 @@ virt_install_command: "{{ virt_install_command_two_nic }}"
|
|||
# Do not use testing repositories on production
|
||||
testing: False
|
||||
|
||||
# These are for fedmsg publication from the bodhi backend.
|
||||
# If you change these iptables rules, you also need to changes the endpoints
|
||||
# list in roles/fedmsg/base/templates/endpoints-bodhi.py
|
||||
tcp_ports: [
|
||||
3000, 3001, 3002, 3003, 3004,
|
||||
3005, 3006, 3007, 3008, 3009,
|
||||
3010, 3011, 3012, 3013, 3014,
|
||||
3015, 3016, 3017, 3018, 3019,
|
||||
]
|
||||
# Make connections from signing bridges stateless, they break sigul connections
|
||||
# https://bugzilla.redhat.com/show_bug.cgi?id=1283364
|
||||
custom_rules: ['-A INPUT --proto tcp --sport 44334 --source 10.5.125.71 -j ACCEPT']
|
||||
|
@ -34,9 +25,10 @@ nrpe_procs_crit: 1000
|
|||
|
||||
host_group: bodhi2
|
||||
|
||||
# These people get told when something goes wrong.
|
||||
fedmsg_error_recipients:
|
||||
- bodhiadmin-members@fedoraproject.org
|
||||
bodhi_message_queue_name: "bodhi{{ env_suffix }}_composer"
|
||||
# Define the topics that our fedora-messaging queue should be subscribed to.
|
||||
bodhi_message_routing_keys:
|
||||
- "org.fedoraproject.*.bodhi.composer.start"
|
||||
|
||||
## XXX -- note that the fedmsg_certs declaration does not happen here, but
|
||||
# happens instead at the inventory/host_vars/ level since bodhi-backend03 and
|
||||
|
|
|
@ -48,9 +48,6 @@
|
|||
nfs_src_dir: 'fedora_ftp/fedora.redhat.com/pub/'
|
||||
|
||||
- bodhi2/backend
|
||||
# We can eliminate this block once we deploy bodhi 4.0.0 to production
|
||||
- role: fedmsg/base
|
||||
when: env == "production"
|
||||
- role: collectd/fedmsg-service
|
||||
process: fedmsg-hub
|
||||
user: masher
|
||||
|
@ -74,16 +71,12 @@
|
|||
cert_src: "{{private}}/files/docker-registry/{{env}}/pki/issued/containerstable.crt"
|
||||
key_src: "{{private}}/files/docker-registry/{{env}}/pki/private/containerstable.key"
|
||||
certs_group: apache
|
||||
# We can remove the when statement when Bodhi 4.0.0 is deployed to production.
|
||||
- role: rabbit/user
|
||||
username: "bodhi{{ env_suffix }}"
|
||||
when: env == "staging"
|
||||
# We can remove the when statement when Bodhi 4.0.0 is deployed to production.
|
||||
- role: rabbit/queue
|
||||
username: "bodhi{{ env_suffix }}"
|
||||
queue_name: "{{ bodhi_message_queue_name }}"
|
||||
routing_keys: "{{ bodhi_message_routing_keys }}"
|
||||
when: env == "staging"
|
||||
|
||||
|
||||
tasks:
|
||||
|
|
|
@ -101,15 +101,10 @@
|
|||
- bodhi2/backend
|
||||
|
||||
tasks:
|
||||
# The when condition on this block can be dropped once we deploy bodhi 4.0.0 to production.
|
||||
- name: Stop the fedora-messaging backend
|
||||
service:
|
||||
name: fm-consumer@config.service
|
||||
state: stopped
|
||||
when: env == "staging"
|
||||
# This block can be dropped once we deploy bodhi 4.0.0 to production.
|
||||
- name: Stop the fedmsg-hub-3 backend
|
||||
service: name="fedmsg-hub-3" state=stopped
|
||||
|
||||
- name: Upgrade the database
|
||||
command: /usr/bin/alembic-3 -c /etc/bodhi/alembic.ini upgrade head
|
||||
|
@ -117,16 +112,10 @@
|
|||
chdir: /usr/share/bodhi/
|
||||
when: inventory_hostname.startswith(('bodhi-backend01.phx2', 'bodhi-backend01.stg.phx2'))
|
||||
|
||||
# The when condition on this block can be dropped once we deploy bodhi 4.0.0 to production.
|
||||
- name: Start the fedora-messaging backend
|
||||
service:
|
||||
name: fm-consumer@config.service
|
||||
state: started
|
||||
when: env == "staging"
|
||||
# This block can be dropped once we deploy bodhi 4.0.0 to production.
|
||||
- name: And... start the backend again
|
||||
service: name="fedmsg-hub-3" state=started
|
||||
when: env == "production"
|
||||
|
||||
post_tasks:
|
||||
- name: tell nagios to unshush w.r.t. the backend
|
||||
|
|
|
@ -19,16 +19,12 @@
|
|||
- include_vars: dir=/srv/web/infra/ansible/vars/all/ ignore_files=README
|
||||
|
||||
roles:
|
||||
# We can remove the when statement when Bodhi 4.0.0 is deployed to production.
|
||||
- role: rabbit/user
|
||||
username: "bodhi{{ env_suffix }}"
|
||||
when: env == "staging"
|
||||
# We can remove the when statement when Bodhi 4.0.0 is deployed to production.
|
||||
- role: rabbit/queue
|
||||
username: "bodhi{{ env_suffix }}"
|
||||
queue_name: "{{ bodhi_message_queue_name }}"
|
||||
routing_keys: "{{ bodhi_message_routing_keys }}"
|
||||
when: env == "staging"
|
||||
- role: openshift/project
|
||||
app: bodhi
|
||||
description: bodhi
|
||||
|
@ -44,39 +40,21 @@
|
|||
secret_name: bodhi-keytab
|
||||
service: bodhi
|
||||
host: "bodhi{{ env_suffix }}.fedoraproject.org"
|
||||
# When we deploy Bodhi 4.0.0 to production, we can drop the env staging statements on the
|
||||
# fedora-messaging items below.
|
||||
- role: openshift/secret-file
|
||||
app: bodhi
|
||||
secret_name: bodhi-fedora-messaging-ca
|
||||
key: cacert.pem
|
||||
privatefile: "rabbitmq/{{env}}/pki/ca.crt"
|
||||
when: env == "staging"
|
||||
- role: openshift/secret-file
|
||||
app: bodhi
|
||||
secret_name: bodhi-fedora-messaging-crt
|
||||
key: bodhi-cert.pem
|
||||
privatefile: "rabbitmq/{{env}}/pki/issued/bodhi{{env_suffix}}.crt"
|
||||
when: env == "staging"
|
||||
- role: openshift/secret-file
|
||||
app: bodhi
|
||||
secret_name: bodhi-fedora-messaging-key
|
||||
key: bodhi-key.pem
|
||||
privatefile: "rabbitmq/{{env}}/pki/private/bodhi{{env_suffix}}.key"
|
||||
when: env == "staging"
|
||||
# When we deploy Bodhi 4.0.0 to production, we can drop these fedmsg secrets.
|
||||
- role: openshift/secret-file
|
||||
app: bodhi
|
||||
secret_name: bodhi-fedmsg-key
|
||||
key: fedmsg-bodhi.key
|
||||
privatefile: fedmsg-certs/keys/bodhi-bodhi-web-temp-bodhi.app.os.fedoraproject.org.key
|
||||
when: env != "staging"
|
||||
- role: openshift/secret-file
|
||||
app: bodhi
|
||||
secret_name: bodhi-fedmsg-crt
|
||||
key: fedmsg-bodhi.crt
|
||||
privatefile: fedmsg-certs/keys/bodhi-bodhi-web-temp-bodhi.app.os.fedoraproject.org.crt
|
||||
when: env != "staging"
|
||||
- role: openshift/imagestream
|
||||
app: bodhi
|
||||
imagename: bodhi-base
|
||||
|
@ -96,7 +74,7 @@
|
|||
app: bodhi
|
||||
template: buildconfig.yml
|
||||
objectname: buildconfig.yml
|
||||
bodhi_version: 3.14.0-1.fc29.infra
|
||||
bodhi_version: 4.0.0-1.fc29.infra
|
||||
when: env == "production"
|
||||
- role: openshift/start-build
|
||||
app: bodhi
|
||||
|
@ -127,10 +105,12 @@
|
|||
dcname: bodhi-consumer
|
||||
|
||||
post_tasks:
|
||||
- name: Scale up pods
|
||||
command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
|
||||
when: env == "production"
|
||||
- name: Scale up pods
|
||||
command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
|
||||
when: env == "production"
|
||||
# Add one of these for production when we deploy 4.0.0
|
||||
- name: Scale up pods
|
||||
command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
|
||||
when: env == "staging"
|
||||
|
|
|
@ -187,20 +187,6 @@
|
|||
- bodhi
|
||||
- cron
|
||||
|
||||
# This task can be dropped once Bodhi 4.0.0 is deployed to production.
|
||||
- name: Install bodhi-dequeue-stable cron job that moves updates from batched to stable
|
||||
cron:
|
||||
name: "bodhi-dequeue-stable"
|
||||
minute: "45"
|
||||
hour: "23"
|
||||
weekday: "*"
|
||||
job: "/usr/bin/bodhi-dequeue-stable && touch /var/lib/bodhi/dequeue-stable"
|
||||
user: "apache"
|
||||
when: inventory_hostname.startswith('bodhi-backend01.phx2') and env == "production"
|
||||
tags:
|
||||
- bodhi
|
||||
- cron
|
||||
|
||||
- name: put the koji sync listener script in place
|
||||
copy:
|
||||
src: koji-sync-listener.py
|
||||
|
@ -356,7 +342,6 @@
|
|||
- config
|
||||
- bodhi
|
||||
|
||||
# Remove the env == "staging" condition when we deploy Bodhi 4.0.0 to production.
|
||||
- name: Install fedora-messaging config
|
||||
template: >
|
||||
src="{{ roles_path }}/bodhi2/base/templates/fedora-messaging.toml.j2"
|
||||
|
@ -364,7 +349,7 @@
|
|||
owner=apache
|
||||
group=apache
|
||||
mode=0600
|
||||
when: inventory_hostname.startswith('bodhi-backend') and env == "staging"
|
||||
when: inventory_hostname.startswith('bodhi-backend')
|
||||
notify:
|
||||
- restart fedora-messaging
|
||||
tags:
|
||||
|
@ -376,31 +361,6 @@
|
|||
tags:
|
||||
- bodhi
|
||||
|
||||
# We can drop this once we deploy Bodhi 4.0.0 to production
|
||||
- name: create the /usr/lib/systemd/system/fedmsg-hub-3.service.d drop-in directory
|
||||
file: path=/usr/lib/systemd/system/fedmsg-hub-3.service.d state=directory mode=0755
|
||||
when: env == "production"
|
||||
tags:
|
||||
- config
|
||||
- bodhi
|
||||
|
||||
# We can drop this once we deploy Bodhi 4.0.0 to production
|
||||
- name: install a femdsg-hub-3.service drop-in to run it as the apache
|
||||
copy: >
|
||||
src="fedmsg-hub.conf"
|
||||
dest="/usr/lib/systemd/system/fedmsg-hub-3.service.d/fedmsg-hub-3.conf"
|
||||
owner=root
|
||||
group=root
|
||||
mode=0644
|
||||
when: env == "production"
|
||||
notify:
|
||||
- reload systemd
|
||||
- restart fedmsg-hub-3
|
||||
tags:
|
||||
- bodhi
|
||||
- config
|
||||
|
||||
# We can drop the when condition when we deploy Bodhi 4.0.0 to production
|
||||
- name: Configure fm-consumer@.service to run as apache
|
||||
copy:
|
||||
src: fm-consumer@.service
|
||||
|
@ -408,7 +368,6 @@
|
|||
owner: root
|
||||
group: root
|
||||
mode: 0644
|
||||
when: env == "staging"
|
||||
notify:
|
||||
- reload systemd
|
||||
- restart fedora-messaging
|
||||
|
@ -426,50 +385,11 @@
|
|||
- config
|
||||
- bodhi
|
||||
|
||||
# We can drop this block when we upgrade production to bodhi 4.0.0
|
||||
- name: have the apache own /var/cache/fedmsg because of course..
|
||||
file: >
|
||||
path="/var/cache/fedmsg"
|
||||
owner=apache
|
||||
group=apache
|
||||
state=directory
|
||||
when: inventory_hostname.startswith('bodhi-backend') and env == "production"
|
||||
tags:
|
||||
- config
|
||||
- bodhi
|
||||
|
||||
# We can drop this block when we upgrade production to bodhi 4.0.0
|
||||
- name: ensure that nrpe has rights to monitor us (dir), including setgid bit.
|
||||
file: >
|
||||
dest=/var/run/fedmsg
|
||||
mode=2775
|
||||
owner=apache
|
||||
group=nrpe
|
||||
state=directory
|
||||
when: env == "production"
|
||||
tags:
|
||||
- fedmsgmonitor
|
||||
- bodhi
|
||||
|
||||
# We can drop this block when we upgrade production to bodhi 4.0.0
|
||||
- name: ensure that nrpe has rights to monitor us (file)
|
||||
file: >
|
||||
dest=/var/run/fedmsg/monitoring-fedmsg-hub-3.socket
|
||||
mode=0775
|
||||
owner=apache
|
||||
group=nrpe
|
||||
state=file
|
||||
when: env == "production"
|
||||
tags:
|
||||
- fedmsgmonitor
|
||||
- bodhi
|
||||
|
||||
- name: ensure apache is disabled on the backend
|
||||
service: name=httpd enabled=no state=stopped
|
||||
tags:
|
||||
- bodhi
|
||||
|
||||
# The when statement should be dropped when we deploy Bodhi 4.0.0 to production.
|
||||
- name: Create /etc/pki/fedora-messaging
|
||||
file:
|
||||
dest: /etc/pki/fedora-messaging
|
||||
|
@ -480,7 +400,6 @@
|
|||
tags:
|
||||
- bodhi
|
||||
|
||||
# The when staging should be dropped when we deploy Bodhi 4.0.0 to production.
|
||||
- name: Deploy the fedora-messaging CA
|
||||
copy:
|
||||
src: "{{ private }}/files/rabbitmq/{{env}}/pki/ca.crt"
|
||||
|
@ -488,13 +407,11 @@
|
|||
mode: 0644
|
||||
owner: apache
|
||||
group: apache
|
||||
when: env == "staging"
|
||||
notify:
|
||||
- restart fedora-messaging
|
||||
tags:
|
||||
- bodhi
|
||||
|
||||
# The when staging should be dropped when we deploy Bodhi 4.0.0 to production.
|
||||
- name: Deploy the fedora-messaging cert
|
||||
copy:
|
||||
src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/bodhi{{env_suffix}}.crt"
|
||||
|
@ -502,13 +419,11 @@
|
|||
mode: 0644
|
||||
owner: apache
|
||||
group: apache
|
||||
when: env == "staging"
|
||||
notify:
|
||||
- restart fedora-messaging
|
||||
tags:
|
||||
- bodhi
|
||||
|
||||
# The when staging should be dropped when we deploy Bodhi 4.0.0 to production.
|
||||
- name: Deploy the fedora-messaging key
|
||||
copy:
|
||||
src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/bodhi{{env_suffix}}.key"
|
||||
|
@ -516,23 +431,13 @@
|
|||
mode: 0600
|
||||
owner: apache
|
||||
group: apache
|
||||
when: env == "staging"
|
||||
notify:
|
||||
- restart fedora-messaging
|
||||
tags:
|
||||
- bodhi
|
||||
|
||||
# When we deploy Bodhi 4.0.0 to production, we can drop the when statement.
|
||||
- name: ensure fedora-messaging is enabled and started on the backend
|
||||
service: name=fm-consumer@config.service enabled=yes state=started
|
||||
when: env == "staging"
|
||||
tags:
|
||||
- bodhi
|
||||
|
||||
# When we deploy Bodhi 4.0.0 to production, we can drop this block.
|
||||
- name: ensure fedmsg-hub-3 is enabled and started on the backend
|
||||
service: name=fedmsg-hub-3 enabled=yes state=started
|
||||
when: env == "production"
|
||||
tags:
|
||||
- bodhi
|
||||
|
||||
|
|
|
@ -28,28 +28,6 @@
|
|||
- config
|
||||
- bodhi
|
||||
|
||||
# When Bodhi 4.0.0 is deployed to production, we should drop this.
|
||||
- name: Copy some fedmsg configuration of our own for fedmsg-hub-3
|
||||
template: >
|
||||
src={{item}}
|
||||
dest=/etc/fedmsg.d/{{item}}
|
||||
owner=root
|
||||
group=root
|
||||
mode=0644
|
||||
with_items:
|
||||
- bodhi-logging.py
|
||||
- bodhi-threading.py
|
||||
- bodhi.py
|
||||
- masher.py
|
||||
- updates_handler.py
|
||||
- signed_handler.py
|
||||
when: env == 'production'
|
||||
notify:
|
||||
- restart fedmsg-hub-3
|
||||
tags:
|
||||
- config
|
||||
- bodhi
|
||||
|
||||
- name: setup /etc/pki/bodhi directory
|
||||
file: path=/etc/pki/bodhi owner=root group=root mode=0755 state=directory
|
||||
tags:
|
||||
|
|
|
@ -1,23 +0,0 @@
|
|||
# This should get merged smartly with the site-wide fedmsg.d/logging.py we have
|
||||
# installed everywhere.
|
||||
config = dict(
|
||||
logging=dict(
|
||||
loggers=dict(
|
||||
bodhi={
|
||||
"level": "DEBUG",
|
||||
"propagate": False,
|
||||
"handlers": ["console", "mailer"],
|
||||
},
|
||||
sqlalchemy={
|
||||
"level": "WARN",
|
||||
"propagate": False,
|
||||
"handlers": ["console", "mailer"],
|
||||
},
|
||||
root={
|
||||
"level": "INFO",
|
||||
"propagate": False,
|
||||
"handlers": ["console", "mailer"],
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
|
@ -1,15 +0,0 @@
|
|||
config = {
|
||||
# This sets uptwo master threads to handle incoming messages.
|
||||
# Each of those master consumer threads then can fork off many masher
|
||||
# threads to mash different repos.
|
||||
# If you crank up this number, you should also crank up:
|
||||
# - the iptables rules in inventory/group_vars/bodhi-backend
|
||||
# - the fedmsg endpoints in roles/fedmsg/base/templates/endpoints-bodhi.py
|
||||
{% if inventory_hostname.startswith('bodhi-backend02') %}
|
||||
# https://github.com/fedora-infra/bodhi/issues/795
|
||||
"moksha.workers_per_consumer": 1,
|
||||
{% else %}
|
||||
"moksha.workers_per_consumer": 2,
|
||||
{% endif %}
|
||||
"moksha.threadpool_size": 22,
|
||||
}
|
|
@ -1,6 +0,0 @@
|
|||
# This file is empty on purpose.
|
||||
#
|
||||
# The bodhi-server rpm ships an /etc/fedmsg.d/bodhi.py file by default that has
|
||||
# some nice local settings for testing, but which don't make sense for our
|
||||
# infrastructure. So... we overwrite it with this empty file.
|
||||
config = {}
|
|
@ -110,8 +110,6 @@ data:
|
|||
{{ load_file('logging.yaml') | indent }}
|
||||
production.ini: |-
|
||||
{{ load_file('production.ini.j2') | indent }}
|
||||
# We can remove this if statement once we deploy Bodhi 4.0.0 to production.
|
||||
{% if env == "staging" %}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
|
@ -122,35 +120,3 @@ metadata:
|
|||
data:
|
||||
config.toml: |-
|
||||
{{ load_file('fedora-messaging.toml.j2') | indent }}
|
||||
{% endif %}
|
||||
# We can drop the next ConfigMap once Bodhi 4.0.0 is deployed to production.
|
||||
{% if env == "production" %}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: fedmsg-configmap
|
||||
labels:
|
||||
app: bodhi
|
||||
data:
|
||||
bodhi-openshift-web.py: |-
|
||||
import socket
|
||||
hostname = socket.gethostname().split('.', 1)[0]
|
||||
config = {
|
||||
'sign_messages': True,
|
||||
'active': True,
|
||||
'cert_prefix': 'bodhi',
|
||||
'certnames': {'bodhi.{}'.format(socket.gethostname()): 'bodhi'},
|
||||
'relay_inbound': ['tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941'],
|
||||
'bodhi.%s' % hostname: ['tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941'],
|
||||
{% if env == 'staging' %}
|
||||
'environment': 'stg',
|
||||
# stg should listen to the stg bus
|
||||
'endpoints': {
|
||||
'staging_gateway': ['tcp://stg.fedoraproject.org:9940']
|
||||
}
|
||||
{% else %}
|
||||
'environment': 'prod',
|
||||
{% endif %}
|
||||
}
|
||||
{% endif %}
|
||||
|
|
|
@ -1,17 +0,0 @@
|
|||
{% if env == 'staging' %}
|
||||
suffix = 'stg.phx2.fedoraproject.org'
|
||||
{% else %}
|
||||
suffix = 'phx2.fedoraproject.org'
|
||||
{% endif %}
|
||||
|
||||
config = dict(
|
||||
# Note, the masher runs on bodhi-backend01, while other consumers will run
|
||||
# on bodhi-backend02.
|
||||
masher={{bodhi_masher_enabled}},
|
||||
masher_topic='bodhi.masher.start',
|
||||
{% if ansible_hostname == 'bodhi-backend01' %}
|
||||
releng_fedmsg_certname='shell-bodhi-backend01.%s' % suffix,
|
||||
{% else %}
|
||||
releng_fedmsg_certname='shell-bodhi-backend03.%s' % suffix,
|
||||
{% endif %}
|
||||
)
|
|
@ -41,13 +41,6 @@ filter-with = proxy-prefix
|
|||
# libravatar.libravatar_url()'s https setting. Defaults to True.
|
||||
# libravatar_prefer_tls =
|
||||
|
||||
# We can remove this block once Bodhi 4.0.0 is deployed to production.
|
||||
{% if env != "staging" %}
|
||||
# Set this to True in order to send fedmsg messages.
|
||||
# fedmsg_enabled = False
|
||||
fedmsg_enabled = True
|
||||
{% endif %}
|
||||
|
||||
|
||||
##
|
||||
### Legal
|
||||
|
@ -145,13 +138,7 @@ message_id_email_domain = admin{{env_suffix}}.fedoraproject.org
|
|||
##
|
||||
## Masher settings
|
||||
##
|
||||
# We can remove this block once Bodhi 4.0.0 is deployed to production.
|
||||
{% if env != "staging" %}
|
||||
releng_fedmsg_certname = shell-bodhi-backend01{{env_suffix}}.phx2.fedoraproject.org
|
||||
{% endif %}
|
||||
|
||||
# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
|
||||
{% if env == "staging" %}
|
||||
# Where to initially mash repositories. You can use %(here)s to reference the location of this file.
|
||||
# compose_dir =
|
||||
{% if inventory_hostname.startswith('bodhi-backend') %}
|
||||
|
@ -160,33 +147,14 @@ compose_dir = /mnt/koji/compose/updates/
|
|||
# do not use on frontends as bodhi will check the mount and refuse to run without it.
|
||||
# compose_dir = /mnt/koji/compose/updates/
|
||||
{% endif %}
|
||||
{% else %}
|
||||
# Where to initially mash repositories. You can use %(here)s to reference the location of this file.
|
||||
# mash_dir =
|
||||
{% if inventory_hostname.startswith('bodhi-backend') %}
|
||||
mash_dir = /mnt/koji/compose/updates/
|
||||
{% else %}
|
||||
# do not use on frontends as bodhi will check the mount and refuse to run without it.
|
||||
#mash_dir = /mnt/koji/compose/updates/
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
|
||||
{% if env == "staging" %}
|
||||
# The max number of composer threads running at the same time
|
||||
# max_concurrent_composes = 2
|
||||
max_concurrent_composes = 3
|
||||
{% else %}
|
||||
# The max number of mash threads running at the same time
|
||||
# max_concurrent_mashes = 2
|
||||
max_concurrent_mashes = 3
|
||||
{% endif %}
|
||||
|
||||
# Whether to clean old composes at the end of each run.
|
||||
clean_old_composes = false
|
||||
|
||||
# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
|
||||
{% if env == "staging" %}
|
||||
# Where to symlink the latest repos by their tag name. You can use %(here)s to reference the
|
||||
# location of this file.
|
||||
# compose_stage_dir =
|
||||
|
@ -196,17 +164,6 @@ compose_stage_dir = /mnt/koji/compose/updates/
|
|||
# do not use on frontends as bodhi will check the mount and refuse to run without it.
|
||||
# compose_stage_dir = /mnt/koji/compose/updates/
|
||||
{% endif %}
|
||||
{% else %}
|
||||
# Where to symlink the latest repos by their tag name. You can use %(here)s to reference the
|
||||
# location of this file.
|
||||
# mash_stage_dir =
|
||||
{% if inventory_hostname.startswith('bodhi-backend') %}
|
||||
mash_stage_dir = /mnt/koji/compose/updates/
|
||||
{% else %}
|
||||
# do not use on frontends as bodhi will check the mount and refuse to run without it.
|
||||
#mash_stage_dir = /mnt/koji/compose/updates/
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
# The following jinja2 template variables are available for use to customize the Pungi configs and
|
||||
# variants files to the Release and Updates:
|
||||
|
@ -720,12 +677,9 @@ pyramid.debug_notfound = false
|
|||
pyramid.debug_routematch = false
|
||||
pyramid.default_locale_name = en
|
||||
|
||||
# We should drop this if statement once we deploy Bodhi 4.0.0 to production.
|
||||
{% if env == "staging" %}
|
||||
pyramid.includes =
|
||||
pyramid_sawing
|
||||
pyramid_sawing.file = /etc/bodhi/logging.yaml
|
||||
{% endif %}
|
||||
|
||||
debugtoolbar.hosts = 127.0.0.1 ::1
|
||||
|
||||
|
@ -797,46 +751,3 @@ port = 6543
|
|||
[pshell]
|
||||
m = bodhi.server.models
|
||||
#db = bodhi.server.util.pshell_db
|
||||
|
||||
# We should remove the logging here once Bodhi 4.0.0 is deployed to production - it is configured in
|
||||
# /etc/bodhi/logging.yaml instead.
|
||||
{% if env == "production" %}
|
||||
# Begin logging configuration
|
||||
|
||||
[loggers]
|
||||
keys = root, bodhi, sqlalchemy
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = INFO
|
||||
handlers = console
|
||||
|
||||
[logger_bodhi]
|
||||
level = DEBUG
|
||||
handlers =
|
||||
qualname = bodhi
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
# "level = INFO" logs SQL queries.
|
||||
# "level = DEBUG" logs SQL queries and results.
|
||||
# "level = WARN" logs neither. (Recommended for production systems.)
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
|
||||
|
||||
# End logging configuration
|
||||
{% endif %}
|
||||
|
|
|
@ -1,3 +0,0 @@
|
|||
config = dict(
|
||||
signed_handler={{bodhi_signed_handler_enabled}},
|
||||
)
|
|
@ -1,5 +0,0 @@
|
|||
config = dict(
|
||||
# Note, the masher runs on bodhi-backend03, but this
|
||||
# runs on bodhi-backend02 (separation of concerns).
|
||||
updates_handler={{bodhi_updates_handler_enabled}},
|
||||
)
|
|
@ -10,8 +10,6 @@ command[check_fedmsg_cp_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_p
|
|||
command[check_fedmsg_cp_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py moksha-hub BugzillaConsumer MonitoringProducer
|
||||
command[check_fedmsg_cp_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer
|
||||
command[check_fedmsg_cp_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
|
||||
command[check_fedmsg_cp_bodhi_backend01_hub]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub-3 Masher MonitoringProducer
|
||||
command[check_fedmsg_cp_bodhi_backend02_hub]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub-3 UpdatesHandler MonitoringProducer
|
||||
command[check_fedmsg_cp_autocloud_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub AutoCloudConsumer MonitoringProducer
|
||||
command[check_fedmsg_cp_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer
|
||||
command[check_fedmsg_cp_bugyou_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugyouConsumer MonitoringProducer
|
||||
|
@ -29,8 +27,6 @@ command[check_fedmsg_cexceptions_notifs_backend]={{libdir}}/nagios/plugins/check
|
|||
command[check_fedmsg_cexceptions_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py moksha-hub BugzillaConsumer 1 10
|
||||
command[check_fedmsg_cexceptions_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10
|
||||
command[check_fedmsg_cexceptions_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
|
||||
command[check_fedmsg_cexceptions_bodhi_backend01_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub-3 Masher 1 10
|
||||
command[check_fedmsg_cexceptions_bodhi_backend02_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub-3 UpdatesHandler 1 10
|
||||
command[check_fedmsg_cexceptions_autocloud_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub AutoCloudConsumer 1 10
|
||||
command[check_fedmsg_cexceptions_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10
|
||||
command[check_fedmsg_cexceptions_bugyou_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugyouConsumer 1 10
|
||||
|
@ -48,8 +44,6 @@ command[check_fedmsg_cbacklog_notifs_backend]={{libdir}}/nagios/plugins/check_fe
|
|||
command[check_fedmsg_cbacklog_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py moksha-hub BugzillaConsumer 10 100
|
||||
command[check_fedmsg_cbacklog_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000
|
||||
command[check_fedmsg_cbacklog_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 1000 5000
|
||||
command[check_fedmsg_cbacklog_bodhi_backend01_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub-3 Masher 500 1000
|
||||
command[check_fedmsg_cbacklog_bodhi_backend02_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub-3 UpdatesHandler 500 1000
|
||||
command[check_fedmsg_cbacklog_autocloud_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub AutoCloudConsumer 100 500
|
||||
command[check_fedmsg_cbacklog_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000
|
||||
command[check_fedmsg_cbacklog_bugyou_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugyouConsumer 5000 10000
|
||||
|
|
|
@ -109,20 +109,6 @@ define service {
|
|||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name bodhi-backend01.phx2.fedoraproject.org
|
||||
service_description Check for fedmsg-hub proc
|
||||
check_command check_by_nrpe!check_fedmsg_composer_proc
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name bodhi-backend02.phx2.fedoraproject.org
|
||||
service_description Check for fedmsg-hub proc
|
||||
check_command check_by_nrpe!check_fedmsg_composer_proc
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name packages03.phx2.fedoraproject.org
|
||||
service_description Check for fedmsg-hub proc
|
||||
|
@ -371,20 +357,6 @@ define service {
|
|||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name bodhi-backend01.phx2.fedoraproject.org
|
||||
service_description Check fedmsg consumers and producers hub
|
||||
check_command check_by_nrpe!check_fedmsg_cp_bodhi_backend01_hub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name bodhi-backend02.phx2.fedoraproject.org
|
||||
service_description Check fedmsg consumers and producers hub
|
||||
check_command check_by_nrpe!check_fedmsg_cp_bodhi_backend02_hub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name packages03.phx2.fedoraproject.org
|
||||
service_description Check fedmsg consumers and producers hub
|
||||
|
@ -468,20 +440,6 @@ define service {
|
|||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name bodhi-backend01.phx2.fedoraproject.org
|
||||
service_description Check fedmsg-hub consumers exceptions
|
||||
check_command check_by_nrpe!check_fedmsg_cexceptions_bodhi_backend01_hub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name bodhi-backend02.phx2.fedoraproject.org
|
||||
service_description Check fedmsg-hub consumers exceptions
|
||||
check_command check_by_nrpe!check_fedmsg_cexceptions_bodhi_backend02_hub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name packages03.phx2.fedoraproject.org
|
||||
service_description Check fedmsg-hub consumers exceptions
|
||||
|
@ -568,20 +526,6 @@ define service {
|
|||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name bodhi-backend01.phx2.fedoraproject.org
|
||||
service_description Check fedmsg-hub consumers backlog
|
||||
check_command check_by_nrpe!check_fedmsg_cbacklog_bodhi_backend01_hub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name bodhi-backend02.phx2.fedoraproject.org
|
||||
service_description Check fedmsg-hub consumers backlog
|
||||
check_command check_by_nrpe!check_fedmsg_cbacklog_bodhi_backend02_hub
|
||||
use defaulttemplate
|
||||
}
|
||||
|
||||
define service {
|
||||
host_name packages03.phx2.fedoraproject.org
|
||||
service_description Check fedmsg-hub consumers backlog
|
||||
|
|
|
@ -400,8 +400,6 @@ command[check_fedmsg_cp_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_p
|
|||
command[check_fedmsg_cp_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py moksha-hub BugzillaConsumer MonitoringProducer
|
||||
command[check_fedmsg_cp_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer
|
||||
command[check_fedmsg_cp_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
|
||||
command[check_fedmsg_cp_bodhi_backend01_hub]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub-3 Masher MonitoringProducer
|
||||
command[check_fedmsg_cp_bodhi_backend02_hub]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub-3 UpdatesHandler MonitoringProducer
|
||||
command[check_fedmsg_cp_autocloud_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub AutoCloudConsumer MonitoringProducer
|
||||
command[check_fedmsg_cp_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer
|
||||
command[check_fedmsg_cp_bugyou_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugyouConsumer MonitoringProducer
|
||||
|
@ -418,8 +416,6 @@ command[check_fedmsg_cexceptions_notifs_backend]=/usr/lib64/nagios/plugins/check
|
|||
command[check_fedmsg_cexceptions_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py moksha-hub BugzillaConsumer 1 10
|
||||
command[check_fedmsg_cexceptions_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10
|
||||
command[check_fedmsg_cexceptions_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
|
||||
command[check_fedmsg_cexceptions_bodhi_backend01_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub-3 Masher 1 10
|
||||
command[check_fedmsg_cexceptions_bodhi_backend02_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub-3 UpdatesHandler 1 10
|
||||
command[check_fedmsg_cexceptions_autocloud_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub AutoCloudConsumer 1 10
|
||||
command[check_fedmsg_cexceptions_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10
|
||||
command[check_fedmsg_cexceptions_bugyou_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugyouConsumer 1 10
|
||||
|
@ -436,8 +432,6 @@ command[check_fedmsg_cbacklog_notifs_backend]=/usr/lib64/nagios/plugins/check_fe
|
|||
command[check_fedmsg_cbacklog_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py moksha-hub BugzillaConsumer 10 100
|
||||
command[check_fedmsg_cbacklog_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000
|
||||
command[check_fedmsg_cbacklog_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 100 500
|
||||
command[check_fedmsg_cbacklog_bodhi_backend01_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub-3 Masher 500 1000
|
||||
command[check_fedmsg_cbacklog_bodhi_backend02_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub-3 UpdatesHandler 500 1000
|
||||
command[check_fedmsg_cbacklog_autocloud_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub AutoCloudConsumer 500 1000
|
||||
command[check_fedmsg_cbacklog_packages_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000
|
||||
command[check_fedmsg_cbacklog_bugyou_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugyouConsumer 5000 10000
|
||||
|
|
|
@ -22,17 +22,6 @@ items:
|
|||
# While dnf has a --nodocs, it doesen't have a --docs...
|
||||
RUN sed -i '/nodocs/d' /etc/dnf/dnf.conf
|
||||
RUN dnf install -y bodhi-server-{{bodhi_version}}
|
||||
# We can drop this block once Bodhi 4.0.0 is deployed to production.
|
||||
{% if env == "production" %}
|
||||
# Put the fedmsg config in place
|
||||
RUN rm -f /etc/fedmsg.d/bodhi.py && \
|
||||
# Let's name it starting with zz so it sorts last.
|
||||
ln -sf /etc/bodhi-fedmsg.d/bodhi-openshift-web.py /etc/fedmsg.d/zzbodhi.py && \
|
||||
# Put the fedmsg cert and key in place
|
||||
mkdir -p /etc/pki/fedmsg/ && \
|
||||
ln -sf /etc/pki/fedmsg/key/fedmsg-bodhi.key /etc/pki/fedmsg/bodhi.key && \
|
||||
ln -sf /etc/pki/fedmsg/crt/fedmsg-bodhi.crt /etc/pki/fedmsg/bodhi.crt
|
||||
{% endif %}
|
||||
# Set up krb5
|
||||
RUN rm -f /etc/krb5.conf && \
|
||||
ln -sf /etc/bodhi/krb5.conf /etc/krb5.conf && \
|
||||
|
@ -68,7 +57,6 @@ items:
|
|||
vendor="Fedora Infrastructure" \
|
||||
license="MIT"
|
||||
RUN dnf install -y bodhi-docs-{{bodhi_version}} python3-pyramid_sawing
|
||||
# We can drop this block once Bodhi 4.0.0 is deployed to production.
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT bash /etc/bodhi/start.sh
|
||||
type: Dockerfile
|
||||
|
@ -88,8 +76,6 @@ items:
|
|||
name: bodhi-web:latest
|
||||
kind: List
|
||||
metadata: {}
|
||||
{# We can drop the if statement here once we deploy 4.0.0 to production #}
|
||||
{% if env == "staging" %}
|
||||
---
|
||||
apiVersion: v1
|
||||
items:
|
||||
|
@ -126,5 +112,3 @@ items:
|
|||
name: bodhi-consumer:latest
|
||||
kind: List
|
||||
metadata: {}
|
||||
{# We can drop this once we deploy 4.0.0 to production #}
|
||||
{% endif %}
|
||||
|
|
|
@ -49,18 +49,6 @@ items:
|
|||
readOnly: true
|
||||
- name: httpdir-volume
|
||||
mountPath: /httpdir
|
||||
# We can remove this if/else once we deploy Bodhi 4.0.0 to production.
|
||||
{% if env == "production" %}
|
||||
- name: fedmsg-config-volume
|
||||
mountPath: /etc/bodhi-fedmsg.d
|
||||
readOnly: true
|
||||
- name: fedmsg-key-volume
|
||||
mountPath: /etc/pki/fedmsg/key
|
||||
readOnly: true
|
||||
- name: fedmsg-crt-volume
|
||||
mountPath: /etc/pki/fedmsg/crt
|
||||
readOnly: true
|
||||
{% else %}
|
||||
- name: fedora-messaging-config-volume
|
||||
mountPath: /etc/fedora-messaging
|
||||
readOnly: true
|
||||
|
@ -76,7 +64,6 @@ items:
|
|||
mountPath: /etc/pki/fedora-messaging/bodhi-key.pem
|
||||
subPath: bodhi-key.pem
|
||||
readOnly: true
|
||||
{% endif %}
|
||||
readinessProbe:
|
||||
timeoutSeconds: 10
|
||||
initialDelaySeconds: 5
|
||||
|
@ -98,8 +85,6 @@ items:
|
|||
secretName: bodhi-keytab
|
||||
- name: httpdir-volume
|
||||
emptyDir: {}
|
||||
# We can remove this if/else once we deploy Bodhi 4.0.0 to production.
|
||||
{% if env == "staging" %}
|
||||
- name: fedora-messaging-config-volume
|
||||
configMap:
|
||||
name: fedora-messaging-configmap
|
||||
|
@ -112,17 +97,6 @@ items:
|
|||
- name: fedora-messaging-key-volume
|
||||
secret:
|
||||
secretName: bodhi-fedora-messaging-key
|
||||
{% else %}
|
||||
- name: fedmsg-config-volume
|
||||
configMap:
|
||||
name: fedmsg-configmap
|
||||
- name: fedmsg-key-volume
|
||||
secret:
|
||||
secretName: bodhi-fedmsg-key
|
||||
- name: fedmsg-crt-volume
|
||||
secret:
|
||||
secretName: bodhi-fedmsg-crt
|
||||
{% endif %}
|
||||
triggers:
|
||||
- imageChangeParams:
|
||||
automatic: true
|
||||
|
@ -136,8 +110,6 @@ items:
|
|||
- type: ConfigChange
|
||||
kind: List
|
||||
metadata: {}
|
||||
# We can remove this if once we deploy Bodhi 4.0.0 to production.
|
||||
{% if env == "staging" %}
|
||||
---
|
||||
apiVersion: v1
|
||||
items:
|
||||
|
@ -233,5 +205,3 @@ items:
|
|||
- type: ConfigChange
|
||||
kind: List
|
||||
metadata: {}
|
||||
{# End of the if staging block for this consumer pod #}
|
||||
{% endif %}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue