infrastructure/ansible
1
0
Fork 0
Code Pull requests 7 Activity Actions

fedimg: signing off...

Browse source 
Thanks for all the uploads fedimg.
You go to a far far better place I'm sure.

There's no point in keeping it around now, as it's actually not working
and the replacement ( cloud-image-uploader) should work soon.

Signed-off-by: Kevin Fenzi <kevin@scrye.com>
This commit is contained in:
Kevin Fenzi 2024-08-13 16:40:01 -07:00
parent eaba6a35e6
commit 0dfa11a6eb
22 changed files with 0 additions and 1038 deletions
Download patch file Download diff file Expand all files Collapse all files

1
inventory/group_vars/batcave
View file

@ -50,7 +50,6 @@ ipa_client_shell_groups:
- sysadmin-cvs
- sysadmin-datanommer
- sysadmin-debuginfod
- sysadmin-fedimg
- sysadmin-koschei
- sysadmin-libravatar
- sysadmin-messaging

36
inventory/group_vars/fedimg
View file

@ -1,36 +0,0 @@
---
# These are consumed by a task in roles/fedmsg/base/main.yml
fedmsg_certs:
- can_send:
- logger.log
group: sysadmin
owner: root
service: shell
- can_send:
- fedimg.image.test
- fedimg.image.upload
- fedimg.image.copy
- fedimg.image.publish
group: fedmsg
owner: root
service: fedimg
# These people get told when something goes wrong.
fedmsg_error_recipients:
- sysadmin-fedimg-members@fedoraproject.org
ipa_client_shell_groups:
- sysadmin-releng
- sysadmin-fedimg
ipa_client_sudo_groups:
- sysadmin-releng
- sysadmin-fedimg
ipa_host_group: fedimg
lvm_size: 20000
mem_size: 6144
num_cpus: 2
primary_auth_source: ipa
# for systems that do not match the above - specify the same parameter in
# the host_vars/$hostname file
tcp_ports: [
# These are all for outgoing fedmsg.
3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, 3012, 3013]
testing: False

35
inventory/group_vars/fedimg_stg
View file

@ -1,35 +0,0 @@
---
# These are consumed by a task in roles/fedmsg/base/main.yml
fedmsg_certs:
- can_send:
- logger.log
group: sysadmin
owner: root
service: shell
- can_send:
- fedimg.image.test
- fedimg.image.upload
- fedimg.image.copy
- fedimg.image.publish
group: fedmsg
owner: root
service: fedimg
fedmsg_debug_loopback: True
# These people get told when something goes wrong.
fedmsg_error_recipients:
- sysadmin-fedimg-members@fedoraproject.org
ipa_client_shell_groups:
- sysadmin-releng
ipa_client_sudo_groups:
- sysadmin-releng
ipa_host_group: fedimg
lvm_size: 20000
mem_size: 6144
num_cpus: 2
# for systems that do not match the above - specify the same parameter in
# the host_vars/$hostname file
tcp_ports: [
# These are all for outgoing fedmsg.
3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, 3012, 3013]
# Use infrastructure-tags-stg repo
testing: True

8
inventory/host_vars/fedimg01.iad2.fedoraproject.org
View file

@ -1,8 +0,0 @@
---
datacenter: iad2
eth0_ipv4_gw: 10.3.163.254
eth0_ipv4_ip: 10.3.163.52
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
vmhost: vmhost-x86-02.iad2.fedoraproject.org
volgroup: /dev/vg_guests

11
inventory/inventory
View file

@ -87,12 +87,6 @@ ibiblio05.fedoraproject.org
[ibiblio_old_virt]
ibiblio05.fedoraproject.org
[fedimg]
fedimg01.iad2.fedoraproject.org
[fedimg_stg]
# fedimg01.stg.iad2.fedoraproject.org
[busgateway]
busgateway01.iad2.fedoraproject.org
@ -578,7 +572,6 @@ db03.stg.iad2.fedoraproject.org
debuginfod01.stg.iad2.fedoraproject.org
oci-candidate-registry01.stg.iad2.fedoraproject.org
oci-registry01.stg.iad2.fedoraproject.org
# fedimg01.stg.iad2.fedoraproject.org
github2fedmsg01.stg.iad2.fedoraproject.org
ipa01.stg.iad2.fedoraproject.org
ipa02.stg.iad2.fedoraproject.org
@ -697,12 +690,10 @@ wiki02.iad2.fedoraproject.org
# assorted categories of fedmsg services, for convenience
[fedmsg_hubs:children]
busgateway
fedimg
pkgs
[fedmsg_hubs_stg:children]
busgateway_stg
fedimg_stg
pkgs_stg
[fedmsg_ircs:children]
@ -1041,7 +1032,6 @@ dbserver
debuginfod
dns_iad2
download_iad2
fedimg
flatpak_cache
github2fedmsg
ipa
@ -1079,7 +1069,6 @@ buildvm_aarch64_stg
busgateway_stg
dbserver_stg
debuginfod_stg
fedimg_stg
github2fedmsg_stg
ipa_stg
ipsilon_stg

1
main.yml
View file

@ -30,7 +30,6 @@
- import_playbook: /srv/web/infra/ansible/playbooks/groups/debuginfod.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/dns.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/download.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/fedimg.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/flatpak-cache.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/github2fedmsg.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/ipa.yml

67
playbooks/groups/fedimg.yml
View file

@ -1,67 +0,0 @@
# create a new fedimg server
# NOTE: make sure there is room/space for this server on the vmhost
- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml"
vars:
myhosts: "fedimg:fedimg_stg"
- name: dole out the generic configuration
hosts: fedimg:fedimg_stg
user: root
gather_facts: True
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
roles:
- base
- rkhunter
# The proxies don't actually need to talk to these hosts so we won't bother
# putting them on the vpn.
#- { role: openvpn/client,
# when: env != "staging" }
- ipa/client
- nagios_client
- hosts
- collectd/base
- fedmsg/base
- sudo
pre_tasks:
- import_tasks: "{{ tasks_path }}/yumrepos.yml"
tasks:
- import_tasks: "{{ tasks_path }}/motd.yml"
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
- name: dole out the service-specific config
hosts: fedimg:fedimg_stg
user: root
gather_facts: True
roles:
- fedmsg/hub
- role: fedimg
aws_keyname: fedimg-dev
aws_keypath: /etc/pki/fedimg/fedimg-dev
aws_pubkeypath: /etc/pki/fedimg/fedimg-dev.pub
when: env == 'staging'
- role: fedimg
aws_keyname: releng-ap-northeast-1
aws_keypath: /etc/pki/fedimg/fedimg-prod
aws_pubkeypath: /etc/pki/fedimg/fedimg-prod.pub
when: env != 'staging'
- role: collectd/fedmsg-service
process: fedmsg-hub
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"

62
playbooks/manual/upgrade/fedimg.yml
View file

@ -1,62 +0,0 @@
- name: push packages out
hosts: fedimg:fedimg_stg
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
tasks:
- name: clean all metadata {%if testing%}(with infrastructure-testing on){%endif%}
command: yum clean all {%if testing%} --enablerepo=infrastructure-tags-stg {%endif%}
check_mode: no
- name: update fedimg packages from main repo
package:
name: ["python2-fedimg", "python2-libcloud", "python2-fedfind", "python2-toml", "python-vcrpy"]
state: latest
when: not testing
- name: update fedimg packages from testing repo
yum:
name: [python2-fedimg", "python2-libcloud", "python2-fedfind", "python2-toml", "python-vcrpy"]
state: latest
enablerepo: infrastructure-tags-stg
when: testing
- name: verify the backend and restart it
hosts: fedimg:fedimg_stg
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
pre_tasks:
- name: tell nagios to shush
nagios: action=downtime minutes=60 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.iad2.fedoraproject.org
ignore_errors: true
roles:
- role: fedimg
aws_keyname: fedimg-dev
aws_keypath: /etc/pki/fedimg/fedimg-dev
aws_pubkeypath: /etc/pki/fedimg/fedimg-dev.pub
when: env == 'staging'
- role: fedimg
aws_keyname: releng-ap-northeast-1
aws_keypath: /etc/pki/fedimg/fedimg-prod
aws_pubkeypath: /etc/pki/fedimg/fedimg-prod.pub
when: env != 'staging'
post_tasks:
- service: name="fedmsg-hub" state=restarted
- name: tell nagios to unshush
nagios: action=unsilence service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.iad2.fedoraproject.org
ignore_errors: true

BIN
roles/apps-fp-o/files/img/icons/fedimg.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 1.5 KiB

76
roles/fedimg/files/partial_upload.py
View file

@ -1,76 +0,0 @@
#!/bin/env python
# -*- coding: utf8 -*-
""" Triggers a partial upload process with the specified raw.xz URL. """
import argparse
import logging
import logging.config
import fedmsg.config
from fedimg.config import AWS_ACCESS_ID
from fedimg.config import AWS_SECRET_KEY
from fedimg.config import AWS_BASE_REGION, AWS_REGIONS
from fedimg.services.ec2.ec2copy import main as ec2copy
from fedimg.services.ec2.ec2initiate import main as ec2main
logging.config.dictConfig(fedmsg.config.load_config()['logging'])
log = logging.getLogger('fedmsg')
def get_args():
parser = argparse.ArgumentParser(
description="Trigger a partial upload based on the arguments")
parser.add_argument(
"-u", "--url", type=str, help=".raw.xz URL", required=True)
parser.add_argument(
"-c", "--compose-id", type=str, help="compose id of the .raw.xz file",
required=True)
parser.add_argument(
"-p", "--push-notifications",
help="Bool to check if we need to push fedmsg notifications",
action="store_true", required=False)
parser.add_argument(
"-v", "--volume", help="volume type for the image", required=False)
args = parser.parse_args()
return (
args.url,
args.compose_id,
args.push_notifications,
args.volume
)
def main():
url, compose_id, push_notifications, volume = get_args()
if volume is not None:
volume = [volume]
images_metadata = ec2main(
image_urls=[url],
access_id=AWS_ACCESS_ID,
secret_key=AWS_SECRET_KEY,
regions=None,
volume_types=volume,
push_notifications=push_notifications,
compose_id=compose_id
)
for image_metadata in images_metadata:
image_id = image_metadata['image_id']
aws_regions = list(set(AWS_REGIONS) - set([AWS_BASE_REGION]))
ec2copy(
aws_regions,
AWS_ACCESS_ID,
AWS_SECRET_KEY,
image_ids=[image_id],
push_notifications=push_notifications,
compose_id=compose_id
)
if __name__ == '__main__':
main()

47
roles/fedimg/files/trigger_upload.py
View file

@ -1,47 +0,0 @@
#!/bin/env python
# -*- coding: utf8 -*-
""" Triggers an upload process with the specified raw.xz URL. """
import argparse
import logging
import logging.config
import multiprocessing.pool
import fedmsg.config
import fedimg.uploader
logging.config.dictConfig(fedmsg.config.load_config()['logging'])
log = logging.getLogger('fedmsg')
def trigger_upload(url, compose_id, push_notifications):
upload_pool = multiprocessing.pool.ThreadPool(processes=4)
fedimg.uploader.upload(upload_pool, [url],
compose_id=compose_id,
push_notifications=push_notifications)
def get_args():
parser = argparse.ArgumentParser(
description="Trigger a manual upload process with the "
"specified raw.xz URL")
parser.add_argument(
"-u", "--url", type=str, help=".raw.xz URL", required=True)
parser.add_argument(
"-c", "--compose-id", type=str, help="compose id of the .raw.xz file",
required=True)
parser.add_argument(
"-p", "--push-notifications",
help="Bool to check if we need to push fedmsg notifications",
action="store_true", required=False)
args = parser.parse_args()
return args.url, args.compose_id, args.push_notifications
def main():
url, compose_id, push_notifications = get_args()
trigger_upload(url, compose_id, push_notifications)
if __name__ == '__main__':
main()

135
roles/fedimg/tasks/main.yml
View file

@ -1,135 +0,0 @@
---
# Configuration for the notifications consumer
- name: install needed packages
package:
state: present
name:
- koji
- fedmsg
- python-paramiko
- python-fedimg
- python-libcloud
- python-fedimg
- python-boto3
tags:
- fedimg
- name: install extra packages in staging
package:
state: present
name:
- euca2ools
tags:
- fedimg
- name: copy manual trigger script
copy: >
src=trigger_upload.py
dest=/usr/local/bin/trigger_upload.py
mode=0755
tags:
- fedimg
- name: copy manual partial trigger script
copy: >
src=partial_upload.py
dest=/usr/local/bin/partial_upload.py
mode=0755
tags:
- fedimg
- name: copy base configuration
template: >
src=fedimg-conf.toml dest=/etc/fedimg/fedimg-conf.toml
owner=fedmsg group=fedmsg mode=0700
notify:
- restart fedmsg-hub
tags:
- fedimg
- name: copy the euca2ools configuration file
template: >
src=fedimg-euca-conf.ini dest=/etc/euca2ools/conf.d/fedimg-euca-conf.ini
owner=fedmsg group=fedmsg mode=0700
notify:
- restart fedmsg-hub
tags:
- fedimg
- name: copy fedimg fedmsg consumer
template: >
src=fedmsg.d/{{item}}
dest=/etc/fedmsg.d/{{item}}
owner=fedmsg
group=fedmsg
mode=0600
with_items:
- fedimg.py
- fedimg-logging.py
notify:
- restart fedmsg-hub
tags:
- fedimg
- name: make pki directory
file: dest=/etc/pki/fedimg/ state=directory
owner=fedmsg group=fedmsg mode=0500
tags:
- fedimg
- name: copy keys into pki directory for staging
copy: src={{private}}/files/fedimg/{{item}} dest=/etc/pki/fedimg/{{item}}
owner=fedmsg group=fedmsg mode=0400
with_items:
- fedimg-dev
- fedimg-dev.pub
notify:
- restart fedmsg-hub
when: env == "staging"
tags:
- fedimg
- name: copy keys into pki directory for production
copy: src={{private}}/files/fedimg/{{item}} dest=/etc/pki/fedimg/{{item}}
owner=fedmsg group=fedmsg mode=0400
with_items:
- fedimg-prod
- fedimg-prod.pub
notify:
- restart fedmsg-hub
when: env != "staging"
tags:
- fedimg
- name: ensure the fedmsg user has a homedir for cron to work in
file: >
state=directory
path=/usr/share/fedmsg
mode=700
owner=fedmsg
group=fedmsg
tags:
- cron
- fedimg
- name: ensure fedimg cron directories exist
file: >
state=directory
path={{ item }}
mode=755
owner=root
with_items:
- /usr/share/fedimg/cronjobs/
- /etc/cron.d/
tags:
- cron
- fedimg
- name: copy the releng script to purge ami to test
template: >
src=clean-amis.py dest=/usr/local/bin/clean-amis.py
owner=fedmsg group=fedmsg mode=0700
tags:
- scripts
- fedimg

410
roles/fedimg/templates/clean-amis.py
View file

@ -1,410 +0,0 @@
#!/usr/bin/python
#
# clean-amis.py - A utility to remove the nightly AMIs every 5 days.
#
#
# Authors:
# Sayan Chowdhury <sayanchowdhury@fedoraproject.org>
# Copyright (C) 2016 Red Hat Inc,
# SPDX-License-Identifier: GPL-2.0+
#
# The script runs as a cron job within the Fedora Infrastructure to delete
# the old AMIs. The permission of the selected AMIs are changed to private.
# This is to make sure that if someone from the community raises an issue
# we have the option to get the AMI back to public.
# After 10 days, if no complaints are raised the AMIs are deleted permanently.
#
# The complete process can be divided in couple of parts:
#
# - Fetching the data from datagrepper.
# Based on the `--days` param, the script starts fetching the fedmsg messages
# from datagrepper for the specified timeframe i.e. for lasts `n` days, where
# `n` is the value of `--days` param. The queried fedmsg
# topic `fedimg.image.upload`.
#
# - Selection of the AMIs:
# After the AMIs are parsed from datagrepper. The AMIs are filtered to remove
# Beta, Two-week Atomic Host and GA released AMIs.
# Composes with `compose_type` set to `nightly` are picked up for deletion.
# Composes which contain date in the `compose label` are also picked up for
# deletion.
# GA composes also have the compose_type set to production. So to distinguish
# then we filter them if the compose_label have date in them. The GA
# composes dont have date whereas they have the version in format of X.Y
#
# - Updated permissions of AMIs
# The permissions of the selected AMIs are changed to private.
#
# - Deletion of AMIs
# After 10 days, the private AMIs are deleted.
from __future__ import print_function
import os
import re
import argparse
import boto3
import functools
import fedfind
import fedfind.release
import requests
from datetime import datetime, timedelta, date
import logging
logging.basicConfig(level=logging.INFO)
log = logging.getLogger()
env = os.environ
aws_access_key_id = os.environ.get("AWS_ACCESS_KEY", '{{ ec2_image_delete_access_key_id }}')
aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY", '{{ ec2_image_delete_access_key }}')
DATAGREPPER_URL = "https://apps.fedoraproject.org/datagrepper/"
NIGHTLY = "nightly"
REGIONS = (
"us-east-1",
"us-east-2",
"us-west-2",
"us-west-1",
"eu-west-1",
"eu-central-1",
"ap-east-1",
"ap-south-1",
"ap-northeast-1",
"ap-northeast-2",
"ap-southeast-1",
"ap-southeast-2",
"ap-southeast-3",
"sa-east-1",
"ca-central-1",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"eu-north-1",
"af-south-1",
)
def _is_timestamp_newer(timestamp1, timestamp2):
""" Return true if timestamp1 is newer than timestamp2
"""
timestamp1_f = datetime.strptime(timestamp1, "%d%m%Y")
timestamp2_f = datetime.strptime(timestamp2, "%d%m%Y")
return timestamp1_f > timestamp2_f
def _get_raw_url():
""" Get the datagrepper raw URL to fetch the message from
"""
return DATAGREPPER_URL + "/raw"
def get_page(page, delta, topic, start=None, end=None):
params = {
"topic": topic,
"delta": delta,
"rows_per_page": 100,
"page": page,
}
if start:
params.update({"start": start})
if end:
params.update({"end": end})
resp = requests.get(_get_raw_url(), params=params)
return resp.json()
def _get_two_week_released_atomic_compose_id(delta, start=None, end=None):
""" Returns the release compose ids for last n days """
topic = "org.fedoraproject.prod.releng.atomic.twoweek.complete"
data = get_page(1, delta, topic, start, end)
messages = data.get("raw_messages", [])
for page in range(1, data["pages"]):
data = get_page(
topic=topic, page=page + 1, delta=delta, start=start, end=end
)
messages.extend(data["raw_messages"])
messages = [msg["msg"] for msg in messages]
released_atomic_compose_ids = []
for msg in messages:
# This is to support the older-format fedmsg messages
if "atomic_raw" in msg:
released_atomic_compose_ids.append(msg["atomic_raw"]["compose_id"])
# We are just trying here multiple archs to get the compose id
elif "aarch64" in msg:
released_atomic_compose_ids.append(
msg["aarch64"]["atomic_raw"]["compose_id"]
)
elif "x86_64" in msg:
released_atomic_compose_ids.append(
msg["x86_64"]["atomic_raw"]["compose_id"]
)
elif "ppc64le" in msg:
released_atomic_compose_ids.append(
msg["ppc64le"]["atomic_raw"]["compose_id"]
)
return set(released_atomic_compose_ids)
def _get_nightly_amis_nd(delta, start=None, end=None):
""" Returns the nightly AMIs for the last n days
:args delta: last delta seconds
"""
amis = []
released_atomic_compose_ids = _get_two_week_released_atomic_compose_id(
delta=delta, start=start, end=end
)
topic = "org.fedoraproject.prod.fedimg.image.publish"
data = get_page(1, delta, topic, start, end)
messages = data.get("raw_messages", [])
for page in range(1, data["pages"]):
data = get_page(
topic=topic, page=page + 1, delta=delta, start=start, end=end
)
messages.extend(data["raw_messages"])
for message in messages:
msg = message.get("msg")
ami_id = msg["extra"]["id"]
region = msg["destination"]
compose_id = msg["compose"]
compose_info = fedfind.release.get_release(cid=compose_id)
compose_type = compose_info.type
compose_label = compose_info.label
# Sometimes the compose label is None
# and they can be blindly put in for deletion
if not compose_label:
amis.append((compose_id, ami_id, region))
if compose_id in released_atomic_compose_ids:
continue
# Include the nightly composes
if compose_type == NIGHTLY:
amis.append((compose_id, ami_id, region))
else:
# Include AMIs that have date in them
# These are the production compose type but not GA
result = re.search("-(\d{8}).", compose_label)
if result is None:
continue
amis.append((compose_id, ami_id, region))
return amis
def delete_amis_nd(deletetimestamp, dry_run=False):
""" Delete the give list of nightly AMIs
:args deletetimestamp: the timestamp for the delete
:args dry_run: dry run the flow
"""
log.info("Deleting AMIs")
for region in REGIONS:
log.info("%s Starting" % region)
# Create a connection to an AWS region
conn = boto3.client(
"ec2",
region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
)
log.info("%s: Connected" % region)
response = conn.describe_images(
Filters=[{"Name": "tag-key", "Values": ["LaunchPermissionRevoked"]}]
)
amis = response.get("Images", [])
for ami in amis:
try:
ami_id = ami["ImageId"]
is_launch_permitted = ami["Public"]
_index = len(ami["BlockDeviceMappings"])
snapshot_id = ami["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"]
tags = ami["Tags"]
revoketimestamp = ""
for tag in tags:
if "LaunchPermissionRevoked" in tag.values():
revoketimestamp = tag["Value"]
if not revoketimestamp:
log.warn(
"%s ami has LaunchPermissionRevoked tag but no value"
% ami_id
)
continue
if is_launch_permitted:
log.warn(
"%s ami has LaunchPermissionRevoked tag "
"but launch permission is still enabled" % ami_id
)
continue
# The revoke timestamp allows us to tell how long ago an image
# had permissions removed. If the permissions have been removed
# for shorter than the waiting period then we can't delete it yet.
if _is_timestamp_newer(revoketimestamp, deletetimestamp):
continue
if not dry_run:
conn.deregister_image(ImageId=ami_id)
conn.delete_snapshot(SnapshotId=snapshot_id)
else:
print(ami_id)
except Exception as ex:
log.error("%s: %s failed\n%s" % (region, ami_id, ex))
def change_amis_permission_nd(amis, dry_run=False):
""" Change the launch permissions of the AMIs to private.
The permission of the AMIs are changed to private first and then delete
after 5 days.
:args amis: list of AMIs
:args dry_run: dry run the flow
"""
log.info("Changing permission for AMIs")
todaystimestamp = date.today().strftime("%d%m%Y")
for region in REGIONS:
log.info("%s: Starting" % region)
# Create a connection to an AWS region
conn = boto3.client(
"ec2",
region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
)
log.info("%s: Connected" % region)
# Filter all the nightly AMIs belonging to this region
r_amis = [(c, a, r) for c, a, r in amis if r == region]
# Loop through the AMIs change the permissions
for _, ami_id, region in r_amis:
try:
if not dry_run:
conn.modify_image_attribute(
ImageId=ami_id,
LaunchPermission={"Remove": [{"Group": "all"}]},
)
conn.create_tags(
Resources=[ami_id],
Tags=[
{
"Key": "LaunchPermissionRevoked",
"Value": todaystimestamp,
}
],
)
else:
print(ami_id)
except Exception as ex:
log.error("%s: %s failed \n %s" % (region, ami_id, ex))
if __name__ == "__main__":
argument_parser = argparse.ArgumentParser()
argument_parser.add_argument(
"--delete",
help="Delete the AMIs whose launch permissions have been removed",
action="store_true",
default=False,
)
argument_parser.add_argument(
"--days",
help="Specify the number of days worth of AMI fedmsg information to fetch from datagrepper.",
type=int,
)
argument_parser.add_argument(
"--deletewaitperiod",
help="Specify the number of days to wait after removing launch perms before deleting",
type=int,
default=10,
)
argument_parser.add_argument(
"--permswaitperiod",
help="Specify the number of days to wait before removing launch perms",
type=int,
default=10,
)
argument_parser.add_argument(
"--change-perms",
help="Change the launch permissions of the AMIs to private",
action="store_true",
default=False,
)
argument_parser.add_argument(
"--dry-run",
help="Dry run the action to be performed",
action="store_true",
default=False,
)
args = argument_parser.parse_args()
if not args.delete and not args.change_perms:
raise Exception(
"Either of the argument, delete or change permission is required"
)
if args.delete and args.change_perms:
raise Exception(
"Both the argument delete and change permission is not allowed"
)
# Ideally, we could search through all the AMIs that ever were created but this
# this would create huge load on datagrepper.
# default to 4 weeks/ 28 days
days = 28
if args.days:
days = args.days
permswaitperiod = args.permswaitperiod
deletewaitperiod = args.deletewaitperiod
# The AMIs deleted are the nightly AMIs that are uploaded via fedimg everyday.
# The clean up of the AMIs happens through a cron job.
# The steps followed while deleting the AMIs:
# - The selected AMIs are made private, so that if people report issue we can make it
# public again.
# - If no issues are reported in 10 days, the AMIs are deleted permanently.
if args.change_perms:
if days < permswaitperiod:
raise Exception(
"permswaitperiod param cannot be more than days param"
)
end = (datetime.now() - timedelta(days=permswaitperiod)).strftime("%s")
amis = _get_nightly_amis_nd(
delta=86400 * (days - permswaitperiod), end=int(end)
)
change_amis_permission_nd(amis, dry_run=args.dry_run)
if args.delete:
deletetimestamp = (
datetime.now() - timedelta(days=deletewaitperiod)
).strftime("%d%m%Y")
delete_amis_nd(deletetimestamp, dry_run=args.dry_run)

41
roles/fedimg/templates/fedimg-conf.toml
View file

@ -1,41 +0,0 @@
[general]
clean_up_on_failure = false
delete_images_on_failure = false
process_count = {{process_count}}
active_services = ['aws']
[aws]
root_volume_size = {{aws_volume_size}}
{% if env == 'staging' %}
access_id = '{{fedimg_aws_stg_access_id}}'
secret_key = '{{fedimg_aws_stg_secret_key}}'
{% else %}
access_id = '{{fedimg_aws_prod_access_id}}'
secret_key = '{{fedimg_aws_prod_secret_key}}'
{% endif %}
{% if env == 'staging' %}
bucket_name = 'fedora-s3-bucket-fedimg-test'
{% else %}
bucket_name = 'fedora-s3-bucket-fedimg'
{% endif %}
base_region = 'us-east-1'
regions = [ 'af-south-1', 'eu-north-1', 'ap-south-1', 'eu-west-3', 'eu-west-2',
'eu-south-1', 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1',
'ap-northeast-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-1',
'ap-southeast-2', 'ap-southeast-3', 'eu-central-1', 'us-east-1', 'us-east-2',
'us-west-1', 'us-west-2' ]
volume_types = ['gp3']
[rackspace]
username = 'someuser'
api_key = 'secretk3y'
[gce]
email = 'someacct@provider.com'
keypath = '/path/to/pem/file'
project_id = 'someprojectid'
[hp]
username = 'aperson'
password = 'somecoolpassword'
tenant = 'theprojectname'

11
roles/fedimg/templates/fedimg-euca-conf.ini
View file

@ -1,11 +0,0 @@
[user fedimg]
{% if env == 'staging' %}
key-id = {{fedimg_aws_stg_access_id}}
secret-key = {{fedimg_aws_stg_secret_key}}
{% else %}
key-id = {{fedimg_aws_prod_access_id}}
secret-key = {{fedimg_aws_prod_secret_key}}
{% endif %}
[region aws:*]
user = fedimg

13
roles/fedimg/templates/fedmsg.d/fedimg-logging.py
View file

@ -1,13 +0,0 @@
# Setup fedmsg logging.
# See the following for constraints on this format https://bit.ly/Xn1WDn
config = dict(
logging=dict(
loggers=dict(
fedimg={
"level": "DEBUG",
"propagate": False,
"handlers": ["console"],
},
),
),
)

36
roles/fedimg/templates/fedmsg.d/fedimg.py
View file

@ -1,36 +0,0 @@
# This file is part of fedimg.
# Copyright (C) 2014 Red Hat, Inc.
#
# fedimg is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# fedimg is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with fedimg; if not, see http://www.gnu.org/licenses,
# or write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors: David Gay <dgay@redhat.com>
# Ralph Bean <rbean@redhat.com>
# Sayan Chowdhury <sayan@redhat.com>
#
{% if env == 'staging' %}
config = {
'fedimgconsumer.dev.enabled': False,
'fedimgconsumer.prod.enabled': False,
'fedimgconsumer.stg.enabled': True,
}
{% else %}
config = {
'fedimgconsumer.dev.enabled': False,
'fedimgconsumer.prod.enabled': True,
'fedimgconsumer.stg.enabled': False,
}
{% endif %}

6
roles/fedimg/vars/main.yml
View file

@ -1,6 +0,0 @@
clean_up_on_failure: false
delete_images_on_failure: false
process_count: 1
aws_volume_size: 6
aws_test_volume_size: 6

1
roles/nagios_client/templates/check_datanommer_history.cfg.j2
View file

@ -22,7 +22,6 @@ command[check_datanommer_compose]={{libdir}}/nagios/plugins/check_datanommer_tim
command[check_datanommer_copr]={{libdir}}/nagios/plugins/check_datanommer_timesince.py copr 21600 86400
command[check_datanommer_fas]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fas 1814400 2628000
command[check_datanommer_badges]={{libdir}}/nagios/plugins/check_datanommer_timesince.py badges 86400 259200
command[check_datanommer_fedimg]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fedimg 259200 604800
command[check_datanommer_fedocal]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fedocal 7884000 23652000
command[check_datanommer_fmn]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fmn 604800 1814400
command[check_datanommer_git]={{libdir}}/nagios/plugins/check_datanommer_timesince.py git 86400 604800

3
roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2
View file

@ -10,7 +10,6 @@ command[check_fedmsg_cp_app]={{libdir}}/nagios/plugins/check_fedmsg_producers_co
command[check_fedmsg_cp_value]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-irc IRCBotConsumer MonitoringProducer
command[check_fedmsg_cp_badges_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedoraBadgesConsumer MonitoringProducer
command[check_fedmsg_cp_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FMNConsumer DigestProducer ConfirmationProducer MonitoringProducer
command[check_fedmsg_cp_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer
command[check_fedmsg_cp_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
command[check_fedmsg_cp_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer
@ -25,7 +24,6 @@ command[check_fedmsg_cexceptions_app]={{libdir}}/nagios/plugins/check_fedmsg_con
command[check_fedmsg_cexceptions_value]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-irc IRCBotConsumer 1 10
command[check_fedmsg_cexceptions_badges_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedoraBadgesConsumer 1 10
command[check_fedmsg_cexceptions_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FMNConsumer 1 10
command[check_fedmsg_cexceptions_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10
command[check_fedmsg_cexceptions_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
command[check_fedmsg_cexceptions_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10
@ -41,7 +39,6 @@ command[check_fedmsg_cbacklog_value]={{libdir}}/nagios/plugins/check_fedmsg_cons
command[check_fedmsg_cbacklog_badges_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedoraBadgesConsumer 25000 35000
command[check_fedmsg_cbacklog_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FMNConsumer 15000 20000
command[check_fedmsg_cbacklog_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py moksha-hub BugzillaConsumer 10 100
command[check_fedmsg_cbacklog_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000
command[check_fedmsg_cbacklog_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 1000 5000
command[check_fedmsg_cbacklog_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000

34
roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg
View file

@ -52,13 +52,6 @@ define service {
use defaulttemplate
}
define service {
host_name fedimg01.iad2.fedoraproject.org
service_description Check for fedmsg-hub proc
check_command check_by_nrpe!check_fedmsg_hub_proc
use defaulttemplate
}
# TODO: Uncomment this if it comes back in iad
#define service {
# host_name packages03.iad2.fedoraproject.org
@ -141,12 +134,6 @@ define service {
check_command check_by_nrpe!check_datanommer_ansible
use defaulttemplate
}
define service {
host_name busgateway01.iad2.fedoraproject.org
service_description Check datanommer for recent fedimg messages
check_command check_by_nrpe!check_datanommer_fedimg
use defaulttemplate
}
define service {
host_name busgateway01.iad2.fedoraproject.org
service_description Check datanommer for recent hotness messages
@ -228,13 +215,6 @@ define service {
# use defaulttemplate
#}
define service {
host_name fedimg01.iad2.fedoraproject.org
service_description Check fedmsg consumers and producers hub
check_command check_by_nrpe!check_fedmsg_cp_fedimg_backend
use defaulttemplate
}
# TODO: Uncomment this if it comes back in iad
#define service {
# host_name packages03.iad2.fedoraproject.org
@ -265,13 +245,6 @@ define service {
# use defaulttemplate
#}
define service {
host_name fedimg01.iad2.fedoraproject.org
service_description Check fedmsg-hub consumers exceptions
check_command check_by_nrpe!check_fedmsg_cexceptions_fedimg_backend
use defaulttemplate
}
# TODO: Uncomment this if it comes back in iad
#define service {
# host_name packages03.iad2.fedoraproject.org
@ -302,13 +275,6 @@ define service {
# use defaulttemplate
#}
define service {
host_name fedimg01.iad2.fedoraproject.org
service_description Check fedmsg-hub consumers backlog
check_command check_by_nrpe!check_fedmsg_cbacklog_fedimg_backend
use defaulttemplate
}
# TODO: Uncomment this if it comes back in iad
#define service {
# host_name packages03.iad2.fedoraproject.org

4
roles/nagios_server/templates/nrpe/nrpe.cfg.j2
View file

@ -364,7 +364,6 @@ command[check_datanommer_compose]=/usr/lib64/nagios/plugins/check_datanommer_tim
command[check_datanommer_copr]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py copr 21600 86400
command[check_datanommer_fas]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py fas 1814400 2628000
command[check_datanommer_badges]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py badges 86400 259200
command[check_datanommer_fedimg]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py fedimg 259200 604800
command[check_datanommer_fedocal]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py fedocal 7884000 23652000
command[check_datanommer_fmn]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py fmn 604800 1814400
command[check_datanommer_git]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py git 86400 604800
@ -388,7 +387,6 @@ command[check_fedmsg_cp_app]=/usr/lib64/nagios/plugins/check_fedmsg_producers_co
command[check_fedmsg_cp_value]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-irc IRCBotConsumer MonitoringProducer
command[check_fedmsg_cp_badges_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedoraBadgesConsumer MonitoringProducer
command[check_fedmsg_cp_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FMNConsumer DigestProducer ConfirmationProducer MonitoringProducer
command[check_fedmsg_cp_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer
command[check_fedmsg_cp_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
command[check_fedmsg_cp_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer
@ -398,7 +396,6 @@ command[check_fedmsg_cexceptions_app]=/usr/lib64/nagios/plugins/check_fedmsg_con
command[check_fedmsg_cexceptions_value]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-irc IRCBotConsumer 1 10
command[check_fedmsg_cexceptions_badges_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedoraBadgesConsumer 1 10
command[check_fedmsg_cexceptions_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FMNConsumer 1 10
command[check_fedmsg_cexceptions_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10
command[check_fedmsg_cexceptions_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
command[check_fedmsg_cexceptions_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10
@ -408,7 +405,6 @@ command[check_fedmsg_cbacklog_app]=/usr/lib64/nagios/plugins/check_fedmsg_consum
command[check_fedmsg_cbacklog_value]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-irc IRCBotConsumer 10 50
command[check_fedmsg_cbacklog_badges_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedoraBadgesConsumer 25000 35000
command[check_fedmsg_cbacklog_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FMNConsumer 10 50
command[check_fedmsg_cbacklog_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000
command[check_fedmsg_cbacklog_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 100 500
command[check_fedmsg_cbacklog_packages_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000