From 1d458e6d8a31ffd8cf0bf4319787db15240ac8d9 Mon Sep 17 00:00:00 2001 From: Michal Konecny Date: Tue, 5 Sep 2023 17:09:58 +0200 Subject: [PATCH] Migrate howto repository to infra docs It doesn't make much sense to have separate guides outside infra documentation. So let's migrate https://pagure.io/fedora-infra/howtos to infra docs. Signed-off-by: Michal Konecny --- modules/howtos/pages/access_rabbitmq_ui.adoc | 14 ++ .../pages/activate_a_spamcheck_account.adoc | 5 + .../pages/add_a_package_to_infra_tag.adoc | 9 + .../pages/add_external_hardware_to_vpn.adoc | 33 ++++ .../pages/archive-old-fedora.adoc | 45 ++++- .../pages/build_against_infra_tags.adoc | 16 ++ .../check_robosignatory_production_logs.adoc | 10 ++ modules/howtos/pages/clean_2f_tokens.adoc | 26 +++ .../pages/clean_monitoring_sidetags.adoc | 23 +++ modules/howtos/pages/create_keytab.adoc | 40 +++++ .../howtos/pages/create_new_mailing_list.adoc | 13 ++ .../howtos/pages/creating_groups_distgit.adoc | 23 +++ .../howtos/pages/delete_mailman_thread.adoc | 25 +++ .../howtos/pages/destroy_a_virt_instance.adoc | 28 +++ modules/howtos/pages/discourse_spam.adoc | 18 ++ .../pages/fedora_messaging_certificates.adoc | 122 +++++++++++++ .../pages/fix_bugzilla_aws_saml_login.adoc | 29 +++ modules/howtos/pages/fix_robosignatory.adoc | 49 ++++++ .../howtos/pages/free_space_in_openshift.adoc | 61 +++++++ modules/howtos/pages/generate_2fa_keys.adoc | 33 ++++ .../howtos/pages/generate_openvpn_keys.adoc | 33 ++++ .../howtos/pages/get_logs_pod_openshift.adoc | 15 ++ .../howtos/pages/give_groups_dist_git.adoc | 12 ++ modules/howtos/pages/groups_in_fedora.adoc | 102 +++++++++++ .../howtos/pages/make_mailman_user_admin.adoc | 21 +++ .../howtos/pages/rebuild_osbs_buildroot.adoc | 16 ++ .../pages/refresh_osbs_odcs_oicd_token.adoc | 33 ++++ .../howtos/pages/remove_branch_distgit.adoc | 96 ++++++++++ .../remove_fedora_user_at_launch_aws.adoc | 19 ++ .../pages/remove_meeting_minutes_meetbot.adoc | 46 +++++ .../remove_monitoring_rabbitmq_queue.adoc | 40 +++++ .../remove_user_from_watchlist_pagure.adoc | 28 +++ .../howtos/pages/restart_sigul_bridge.adoc | 14 ++ .../pages/scale_up_or_down_deployment.adoc | 27 +++ modules/howtos/pages/share_tmux_session.adoc | 31 ++++ .../pages/unblock_bodhi_rawhide_updates.adoc | 18 ++ .../howtos/pages/update_watch_dist_git.adoc | 61 +++++++ modules/sysadmin_guide/nav.adoc | 126 +------------ modules/sysadmin_guide/pages/index.adoc | 165 ++++++++++++++++++ 39 files changed, 1399 insertions(+), 126 deletions(-) create mode 100644 modules/howtos/pages/access_rabbitmq_ui.adoc create mode 100644 modules/howtos/pages/activate_a_spamcheck_account.adoc create mode 100644 modules/howtos/pages/add_a_package_to_infra_tag.adoc create mode 100644 modules/howtos/pages/add_external_hardware_to_vpn.adoc rename modules/{sysadmin_guide => howtos}/pages/archive-old-fedora.adoc (62%) create mode 100644 modules/howtos/pages/build_against_infra_tags.adoc create mode 100644 modules/howtos/pages/check_robosignatory_production_logs.adoc create mode 100644 modules/howtos/pages/clean_2f_tokens.adoc create mode 100644 modules/howtos/pages/clean_monitoring_sidetags.adoc create mode 100644 modules/howtos/pages/create_keytab.adoc create mode 100644 modules/howtos/pages/create_new_mailing_list.adoc create mode 100644 modules/howtos/pages/creating_groups_distgit.adoc create mode 100644 modules/howtos/pages/delete_mailman_thread.adoc create mode 100644 modules/howtos/pages/destroy_a_virt_instance.adoc create mode 100644 modules/howtos/pages/discourse_spam.adoc create mode 100644 modules/howtos/pages/fedora_messaging_certificates.adoc create mode 100644 modules/howtos/pages/fix_bugzilla_aws_saml_login.adoc create mode 100644 modules/howtos/pages/fix_robosignatory.adoc create mode 100644 modules/howtos/pages/free_space_in_openshift.adoc create mode 100644 modules/howtos/pages/generate_2fa_keys.adoc create mode 100644 modules/howtos/pages/generate_openvpn_keys.adoc create mode 100644 modules/howtos/pages/get_logs_pod_openshift.adoc create mode 100644 modules/howtos/pages/give_groups_dist_git.adoc create mode 100644 modules/howtos/pages/groups_in_fedora.adoc create mode 100644 modules/howtos/pages/make_mailman_user_admin.adoc create mode 100644 modules/howtos/pages/rebuild_osbs_buildroot.adoc create mode 100644 modules/howtos/pages/refresh_osbs_odcs_oicd_token.adoc create mode 100644 modules/howtos/pages/remove_branch_distgit.adoc create mode 100644 modules/howtos/pages/remove_fedora_user_at_launch_aws.adoc create mode 100644 modules/howtos/pages/remove_meeting_minutes_meetbot.adoc create mode 100644 modules/howtos/pages/remove_monitoring_rabbitmq_queue.adoc create mode 100644 modules/howtos/pages/remove_user_from_watchlist_pagure.adoc create mode 100644 modules/howtos/pages/restart_sigul_bridge.adoc create mode 100644 modules/howtos/pages/scale_up_or_down_deployment.adoc create mode 100644 modules/howtos/pages/share_tmux_session.adoc create mode 100644 modules/howtos/pages/unblock_bodhi_rawhide_updates.adoc create mode 100644 modules/howtos/pages/update_watch_dist_git.adoc diff --git a/modules/howtos/pages/access_rabbitmq_ui.adoc b/modules/howtos/pages/access_rabbitmq_ui.adoc new file mode 100644 index 0000000..0fb7cdd --- /dev/null +++ b/modules/howtos/pages/access_rabbitmq_ui.adoc @@ -0,0 +1,14 @@ += How to access the rabbitmq administrative UI? + +The UI is only available on localhost, so you will need to do a port forwarding +using ssh. +This can be done using: + +---- +ssh -L 15672:localhost:15672 rabbitmq01.iad2.fedoraproject.org +---- + +You can then visit: http://localhost:15672/ to see the web UI for rabbitmq. + +It will ask you to log in, use the username and the password set in the private +repository. diff --git a/modules/howtos/pages/activate_a_spamcheck_account.adoc b/modules/howtos/pages/activate_a_spamcheck_account.adoc new file mode 100644 index 0000000..d06b407 --- /dev/null +++ b/modules/howtos/pages/activate_a_spamcheck_account.adoc @@ -0,0 +1,5 @@ += How to activate a FAS account that was marked as `spamcheck_awaiting` + +For this you need to be in the `accounts` FAS group that will give you admin permission in FAS. +Then log to https://admin.fedoraproject.org/accounts[https://admin.fedoraproject.org/accounts], +search for the user profile and edit it to set the status to `active`. diff --git a/modules/howtos/pages/add_a_package_to_infra_tag.adoc b/modules/howtos/pages/add_a_package_to_infra_tag.adoc new file mode 100644 index 0000000..2c4a4a8 --- /dev/null +++ b/modules/howtos/pages/add_a_package_to_infra_tag.adoc @@ -0,0 +1,9 @@ += How to add a package to an infra tag + +In order to be able to build a package in an infra tag we need first to add that +package to the tag. +This need koji admin permissions. + +---- +$ koji add-pkg epel7-infra packagename --owner owner +---- diff --git a/modules/howtos/pages/add_external_hardware_to_vpn.adoc b/modules/howtos/pages/add_external_hardware_to_vpn.adoc new file mode 100644 index 0000000..991031a --- /dev/null +++ b/modules/howtos/pages/add_external_hardware_to_vpn.adoc @@ -0,0 +1,33 @@ += Add external servers to vpn + +. In the Fedora Infra Ansible repo edit the file **roles/batcave/files/allows**. +Under the correct section add **require ip** ****** + +. When this change is pushed run the batcave ansible playbook on the batcave. +You will need sysadmin-main access for this + +. Create openvpn and 2fa certificates for the new server. +This requires sysadmin main access + +. <> +. <> + +. In the dns repo on batcave edit the file master/168.192.in-addr.arpa +Add the new host to one of the unused adresses. +Ensure the hostname ends in .vpn.fedoraproject.org. +Don't forget to update the serial before saving. + +. Also edit the master/vpn.fedoraproject.org file to add the server with +the new 192.168.*.* address created in the previous step to the required section +Don't forget to update the serial before saving. + +. When the above edits are done follow the instructions in the DNS sysadmin sop +about signing and pushing new dns chnages. + +. <> + +. Finally in the Fedora Infra Ansible repo add a new file +**roles/openvpn/server/files/ccd/**** with the new 192.168.*.* address. +View one of the existing files in the repo for a sample of formatting. +This change will be run when the server is provisioned. + diff --git a/modules/sysadmin_guide/pages/archive-old-fedora.adoc b/modules/howtos/pages/archive-old-fedora.adoc similarity index 62% rename from modules/sysadmin_guide/pages/archive-old-fedora.adoc rename to modules/howtos/pages/archive-old-fedora.adoc index a34d28c..553f444 100644 --- a/modules/sysadmin_guide/pages/archive-old-fedora.adoc +++ b/modules/howtos/pages/archive-old-fedora.adoc @@ -8,7 +8,7 @@ archives section (`/pub/archive/fedora/linux`) == Steps Involved [arabic] -. log into batcave01.phx2.fedoraproject.org and ssh to bodhi-backend01 +. log into batcave01.iad2.fedoraproject.org and ssh to bodhi-backend01 + [source] ---- @@ -37,6 +37,15 @@ copy of the tree you want to the target ---- $ cp -lvpnr 21 /pub/archive/fedora/linux/releases/21 ---- ++ +Common errors which might occur at this time is finding that the +NFS partition is not mounted read-write. Other errors can occur +where the NFS is read-write, but ftpsync does not have rights to +copy a file. These need to be dealt with individually. ++ +Another error which happens when I script things is that you find +out that everything got copied into +/pub/archive/fedora/linux/releases/21/21 even though it shouldn't. . If the target directory already exists, then we need to do a recursive rsync to update any changes in the trees since the previous copy. @@ -67,6 +76,23 @@ $ rsync -avAXSHP 21/ /pub/archive/fedora/linux/updates/testing/21/ ---- . Do the same with fedora-secondary. ++ +[source] +---- +$ cd /pub/fedora-secondary/releases/ +$ cp -lpnr 21 /pub/archive/fedora-secondary/releases/21 +$ cd ../updates/ +$ cp -lpnr 21 /pub/archive/fedora-secondary/updates/21 +$ cd testing +$ cp -lpnr 21 /pub/archive/fedora-secondary/updates/testing/21 + +$ cd /pub/fedora-secondary/releases/ +$ rsync -avSAXHP --delete 21/ /pub/archive/fedora-secondary/releases/21/ +$ cd ../updates/ +$ rsync -avSAXHP --delete 21/ /pub/archive/fedora-secondary/updates/21/ +$ cd testing +$ rsync -avSAXHP --delete 21/ /pub/archive/fedora-secondary/updates/testing/21/ +---- . Announce to the mirror list this has been done and that in 2 weeks you will move the old trees to archives. @@ -75,6 +101,7 @@ will move the old trees to archives. + [source] ---- +$ sudo -i ssh root@mm-backend01.iad2.fedoraproject.org $ sudo -u mirrormanager mm2_move-to-archive --originalCategory="Fedora Linux" --archiveCategory="Fedora Archive" --directoryRe='/21/Everything' ---- @@ -85,7 +112,7 @@ to get a DBA to update the backend to fix items. + [source] ---- -$ ssh bodhi-backend01 +$ sudo -i ssh bodhi-backend01.iad2.fedoraproject.org $ cd /pub/fedora/linux $ cd releases/21 $ ls # make sure you have stuff here @@ -99,6 +126,20 @@ $ cd ../testing/21 $ ls # make sure you have stuff here $ rm -rf * $ ln ../20/README . + +$ cd /pub/fedora-secondary/releases +$ cd 21 +$ ls # make sure you have stuff here +$ rm -rf * +$ ln ../20/README . +$ cd ../../updates/21 +$ ls # make sure you have stuff here +$ rm -rf * +$ ln ../20/README . +$ cd ../testing/21 +$ ls # make sure you have stuff here +$ rm -rf * +$ ln ../20/README . ---- This should complete the archiving. diff --git a/modules/howtos/pages/build_against_infra_tags.adoc b/modules/howtos/pages/build_against_infra_tags.adoc new file mode 100644 index 0000000..9dcbf66 --- /dev/null +++ b/modules/howtos/pages/build_against_infra_tags.adoc @@ -0,0 +1,16 @@ += How to build against an infra tag in koji + +If you want a build to be available in the infra repo instead of +the fedora official repos. + +---- +$ koji build f30-infra bodhi-5.2.0-1.fc30.infra.src.rpm +---- + +If you get an permission error, you need to make sure you have the infra permission in koji + +---- +$ koji list-permissions --user cverna infra +---- + +If you don't, open an link:https://pagure.io/fedora-infrastructure/issues[infra ticket] to request that permission. diff --git a/modules/howtos/pages/check_robosignatory_production_logs.adoc b/modules/howtos/pages/check_robosignatory_production_logs.adoc new file mode 100644 index 0000000..16b9782 --- /dev/null +++ b/modules/howtos/pages/check_robosignatory_production_logs.adoc @@ -0,0 +1,10 @@ += How to check robosignatory productions logs + +Robosignatory in production does not have ssh enabled so we cannot connect to the box to +check the logs. +However we can use log1.iad2.fedoraproject.org to check the logs of the service. + +---- +$ ssh log01.iad2.fedoraproject.org +$ grep autosign01.iad2.fedoraproject.org /var/log/merged/messages.log +---- diff --git a/modules/howtos/pages/clean_2f_tokens.adoc b/modules/howtos/pages/clean_2f_tokens.adoc new file mode 100644 index 0000000..8f6978a --- /dev/null +++ b/modules/howtos/pages/clean_2f_tokens.adoc @@ -0,0 +1,26 @@ += How to remove 2 factor authentication tokens in IPA + +== How to remove 2 factor authentication tokens in IPA using UI + +. Log into `https://id.fedoraproject.org/ipa/ui/` using FAS credentials + +. Click on the `Authentication` tab + +. Click on the `OTP Tokens` sub tab + +. Enter the username into the search bar. This will display a list of tokens associated with the user. + +. Select the desired token and click delete. A popup will appear, click delete again to confirm. + +== How to remove 2 factor authentication tokens in IPA using cli + +. kinit as user with admin privileges on IPA server + +. Run `ipa otptoken-find --owner=` ++ +A list of the users OTP tokens will be displayed. Copy `Unique ID` vlaue of the desired token + +. Run `ipa otptoken-del ` ++ +The token is now removed + diff --git a/modules/howtos/pages/clean_monitoring_sidetags.adoc b/modules/howtos/pages/clean_monitoring_sidetags.adoc new file mode 100644 index 0000000..ac29b0b --- /dev/null +++ b/modules/howtos/pages/clean_monitoring_sidetags.adoc @@ -0,0 +1,23 @@ += How to clean up the side-tags created by the monitor-gating project + +Monitor-gating runs in OpenShift and monitors the packager workflow end to end. +Doing this for single build and multi-builds updates by gettings few tests +packages go through that workflow. + +If something happens and side-tags accumulates in koji, they can be cleaned +using the following steps: + +. Go to link:OpenShift[https://console-openshift-console.apps.ocp.fedoraproject.org/project-details/ns/monitor-gating] + (this links points directly to the monitor-gating project) + +. Access the running pod + +. Click on the "Terminal" tab to have a shell in that pod + +. Run the following command: + + python3 /opt/code/clean_up_side_tags.py /opt/config/runner.cfg + +It will go through all the side-tags created by this user/keytab and delete +them using `fedpkg remove-side-tag`. It will do this for all but one (which may be +in use by the script when the command is ran). diff --git a/modules/howtos/pages/create_keytab.adoc b/modules/howtos/pages/create_keytab.adoc new file mode 100644 index 0000000..9fa60ae --- /dev/null +++ b/modules/howtos/pages/create_keytab.adoc @@ -0,0 +1,40 @@ += How to create a keytab for an user + +First obtain Kerberos ticket with kinit: + +---- +$ kinit myusername@FEDORAPROJECT.ORG +Password for myusername@FEDORAPROJECT.ORG: +---- + +Then obtain kvno value: + +---- +$ kvno myusername@FEDORAPROJECT.ORG +myusername@FEDORAPROJECT.ORG: kvno = 42 +---- + +Ticket is no longer needed and can be destroyed: + +---- +$ kdestroy -p myusername@FEDORAPROJECT.ORG +---- + +Generate keytab and write it to disk: + +---- +$ ktutil +ktutil: addent -password -p myusername@FEDORAPROJECT.ORG -k 42 -f +Password for myusername@FEDORAPROJECT.ORG: +ktutil: wkt /tmp/kt/fedora +ktutil: q +---- + +Done. You can now use the keytab to obtain the ticket without typing password: + +---- +$ kinit -kt /tmp/kt/fedora myusername@FEDORAPROJECT.ORG +---- + + +(source: https://pagure.io/fedora-infrastructure/issue/9544#comment-706949) diff --git a/modules/howtos/pages/create_new_mailing_list.adoc b/modules/howtos/pages/create_new_mailing_list.adoc new file mode 100644 index 0000000..b7af8bb --- /dev/null +++ b/modules/howtos/pages/create_new_mailing_list.adoc @@ -0,0 +1,13 @@ += Creating a new mailing list + +. Log into mailman01.iad2.fedoraproject.org +. Run the following command: ++ +---- +sudo -u mailman mailman3 create @lists.fedora(project|hosted).org --owner @fedoraproject.org --notify +---- ++ +Note that list names should make sense, and not contain the words `fedora` +or `list` - the fact that it has to do with Fedora and that it's a list +are both obvious from the domain of the email address. + diff --git a/modules/howtos/pages/creating_groups_distgit.adoc b/modules/howtos/pages/creating_groups_distgit.adoc new file mode 100644 index 0000000..3a0c8bf --- /dev/null +++ b/modules/howtos/pages/creating_groups_distgit.adoc @@ -0,0 +1,23 @@ += How to create a group in dist-git + +[NOTE] +==== +You may want to read first xref:groups_in_fedora.adoc[Groups in Fedora] for information on the +different types of groups in Fedora. +==== + +Dist-git does not allow anyone to create groups, only admins can do it via the +`pagure-admin` CLI tool. + +To create a group, you can simply run the command on pkgs01.iad2.fedoraproject.org: ++ +---- +pagure-admin new-group \ + --description="short description" \ + --display="Name used in the UI" +---- + +The group members are then sync upon their login in dist-git (ie: people will +have to log out and back in for their group membership to be refreshed). + +How to give groups from someone to someone else can be found in xref:give_groups_dist_git.adoc[How to give a group from someone to someone else in dist-git]. diff --git a/modules/howtos/pages/delete_mailman_thread.adoc b/modules/howtos/pages/delete_mailman_thread.adoc new file mode 100644 index 0000000..dd2dcad --- /dev/null +++ b/modules/howtos/pages/delete_mailman_thread.adoc @@ -0,0 +1,25 @@ += How to delete a thread in mailman + +[NOTE] +==== +This should only ever be done if requested by management. +Once a thread is deleted it cannot be recovered +==== + + +Go to the link:https://lists.fedoraproject.org/archives/[mailman site] +on the top right corner select `Sign In` + +Do not login through the username and password fields, +login using the third party `Fedora` link to assume FAS permissions. + + +Once logged in select the thread you wish to delete either directly by url +or by searching through the archive. +On the right hand side of the page there will be a box containg the +`Delete this thread` option. +If you do not see this then you don't have the required permissions. +Click this and you will be asked to confirm deletion. + +After deletion try to navigate back to the page to ensure the thread is +removed. diff --git a/modules/howtos/pages/destroy_a_virt_instance.adoc b/modules/howtos/pages/destroy_a_virt_instance.adoc new file mode 100644 index 0000000..b354f34 --- /dev/null +++ b/modules/howtos/pages/destroy_a_virt_instance.adoc @@ -0,0 +1,28 @@ += How to destroy a virt instance + +A playbook is available to destroy a virtual instance. + +---- +sudo -i ansible-playbook -vvv /srv/web/infra/ansible/playbooks/destroy_virt_inst.yml -e target=osbs-master01.stg.iad2.fedoraproject.org +---- + +In some cases the instance that you are trying to delete was not in a clean state. You could then try to run the following: + +. Undefine the instance ++ +---- +sudo -i ssh virthost04.stg.iad2.fedoraproject.org 'virsh undefine osbs-node02.stg.phx2.fedoraproject.org' +---- + +. Remove the logical volume ++ +---- +sudo -i ssh virthost04.stg.iad2.fedoraproject.org 'lvremove /dev/vg_guests/bodhi-backend01.phx2.fedoraproject.org' +---- + +To connect to a virtual instance console you need to first ssh to the virthost box. For example + +---- +sudo -i ssh virthost04.stg.iad2.fedoraproject.org +(virthost04.stg.iad2.fedoraproject.org) virsh console osbs-node02.stg.iad2.fedoraproject.org +---- diff --git a/modules/howtos/pages/discourse_spam.adoc b/modules/howtos/pages/discourse_spam.adoc new file mode 100644 index 0000000..c548e74 --- /dev/null +++ b/modules/howtos/pages/discourse_spam.adoc @@ -0,0 +1,18 @@ += How to deal with spam posts on discourse + +When a ticket is raised for spam user on the Fedora Infra tracker take the +following actions. + +== Disable user in IPA + +=== Frontend + +. Login to the link:https://id.fedoraproject.org/ipa/ui[ipa server] and search the users name. +. Click on the user. +. In the Actions dropdown menu click disable + + +=== Command line + +. ssh to `ipa01.iad2.fedoraproject.org` +. run `ipa user-disable ` diff --git a/modules/howtos/pages/fedora_messaging_certificates.adoc b/modules/howtos/pages/fedora_messaging_certificates.adoc new file mode 100644 index 0000000..b1f78b8 --- /dev/null +++ b/modules/howtos/pages/fedora_messaging_certificates.adoc @@ -0,0 +1,122 @@ += How to create TLS certificates for fedora-messaging + +== Creating TLS Certificates + +In ansible-private, find the files/rabbitmq/ folder. In that is a production +and a staging subdirectory + +. Create the staging certificates: ++ +In the staging subdir, run: ++ +---- +/usr/share/easy-rsa/3/easyrsa build-client-full .stg nopass +---- ++ +For example: ++ +---- +/usr/share/easy-rsa/3/easyrsa build-client-full monitor-gating.stg nopass +---- ++ +[NOTE] +==== +For staging we always make the name `.stg` so that ansible scripts +work with it +==== + +. Create the production certificates: ++ +In the production subdir, run: ++ +---- +/usr/share/easy-rsa/3/easyrsa build-client-full nopass +---- ++ +For example: ++ +---- +/usr/share/easy-rsa/3/easyrsa build-client-full monitor-gating nopass +---- ++ +[NOTE] +==== +No `.stg` here. +==== + +. Add the certificates to the git repo: ++ +Run the usual commands: ++ +---- +git add . +git commit -s -a -m 'Added fedora-messaging certs for ' +git push +---- + +Source: https://pagure.io/fedora-infrastructure/issue/8638 + +== Regenerating TLS certificates + +. Revoke old certificate ++ +In the staging subdir run ++ +---- +/usr/share/easy-rsa/3/easyrsa revoke +---- ++ +Same for production, but without `.stg` in name and the commands need to be run from production subdir + +. Remove the cert from `index.txt` and `index.txt.old` ++ +The easiest way around this is to simply do a ``git grep ``. +It should tell you that the name can be found in `index.txt` (and potentially +`index.txt.old` if another certificate was generated since the first attempt +and yours). + +. Follow the <> + +== Debugging + +If you run into the following error when during generating the certificates: + +---- +failed to update database +TXT_DB error number 2 +---- + +The full output looking something like +---- + +Using SSL: openssl OpenSSL 1.0.2k-fips 26 Jan 2017 +Generating a 2048 bit RSA private key +.+++ +.....+++ +writing new private key to '/..../files/rabbitmq/staging/pki/private/monitor-gating.stg.key.PhSK949Ny8' +\---- +Using configuration from /..../files/rabbitmq/staging/pki/safessl-easyrsa.cnf +Check that the request matches the signature +Signature ok +The Subject's Distinguished Name is as follows +commonName :ASN.1 12:'monitor-gating.stg' +Certificate is to be certified until Feb 9 14:52:07 2023 GMT (1080 days) +failed to update database +TXT_DB error number 2 + +Easy-RSA error: + +signing failed (openssl output above may have more detail) +---- + +This is because you're trying to generate a certificate for a name that already +exists in the database (as explained in: +https://zeldor.biz/2013/11/txt_db-error-number-2-failed-to-update-database/ + +The easiest way around this is to simply do a ``git grep ``. +It should tell you that the name can be found in `index.txt` (and potentially +`index.txt.old` if another certificate was generated since the first attempt +and yours). + +Edit this/these file(s) and remove the line concerning your certificate, then +re-run the `easyrsa` command as above. diff --git a/modules/howtos/pages/fix_bugzilla_aws_saml_login.adoc b/modules/howtos/pages/fix_bugzilla_aws_saml_login.adoc new file mode 100644 index 0000000..cee49a0 --- /dev/null +++ b/modules/howtos/pages/fix_bugzilla_aws_saml_login.adoc @@ -0,0 +1,29 @@ += How to fix SAML login in bugzilla/AWS + +Basically, we've observed that every now and then one of the ipsilon pod looses +its config (or so it seems) which leads to make that pod unable to properly +serve SAML requests. + +Here are the steps to debug and fix this situation: + +. Open all the web pod of Ipsilon in different tabs - open the logs tab for + each +. Open https://bugzilla.redhat.com in a tab using the private browser mode +. Try logging into bugzilla using the Fedora contributor button in the drop-down +. Check the recent logs of the pods and find the one saying something like: ++ +---- +DEBUG(providers/saml2/auth.py:176 Redirect.saml2login()): saml2: 'NoneType' object has no attribute 'get_login_handler' +---- +. Copy the name of that pod +. Go to the openshift's main node +. Delete the pod using: ++ +---- +oc -n ipsilon delete pod `` (for example: ``oc -n ipsilon delete pod ipsilon-24-8ztdd``) +---- ++ +Openshift will automatically spawn a new one + +. Try login on bugzilla again - it should work (or maybe another pod is mis-behaving?) + diff --git a/modules/howtos/pages/fix_robosignatory.adoc b/modules/howtos/pages/fix_robosignatory.adoc new file mode 100644 index 0000000..6324d8c --- /dev/null +++ b/modules/howtos/pages/fix_robosignatory.adoc @@ -0,0 +1,49 @@ += How to check/fix robosignatory + +[NOTE] +==== +sysadmin-main can access robosignatory in staging, but production is only +accessible to a very limited set of people. +==== + +. Check the status of robosignatory: +.. Log into `autosign01.stg.iad2.fedoraproject.org` +.. Check the logs: ++ +---- +journalctl -f -l -u fm-consumer@robosignatory +---- + +.. If the service is not running properly, restart it: ++ +---- +systemctl restart fm-consumer@robosignatory.service +---- + +. Check the status of the signing-vault +.. Log into `sign-vault01.stg.iad2.fedoraproject.org` +.. Check the status of sigul server: ++ +---- +tail -f /var/log/sigul_server.log +---- + +.. If needed, restart the sigul server: ++ +---- +sigul_server -dvv +---- + +. Check the status of the signing-bridge +.. Log into `sign-bridge01.stg.iad2.fedoraproject.org` +.. Check the status of the sigul bridge: ++ +---- +tail -f /var/log/sigul_bridge.log +---- + +.. If needed, restart the sigul bridge: ++ +---- +sigul_bridge -dvv +---- diff --git a/modules/howtos/pages/free_space_in_openshift.adoc b/modules/howtos/pages/free_space_in_openshift.adoc new file mode 100644 index 0000000..a04a33b --- /dev/null +++ b/modules/howtos/pages/free_space_in_openshift.adoc @@ -0,0 +1,61 @@ += How to free some space in OpenShift + +If your builds run into an error looking like: + +---- +error: build error: devmapper: Thin Pool has 11561 free data blocks which is +less than minimum required 11944 free data blocks. Create more free space in +thin pool or use dm.min_free_space option to change behavior +---- + +It is likely that one or several of the nodes has some disk space issues. + +== Check the disk space on the nodes + +You can do this via ansible by simply running: +---- +ansible -a 'lvs' os_nodes_stg +---- + +The output will look somthing like this: +---- + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + root GuestVolGroup00 -wi-ao---- <58.59g + docker-pool vg-docker twi-a-t--- 58.32g 42.00 18.44 + +os-node02.stg.iad2.fedoraproject.org | CHANGED | rc=0 >> + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + root GuestVolGroup00 -wi-ao---- <58.59g + docker-pool vg-docker twi-a-t--- <48.60g 32.37 14.81 + +os-node01.stg.iad2.fedoraproject.org | CHANGED | rc=0 >> + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + root GuestVolGroup00 -wi-ao---- <58.59g + docker-pool vg-docker twi-a-t--- 58.32g 40.75 17.38 + +os-node04.stg.iad2.fedoraproject.org | CHANGED | rc=0 >> + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + root GuestVolGroup00 -wi-ao---- <58.59g + docker-pool vg-docker twi-a-t--- 58.32g 90.32 28.35 +---- + +In this case you can see that `os-node04.stg` has a disk full at 90%. + + +== Free space on a node + +There is a cron job running regularly to clean up docker images that aren't +needed anymore, but you can also run it manually to free some space. + +The command to run is then: +---- +docker rmi $(docker images --filter dangling=true -q) +---- +[NOTE] +==== +This needs to be run on the `os-node` that is running out of space. +==== + + + +Source: https://pagure.io/fedora-infrastructure/issue/8643 diff --git a/modules/howtos/pages/generate_2fa_keys.adoc b/modules/howtos/pages/generate_2fa_keys.adoc new file mode 100644 index 0000000..7917c6c --- /dev/null +++ b/modules/howtos/pages/generate_2fa_keys.adoc @@ -0,0 +1,33 @@ += How to generate 2 Factor Authentication key and certificate + +Doing this requires membership in sysadmin-main FAS group. + +All the following commands should be ran on one of batcave hosts. + +Clone `ansible-private` repo: + +---- +git clone /srv/git/ansible-private +---- + +Change into `files/vpn` subdirectory in cloned repo: + +---- +cd ansible-private/files/2fa-certs +---- + +The process is described in the README but is basically: + +---- +. ./vars; ./build-and-sign-key +---- + +Add generated files to git index, commit, push: + +---- +git status +git add . +git commit -a -m "Add 2 FA key/cert for " +git show +git push +---- diff --git a/modules/howtos/pages/generate_openvpn_keys.adoc b/modules/howtos/pages/generate_openvpn_keys.adoc new file mode 100644 index 0000000..ab81c2f --- /dev/null +++ b/modules/howtos/pages/generate_openvpn_keys.adoc @@ -0,0 +1,33 @@ += How to generate private key and certificate for OpenVPN client + +Doing this requires membership in `sysadmin-main` FAS group. + +All the following commands should be ran on one of batcave hosts. + +Clone `ansible-private` repo: + +---- +git clone /srv/git/ansible-private +---- + +Change into `files/vpn` subdirectory in cloned repo: + +---- +cd ansible-private/files/vpn +---- + +Run `addhost.sh` script to generate keys and cert, eg.: + +---- +./addhost.sh proxy33.fedoraproject.org +---- + +Add generated files to git index, commit, push: + +---- +git status +git add . +git commit -a -m "Add VPN key/cert for proxy33.fedoraproject.org" +git show +git push +---- diff --git a/modules/howtos/pages/get_logs_pod_openshift.adoc b/modules/howtos/pages/get_logs_pod_openshift.adoc new file mode 100644 index 0000000..8958897 --- /dev/null +++ b/modules/howtos/pages/get_logs_pod_openshift.adoc @@ -0,0 +1,15 @@ += How to get logs of a pod in OpenShift + +You'll need the pod's name, the project's name and access to the openshift +master. + +Then you can run the command: + +---- +oc logs -n +---- + +for example: +---- +oc logs toddlers-47-2tjj2 -n toddlers +---- diff --git a/modules/howtos/pages/give_groups_dist_git.adoc b/modules/howtos/pages/give_groups_dist_git.adoc new file mode 100644 index 0000000..efbc76d --- /dev/null +++ b/modules/howtos/pages/give_groups_dist_git.adoc @@ -0,0 +1,12 @@ += How to give a group from someone to someone else in dist-git + +Groups are linked to the user who created it and it used to not be possible to +give the group to another user, so lots of the early groups were created under +the `releng` user, thus allowing to easily remove the user who asked for the +group if they left the project. + +It is nowadays possible to give groups from an user to another. This can be done +by the person who is the current main admin of the group or a pagure admin. + +The URL to give a group is `https://src.fedoraproject.org/group//edit` + diff --git a/modules/howtos/pages/groups_in_fedora.adoc b/modules/howtos/pages/groups_in_fedora.adoc new file mode 100644 index 0000000..c8036b7 --- /dev/null +++ b/modules/howtos/pages/groups_in_fedora.adoc @@ -0,0 +1,102 @@ += Groups in Fedora + +## What are the different types of groups? + +Fedora has a few types of groups: + +* Tracking groups ++ +The tracking groups are primarily meant to indicate that people are contributors +in a certain part of the Fedora Project. +These group will allow new accounts to become CLA+1 which is the way of saying +that someone has signed the FPCA (which used to be called CLA, but the FPCA is +not a CLA, thus the rename) and is a member of one other group. +Being CLA+1 is the criteria the project has set to recognize someone has been +an active contributor. +With this trust comes a few "priviledges" such as: ++ +-- +- Access to the @fedoraproject.org email alias +- Access to fedorapeople.org +- Possibility to vote on some of the elections (including FESCo and the Council) +-- ++ +These groups are also the ones used when one wants to share the maintenance of +COPR projects with other people via a group. + +* Shell groups ++ +These groups are used when contributors gain access to some of the machines +in the Fedora Infrastructure. +For example, members of the sysadmin group are allowed to commit on the +infrastructure's git repository and this is allowed by giving write access to the +git repository to this group, on the file-system. + +* pkgdb groups ++ +These groups are meant to facilitate the collective maintenance of packages in +Fedora. +In other words, by giving commit access to the group on a package (or set of +packages), everyone in the group can commit to the package and thus the +maintenance of the package can be shared amongs the members of the group. + + +== How to request a group to be created? + +The short answer is to open a ticket on the fedora-infrastructure issue tracker: +https://pagure.io/fedora-infrastructure/ + +It is unlikely you will need to create "shell" group, but you can ask for the +creation of a "tracking" or "pkgdb" group. + + +=== COPR groups + +COPR supports giving access to project based on group membership. These groups +are regular `tracking` groups. + + +=== "pkgdb" groups name + +In the days where pkgdb was an application in addition to a type of group, groups +meant to share the collaboration of packages had to end with `-sig`. +This is technically no longer a requirement today, however, it remains a standard +we try to maintain as it makes it easier to identify "pkgdb" group by their +names. + +*When requesting a "pkgdb" group, please try to adhere to this standard.* + + +=== "pkgdb" groups and mailing list + +"pkgdb" groups are created to maintain packages. They are meant to be CC'ed on +bugzilla tickets associated with these packages. +The tools syncing the information from dist-git to bugzilla uses the +"mailing list address" field in the group's settings. + +This means that the email address associated with the group will be notified of +every bug reports opened against all the packages the group maintain, including +security sensitive reports. + +In consequence, it is highly recommended that the email set in the "mailing list +address" in the group's settings in FAS be something that does not have publicly +accessible archives. + +A simple answer to this is to create a new mailing list when creating the "pkgdb" +group. This list can then be made private and invite-only. + +*When requesting a pkgdb group, please let us know if you need a new mailing list +to be created or not.* + +== How to create a group? + +. Log in to link:https://id.fedoraproject.org/ipa/ui[IPA interface] +. Click on the Groups tab +. Click on `+Add` button +. Fill in the Group name, description (optional) +. Tick the FAS group checkbox +. Click Add +. Click on the newly created group +. Add group sponsors (` member managers` -> Users on the right) using `+Add` button +. Add members (if requested) using `+Add` button in ` members` -> Users +. In case of "pkgdb" group you need to also create a mailing list. See xref:create_new_mailing_list.adoc[Creating a new mailing list] for details diff --git a/modules/howtos/pages/make_mailman_user_admin.adoc b/modules/howtos/pages/make_mailman_user_admin.adoc new file mode 100644 index 0000000..d745412 --- /dev/null +++ b/modules/howtos/pages/make_mailman_user_admin.adoc @@ -0,0 +1,21 @@ += Make mailman user an admin + +[NOTE] +==== +You need to be mailman admin to be able to follow this guide. +==== + +Go to the link:https://lists.fedoraproject.org/archives/[mailman site] +on the top right corner select `Sign In` + +Do not login through the username and password fields, login using the +third party `Fedora` link to assume fas permissions. + +Once logged in navigate to the link:https://lists.fedoraproject.org/django-admin[django admin page] +you should see in the top right corner that you are logged in. + +Search for the user who you wish to make admin, click on the name and +tick the `staff status` box. Click the save button at the bottom. +Admin access should now be present for the user. + +--- diff --git a/modules/howtos/pages/rebuild_osbs_buildroot.adoc b/modules/howtos/pages/rebuild_osbs_buildroot.adoc new file mode 100644 index 0000000..b8b6173 --- /dev/null +++ b/modules/howtos/pages/rebuild_osbs_buildroot.adoc @@ -0,0 +1,16 @@ += How to rebuild OSBS buildroot image. + +OSBS build are done from a buildroot container image which is built on the OSBS nodes. +The dockerfile used to build the image are in ansible +link:https://infrastructure.fedoraproject.org/cgit/ansible.git/tree/files/osbs/buildroot-Dockerfile-production.j2[Dockerfile] +for production and +link:https://infrastructure.fedoraproject.org/cgit/ansible.git/tree/files/osbs/buildroot-Dockerfile-staging.j2[Dockerfile] +for staging. + +To rebuild the image you can use the following playbook. + +---- +sudo -i ansible-playbook /srv/web/infra/ansible/playbooks/groups/osbs/rebuild-osbs-buildroot.yml +---- + +This will backup the image currently used and build a new image fetching the latest base image. diff --git a/modules/howtos/pages/refresh_osbs_odcs_oicd_token.adoc b/modules/howtos/pages/refresh_osbs_odcs_oicd_token.adoc new file mode 100644 index 0000000..739f7f2 --- /dev/null +++ b/modules/howtos/pages/refresh_osbs_odcs_oicd_token.adoc @@ -0,0 +1,33 @@ += How to refresh the ODCS OIDC token used by OSBS + +OSBS needs to trigger composes to ODCS for flatpak build, this is done using an OIDC token +to authenticate the requests. +This token expires every 365 days so it needs to be regenerated every year. + +== Generate a new token +In the ansible repo run the following command: + +---- +scripts/generate-oidc-token osbs -e 365 -s https://id.fedoraproject.org/scope/groups -s https://pagure.io/odcs/new-compose -s https://pagure.io/odcs/renew-compose -s https://pagure.io/odcs/delete-compose +---- + +Follow the instructions given by the script and run the SQL command on the ipsilon database server: + +---- +ssh db-fas01.iad2.fedoraproject.org +sudo -u postgres -i ipsilon +ipsilon=# BEGIN; +.... +ipsilon=# COMMIT; +---- + +Save the value of the token generated by the script in the ansible-private repo under +`ansible-private/files/osbs/production/odcs-oidc-token` (same needs to be done for the +staging cluster) + +== Deploy the change + +Run the following playbook to deploy the new token +---- +ansible-playbook /srv/web/infra/ansible/playbooks/groups/osbs/configure-osbs.yml +---- diff --git a/modules/howtos/pages/remove_branch_distgit.adoc b/modules/howtos/pages/remove_branch_distgit.adoc new file mode 100644 index 0000000..5413899 --- /dev/null +++ b/modules/howtos/pages/remove_branch_distgit.adoc @@ -0,0 +1,96 @@ += How to remove a git branch in a dist-git repository + +Historically we did not allow to remove git branches in dist-git repository, but +FESCo recently approved their removal *if and only if* all the commits in the +branch to be deleted can be reached from another branch. +This is a requirement to ensure that if any of the commits were used in a build, +we still have the commit accessible and thus we are able to reproduce the build +if needed. + +There is a script in the releng repository to use to check if a branch can +safely be deleted. + +So here are the steps to follow to remove the branch `` from the package +``. + +. Clone the releng repo if you do not already have it: ++ +---- +git clone https://pagure.io/releng.git +---- + +. Pull the latest changes: ++ +---- +pushd releng && git pull --rebase && popd +---- + +. Clone the `` package locally: ++ +---- +fedpkg clone +---- + +. Checkout the branch: ++ +---- +cd clone && git checkout +---- + +. Run the script: ++ +---- +python ../releng/scripts/distgit-branch-unused.py +---- ++ +(If needed, see the ``--help`` of the script for more information) + + +. If the script returns that the branch is safe to delete: + +.. Go to pkgs01 as root ++ +---- +ssh pkgs01.iad2.fedoraproject.org +---- + +.. Go to the git repository: ++ +---- +cd /srv/git/repositories//.git +---- + +.. Move the head of the branch (this allows to recover it later if needed): ++ +-- + +---- +mv refs/heads/ heads_ +---- + +Sometimes the ref is in the `packed-refs` file, in that case: + +---- +grep packed-refs > heads_ +---- + +Then remove the line from `packed-refs` file +-- + +. On your local clone of ``, check that the branch was deleted upstream: ++ +-- + +---- +git fetch -p +---- + +This should show something like: + +---- +$ git fetch -p +From ssh://pkgs.fedoraproject.org// +- [deleted] (none) -> origin/ +---- + +-- diff --git a/modules/howtos/pages/remove_fedora_user_at_launch_aws.adoc b/modules/howtos/pages/remove_fedora_user_at_launch_aws.adoc new file mode 100644 index 0000000..a0da0f6 --- /dev/null +++ b/modules/howtos/pages/remove_fedora_user_at_launch_aws.adoc @@ -0,0 +1,19 @@ += How to add allow root ssh login and remove fedora user + +This will allow you to run ansible playbooks as the root user. +It also removes the fedora user so that `fas_client` can install sucessfully in `/home/fedora` + +Simply add the below text in to the `User Data` field when launching an EC2 instance. +This appears as a text box at the bottom of the `Configure Instance` step when launching +an instance (Step 3). + + +---- +#!/bin/bash +cat /home/fedora/.ssh/authorized_keys > /root/.ssh/authorized_keys +userdel -r fedora +---- + +The script copies the public key from the fedora user and replaces the root users key with it. +This is the public key which is associated with the keypair selected when launching the instance. +It then deletes the Fedora user leaving root as the only ssh user on the instance. diff --git a/modules/howtos/pages/remove_meeting_minutes_meetbot.adoc b/modules/howtos/pages/remove_meeting_minutes_meetbot.adoc new file mode 100644 index 0000000..e22ad18 --- /dev/null +++ b/modules/howtos/pages/remove_meeting_minutes_meetbot.adoc @@ -0,0 +1,46 @@ += How to remove meeting minutes from meetbot? + +Every now and then, we're asked to remove meeting minutes that meetbot recorded. + +Meetbot records meeting on disk on value01. +The minutes can be found in `/srv/web/meetbot` + +Most often people asking to have some minutes removed, will give you an URL to +that meeting. +You can use it to find the corresponding file, based on this (see above) root +directory. + +For example the url +https://meetbot.fedoraproject.org/teams/workstation/workstation.2020-08-19-18.13.log.html +points to the file `/srv/web/meetbot/teams/workstation/workstation.2020-08-19-18.13.log.html` + + +If you check this file, you will see that it is in fact a symlink to +`/srv/web/meetbot/fedora-meeting-2/2020-08-19/workstation.2020-08-19-18.13.log.html` + +(this will change for each meeting room and date of course). + + +So to remove this meeting we need to: + +. Create a backup in case we've messed things up:: ++ +---- +mkdir meeting_workstation_20200819 +---- +. Move all the minutes from this meeting to this directory:: ++ +---- +mv /srv/web/meetbot/fedora-meeting-2/2020-08-19/workstation.2020-08-19-18.13.* meeting_workstation_20200819 +---- + +. Remove the symlink:: ++ +---- +rm /srv/web/meetbot/teams/workstation/workstation.2020-08-19-18.13.* +---- + +What about the UI? + +Well, Mote seats on the top of the files meetbot generates, so cleaning up the +files on disk will eventually end up cleaning things in the UI. diff --git a/modules/howtos/pages/remove_monitoring_rabbitmq_queue.adoc b/modules/howtos/pages/remove_monitoring_rabbitmq_queue.adoc new file mode 100644 index 0000000..ba9978f --- /dev/null +++ b/modules/howtos/pages/remove_monitoring_rabbitmq_queue.adoc @@ -0,0 +1,40 @@ += How to remove the monitoring of a rabbitmq queue + +The monitoring of queue sizes of rabbitmq queue is done by nagios using two +small configuration file. + +To remove that monitoring, you simply need to remove these two files. + +== On the rabbitmq server (rabbitmq01) + +. Remove the local nagios configuration file ++ +---- +rm /etc/nrpe.d/check_rabbitmq_queue_.cfg +---- + +. Then restart the local nagios plugin ++ +---- +systemctl restart nrpe +---- + +== On the nagios server (noc01) + +. Remove the nagios configuration file ++ +---- +rm /etc/nagios/services/rabbitmq-queue-.cfg +---- + +. Ensure the configuration is still valid: ++ +---- +nagios -v /etc/nagios/nagios.cfg +---- + +. If and only *if* the check above passed, restart nagios on the nagios server ++ +---- +systemctl restart nagios +---- diff --git a/modules/howtos/pages/remove_user_from_watchlist_pagure.adoc b/modules/howtos/pages/remove_user_from_watchlist_pagure.adoc new file mode 100644 index 0000000..3e5ffee --- /dev/null +++ b/modules/howtos/pages/remove_user_from_watchlist_pagure.adoc @@ -0,0 +1,28 @@ += How to remove someone from a watch list on Pagure + +Log on the host running pagure (src.fedoraproject.org or pagure.io, ie: pkgs02 +or pagure01) and run the following command: + +---- +pagure-admin update-watch +---- + +For example: +---- +pagure-admin update-watch rpms/pcp lberk +---- + +It will then ask you which watch status you want to set to the user. In the +example below, the status is "unwatch everything" which means the user won't be +notified of anything unless explicitely @mentioned or for ticket they opened. + +---- +The watch status can be one of the following: +1: watch issues and PRs +0: unwatch, don't notify the user of anything +3: watch issues, PRs and commits +2: watch commits +-1: reset the watch status to default +Status:0 +Updating watch status of lberk to 0 (unwatch, don't notify the user of anything) on rpms/pcp +---- diff --git a/modules/howtos/pages/restart_sigul_bridge.adoc b/modules/howtos/pages/restart_sigul_bridge.adoc new file mode 100644 index 0000000..2117af8 --- /dev/null +++ b/modules/howtos/pages/restart_sigul_bridge.adoc @@ -0,0 +1,14 @@ += How to restart the sigul bridge + +The sigul bridge tends to grow in memory leading to an nagios alert about Low swap. + +To fix that you can restart the bridge. + +From `batcave01`: + +---- +$ sudo -i ssh sign-bridge01.phx2.fedoraproject.org +# pkill -f sigul -9; sigul_bridge -d -v -v; tail -f /var/log/sigul_bridge.log +---- + +Wait for the service to restart. diff --git a/modules/howtos/pages/scale_up_or_down_deployment.adoc b/modules/howtos/pages/scale_up_or_down_deployment.adoc new file mode 100644 index 0000000..e01e0bc --- /dev/null +++ b/modules/howtos/pages/scale_up_or_down_deployment.adoc @@ -0,0 +1,27 @@ += How to scale up/down a deployment in OpenShift + +. Log into `os-master01` (prod or stg depending on what you want) + +. Retrieve the list of projects if you don't know how it's named ++ +---- +oc projects +---- + +. Find the one of interest + +.. Get its deployments ++ +---- +oc -n get dc +---- ++ +This will return you the different deployments in the project. + +.. Scale the deployment up/down +---- +oc -n scale dc/ --replicas= +---- + +If you do not want any running pods, simply use `--replicas=0` + diff --git a/modules/howtos/pages/share_tmux_session.adoc b/modules/howtos/pages/share_tmux_session.adoc new file mode 100644 index 0000000..6d6013d --- /dev/null +++ b/modules/howtos/pages/share_tmux_session.adoc @@ -0,0 +1,31 @@ += How to share a tmux session accross users + +It can be handy to share a tmux session between different user accounts, for +example to allow a someone to look over your shoulder while you work on +something. + +To do this, we need to have a folder with group permission shared between the +two persons. + +In the Fedora infrastructure we can easily use the `fi-apprentice` group for +this. + +So we have created the folder `/var/tmux/`, which is writable for both the `fi-apprentice` and +`sysadmin` groups + + +You can then create a tmux session using: + +---- +tmux -S /var/tmux/ new -s +---- +(The socket and session names can be different, but they don't have to) + +And the other person(s) can join it using: + +---- +tmux -S /var/tmux/ att -t +---- + +Once the person who created the shared session stops it, the people who have +joined will be kicked out of the session as well. diff --git a/modules/howtos/pages/unblock_bodhi_rawhide_updates.adoc b/modules/howtos/pages/unblock_bodhi_rawhide_updates.adoc new file mode 100644 index 0000000..397fdcc --- /dev/null +++ b/modules/howtos/pages/unblock_bodhi_rawhide_updates.adoc @@ -0,0 +1,18 @@ += How to unblock Bodhi rawhide updates + +In the case where the rawhide updates in Bodhi are stuck in testing, it often means +that one of the following process is not working properly. + +Either the `bodhi-celery-beat` pod in openshift (used to schedule tasks at regular interval) +is not triggering the `approve_testing_task` or the `bodhi-celery` pod is failing to +process that task. + +To unblock these update one can restart these pods. To do so ssh to +the `os-master01` hosts and run the following commands + +---- +$ oc -n bodhi rollout latest dc/bodhi-celery-beat +$ oc -n bodhi rollout latest dc/bodhi-celery +---- + +Note using the `rollout` command allow us not to have any downtime. diff --git a/modules/howtos/pages/update_watch_dist_git.adoc b/modules/howtos/pages/update_watch_dist_git.adoc new file mode 100644 index 0000000..e66469d --- /dev/null +++ b/modules/howtos/pages/update_watch_dist_git.adoc @@ -0,0 +1,61 @@ += How to udpate the watch status of someone in dist-git + +== How to see someone's watch status + +There are two ways you can check this: + +* Check the API ++ +-- +Visit https://src.fedoraproject.org/api/0///watchers + +For example: https://src.fedoraproject.org/api/0/rpms/borgbackup/watchers + +It will give you information about all the watchers and what they watch. +-- + +* Using pagure-admin ++ +-- +On the pagure host, run: +---- +pagure-admin get-watch / +---- + +For example: +---- +pagure-admin get-watch rpms/borgbackup bpereto +---- + +It will return you the current watch status of that user on that project (or None if it is the +default status). +-- + +== How do you update status + +* Using pagure-admin ++ +-- +On the pagure host, run: +---- +pagure-admin update-watch / +---- +You will be prompted with options about which status to set, use `-1` to +revert to the default status. + +Example: + +---- +# pagure-admin update-watch rpms/borgbackup bpereto +Using configuration file `/etc/pagure/pagure.cfg` +The watch status can be one of the following: +-1: reset the watch status to default +0: unwatch, don't notify the user of anything +1: watch issues and PRs +2: watch commits +3: watch issues, PRs and commits +Status:-1 +Updating watch status of bpereto to -1 (reset the watch status to default) on rpms/borgbackup +---- + +-- diff --git a/modules/sysadmin_guide/nav.adoc b/modules/sysadmin_guide/nav.adoc index 5f526ce..e80b3d0 100644 --- a/modules/sysadmin_guide/nav.adoc +++ b/modules/sysadmin_guide/nav.adoc @@ -1,126 +1,4 @@ * xref:orientation.adoc[Orientation for Sysadmin Guide] * xref:index.adoc[Sysadmin Guide] -** xref:2-factor.adoc[Two factor auth] -** xref:accountdeletion.adoc[Account Deletion SOP] -** xref:anitya.adoc[Anitya Infrastructure SOP] -** xref:ansible.adoc[ansible - SOP] -** xref:apps-fp-o.adoc[apps-fp-o - SOP] -** xref:archive-old-fedora.adoc[How to Archive Old Fedora Releases - SOP] -** xref:arm.adoc[Fedora ARM Infrastructure - SOP] -** xref:aws-access.adoc[Amazon Web Services Access - SOP] -** xref:bastion-hosts-info.adoc[Fedora Bastion Hosts - SOP] -** xref:blockerbugs.adoc[Blockerbugs Infrastructure - SOP] -** xref:bodhi-deploy.adoc[Bodhi Infrastructure - Deployment SOP] -** xref:bodhi.adoc[Bodhi Infrastructure - Releng SOP] -** xref:bugzilla.adoc[Bugzilla Sync Infrastructure - SOP] -** xref:bugzilla2fedmsg.adoc[bugzilla2fedmsg - SOP] -** xref:collectd.adoc[Collectd - SOP] -** xref:compose-tracker.adoc[Compose Tracker - SOP] -** xref:contenthosting.adoc[Content Hosting Infrastructure - SOP] -** xref:copr.adoc[Copr - SOP] -** xref:coreos-cincinnati.adoc[CoreOS Cincinnati - SOP] -** xref:database.adoc[Database Infrastructure - SOP] -** xref:datanommer.adoc[datanommer - SOP] -** xref:debuginfod.adoc[Fedora Debuginfod Service - SOP] -** xref:departing-admin.adoc[Departing admin - SOP] -** xref:dnf-counting.adoc[DNF Counting - SOP] -** xref:dns.adoc[DNS repository for fedoraproject - SOP] -** xref:docs.fedoraproject.org.adoc[Docs - SOP] -** xref:externally-hosted-services.adoc[Externally Hosted Services - SOP] -** xref:fas-notes.adoc[Fedora Account System - SOP] -** xref:fas-openid.adoc[FAS-OpenID - SOP] -** xref:fedmsg-certs.adoc[fedmsg (Fedora Messaging) Certs, Keys, and CA - SOP] -** xref:fedmsg-gateway.adoc[fedmsg-gateway - SOP] -** xref:fedmsg-introduction.adoc[fedmsg introduction and basics - SOP] -** xref:fedmsg-irc.adoc[fedmsg-irc - SOP] -** xref:fedmsg-new-message-type.adoc[Adding a new fedmsg message type - SOP] -** xref:fedmsg-relay.adoc[fedmsg-relay - SOP] -** xref:fedmsg-websocket.adoc[WebSocket - SOP] -** xref:fedocal.adoc[Fedocal - SOP] -** xref:fedora-releases.adoc[Fedora Release Infrastructure - SOP] -** xref:fedorawebsites.adoc[Websites Release - SOP] -** xref:fmn.adoc[FedMsg Notifications (FMN) - SOP] -** xref:gather-easyfix.adoc[Fedora gather easyfix - SOP] -** xref:gdpr_delete.adoc[GDPR Delete - SOP] -** xref:gdpr_sar.adoc[GDPR SAR - SOP] -** xref:geoip-city-wsgi.adoc[geoip-city-wsgi - SOP] -** xref:github.adoc[Using github for Infra Projects - SOP] -** xref:github2fedmsg.adoc[github2fedmsg - SOP] -** xref:greenwave.adoc[Greenwave - SOP] -** xref:guest_migrate.adoc[Migrate Guest VMs - SOP] -** xref:guestdisk.adoc[Guest Disk Resize - SOP] -** xref:guestedit.adoc[Guest Editing - SOP] -** xref:haproxy.adoc[Haproxy Infrastructure - SOP] -** xref:hotfix.adoc[HOTFIXES - SOP] -** xref:hotness.adoc[The New Hotness - SOP] -** xref:infra-git-repo.adoc[Infrastructure Git Repos - SOP] -** xref:infra-hostrename.adoc[Infrastructure Host Rename - SOP] -** xref:infra-raidmismatch.adoc[Infrastructure Raid Mismatch Count - SOP] -** xref:infra-repo.adoc[Infrastructure Yum Repo - SOP] -** xref:infra-retiremachine.adoc[Infrastructure retire machine - SOP] -** xref:infra_handover.adoc[Initiative Handover - SOP] -** xref:ipsilon.adoc[Ipsilon Infrastructure - SOP] -** xref:iscsi.adoc[iSCSI - SOP] -** xref:jenkins-fedmsg.adoc[Jenkins Fedmsg - SOP] -** xref:kerneltest-harness.adoc[Kerneltest-harness - SOP] -** xref:kickstarts.adoc[Kickstart Infrastructure - SOP] -** xref:koji-archive.adoc[Koji Archive - SOP] -** xref:koji-builder-setup.adoc[Setup Koji Builder - SOP] -** xref:koji.adoc[Koji Infrastructure - SOP] -** xref:koschei.adoc[Koschei - SOP] -** xref:layered-image-buildsys.adoc[Layered Image Build System - SOP] -** xref:mailman.adoc[Mailman Infrastructure - SOP] -** xref:making-ssl-certificates.adoc[SSL Certificate Creation - SOP] -** xref:massupgrade.adoc[Mass Upgrade Infrastructure - SOP] -** xref:mastermirror.adoc[Master Mirror Infrastructure - SOP] -** xref:mbs.adoc[Module Build Service Infra - SOP] -** xref:memcached.adoc[Memcached Infrastructure - SOP] -** xref:message-tagging-service.adoc[Message Tagging Service - SOP] -** xref:mini_initiatives.adoc[Mini initiative Process - SOP] -** xref:mirrorhiding.adoc[Mirror Hiding Infrastructure - SOP] -** xref:mirrormanager-S3-EC2-netblocks.adoc[AWS Mirrors - SOP] -** xref:mirrormanager.adoc[MirrorManager Infrastructure - SOP] -** xref:mote.adoc[mote - SOP] -** xref:nagios.adoc[Fedora Infrastructure Nagios - SOP] -** xref:netapp.adoc[Netapp Infrastructure - SOP] -** xref:new-hosts.adoc[DNS Host Addition - SOP] -** xref:nonhumanaccounts.adoc[Non-human Accounts Infrastructure - SOP] -** xref:nuancier.adoc[Nuancier - SOP] -** xref:ocp4:sops.adoc[Openshift SOPs] -** xref:odcs.adoc[On Demand Compose Service - SOP] -** xref:openqa.adoc[OpenQA Infrastructure - SOP] -** xref:openvpn.adoc[OpenVPN - SOP] -** xref:outage.adoc[Outage Infrastructure - SOP] -** xref:packagereview.adoc[Package Review - SOP] -** xref:pagure.adoc[Pagure Infrastructure - SOP] -** xref:pdc.adoc[PDC - SOP] -** xref:pesign-upgrade.adoc[Pesign upgrades/reboots - SOP] -** xref:planetsubgroup.adoc[Planet Subgroup Infrastructure - SOP] -** xref:publictest-dev-stg-production.adoc[Fedora Infrastructure Machine Classes - SOP] -** xref:rabbitmq.adoc[RabbitMQ - SOP] -** xref:rdiff-backup.adoc[rdiff-backup - SOP] -** xref:registry.adoc[Container registry - SOP] -** xref:requestforresources.adoc[Request for resources - SOP] -** xref:resultsdb.adoc[ResultsDB - SOP] -** xref:retrace.adoc[Retrace - SOP] -** xref:scmadmin.adoc[SCM Admin - SOP] -** xref:selinux.adoc[SELinux Infrastructure - SOP] -** xref:sigul-upgrade.adoc[Sigul servers upgrades/reboots - SOP] -** xref:simple_koji_ci.adoc[simple_koji_ci - SOP] -** xref:sshaccess.adoc[SSH Access Infrastructure - SOP] -** xref:sshknownhosts.adoc[SSH known hosts Infrastructure - SOP] -** xref:staging.adoc[Staging - SOP] -** xref:status-fedora.adoc[Fedora Status Service - SOP] -** xref:syslog.adoc[Log Infrastructure - SOP] -** xref:tag2distrepo.adoc[Tag2DistRepo Infrastructure - SOP] -** xref:tickets.adoc[How to handle new tickets in fedora-infrastructure - SOP] -** xref:unbound.adoc[Fedora Infra Unbound Notes - SOP] -** xref:virt-image.adoc[Fedora Infrastructure Kpartx Notes - SOP] -** xref:virt-notes.adoc[Fedora Infrastructure Libvirt Notes - SOP] -** xref:virtio.adoc[Virtio Notes - SOP] -** xref:voting.adoc[Voting Infrastructure - SOP] -** xref:waiverdb.adoc[WaiverDB - SOP] -** xref:wcidff.adoc[What Can I Do For Fedora - SOP] -** xref:wiki.adoc[Wiki Infrastructure - SOP] -** xref:zabbix.adoc[Zabbix Server Infrastructure - SOP] -** xref:zodbot.adoc[Zodbot Infrastructure - SOP] +** xref:index.adoc#_standard_operating_procedures[Standard Operation Procedures] +** xref:index.adoc#_howtos[HOWTOs] diff --git a/modules/sysadmin_guide/pages/index.adoc b/modules/sysadmin_guide/pages/index.adoc index 41665d0..e0f5d7a 100644 --- a/modules/sysadmin_guide/pages/index.adoc +++ b/modules/sysadmin_guide/pages/index.adoc @@ -69,3 +69,168 @@ Below is a table of contents containing all the standard operating procedures for Fedora Infrastructure applications. For information on how to write a new standard operating procedure, consult the guide on xref:developer_guide:sops.adoc[Developing Standard Operating Procedures]. + +* xref:accountdeletion.adoc[Account Deletion SOP] +* xref:fedmsg-new-message-type.adoc[Adding a new fedmsg message type] +* xref:anitya.adoc[Anitya Infrastructure SOP] +* xref:ansible.adoc[Ansible] +* xref:apps-fp-o.adoc[apps.fedoraproject.org] +* xref:arm.adoc[ARM Infrastructure] +* xref:aws-access.adoc[Amazon Web Services Access] +* xref:mirrormanager-S3-EC2-netblocks.adoc[Amazon Web Services Mirrors] +* xref:bastion-hosts-info.adoc[Bastion Hosts] +* xref:blockerbugs.adoc[Blockerbugs Infrastructure] +* xref:bodhi.adoc[Bodhi Infrastructure - Releng] +* xref:bodhi-deploy.adoc[Bodhi Infrastructure - Deployment] +* xref:bugzilla.adoc[Bugzilla Sync Infrastructure] +* xref:bugzilla2fedmsg.adoc[bugzilla2fedmsg] +* xref:collectd.adoc[Collectd] +* xref:compose-tracker.adoc[Compose Tracker] +* xref:registry.adoc[Container registry] +* xref:contenthosting.adoc[Content Hosting Infrastructure] +* xref:copr.adoc[Copr] +* xref:coreos-cincinnati.adoc[CoreOS Cincinnati] +* xref:database.adoc[Database Infrastructure] +* xref:datanommer.adoc[Datanommer] +* xref:debuginfod.adoc[Debuginfod Service] +* xref:departing-admin.adoc[Departing admin] +* xref:dnf-counting.adoc[DNF Counting] +* xref:new-hosts.adoc[DNS Host Addition] +* xref:dns.adoc[DNS repository for fedoraproject] +* xref:docs.fedoraproject.org.adoc[Docs] +* xref:externally-hosted-services.adoc[Externally Hosted Services] +* xref:fas-notes.adoc[Fedora Account System] +* xref:fas-openid.adoc[FAS-OpenID] +* xref:fedmsg-certs.adoc[fedmsg (Fedora Messaging) Certs, Keys, and CA] +* xref:fedmsg-gateway.adoc[fedmsg-gateway] +* xref:fedmsg-introduction.adoc[fedmsg introduction and basics] +* xref:fedmsg-irc.adoc[fedmsg-irc] +* xref:fedmsg-relay.adoc[fedmsg-relay] +* xref:fedocal.adoc[Fedocal] +* xref:fedora-releases.adoc[Fedora Release Infrastructure] +* xref:fmn.adoc[FedMsg Notifications (FMN)] +* xref:gather-easyfix.adoc[Fedora gather easyfix] +* xref:status-fedora.adoc[Fedora Status Service] +* xref:gdpr_delete.adoc[GDPR Delete] +* xref:gdpr_sar.adoc[GDPR SAR] +* xref:geoip-city-wsgi.adoc[geoip-city-wsgi] +* xref:github.adoc[Using github for Infra Projects] +* xref:github2fedmsg.adoc[github2fedmsg] +* xref:greenwave.adoc[Greenwave] +* xref:guestdisk.adoc[Guest Disk Resize] +* xref:guestedit.adoc[Guest Editing] +* xref:guest_migrate.adoc[Migrate Guest VMs] +* xref:haproxy.adoc[Haproxy Infrastructure] +* xref:hotfix.adoc[HOTFIXES] +* xref:tickets.adoc[How to handle new tickets in fedora-infrastructure] +* xref:infra-git-repo.adoc[Infrastructure Git Repos] +* xref:infra-hostrename.adoc[Infrastructure Host Rename] +* xref:infra_handover.adoc[Initiative Handover] +* xref:infra-raidmismatch.adoc[Infrastructure Raid Mismatch Count] +* xref:infra-repo.adoc[Infrastructure Yum Repo] +* xref:infra-retiremachine.adoc[Infrastructure retire machine] +* xref:ipsilon.adoc[Ipsilon Infrastructure] +* xref:iscsi.adoc[iSCSI] +* xref:jenkins-fedmsg.adoc[Jenkins Fedmsg] +* xref:kerneltest-harness.adoc[Kerneltest-harness] +* xref:kickstarts.adoc[Kickstart Infrastructure] +* xref:koji-archive.adoc[Koji Archive] +* xref:virt-image.adoc[Kpartx Notes] +* xref:koji.adoc[Koji Infrastructure] +* xref:koschei.adoc[Koschei] +* xref:layered-image-buildsys.adoc[Layered Image Build System] +* xref:virt-notes.adoc[Libvirt Notes] +* xref:syslog.adoc[Log Infrastructure] +* xref:publictest-dev-stg-production.adoc[Machine Classes] +* xref:mailman.adoc[Mailman Infrastructure] +* xref:massupgrade.adoc[Mass Upgrade Infrastructure] +* xref:mastermirror.adoc[Master Mirror Infrastructure] +* xref:mbs.adoc[Module Build Service Infra] +* xref:memcached.adoc[Memcached Infrastructure] +* xref:message-tagging-service.adoc[Message Tagging Service] +* xref:mini_initiatives.adoc[Mini initiative Process] +* xref:mirrorhiding.adoc[Mirror Hiding Infrastructure] +* xref:mirrormanager.adoc[MirrorManager Infrastructure] +* xref:mote.adoc[mote] +* xref:nagios.adoc[Nagios] +* xref:netapp.adoc[Netapp Infrastructure] +* xref:nonhumanaccounts.adoc[Non-human Accounts Infrastructure] +* xref:nuancier.adoc[Nuancier] +* xref:ocp4:sops.adoc[Openshift SOPs] +* xref:odcs.adoc[On Demand Compose Service] +* xref:openqa.adoc[OpenQA Infrastructure] +* xref:openvpn.adoc[OpenVPN] +* xref:outage.adoc[Outage Infrastructure] +* xref:packagereview.adoc[Package Review] +* xref:pagure.adoc[Pagure Infrastructure] +* xref:pdc.adoc[PDC] +* xref:pesign-upgrade.adoc[Pesign upgrades/reboots] +* xref:planetsubgroup.adoc[Planet Subgroup Infrastructure] +* xref:rabbitmq.adoc[RabbitMQ] +* xref:rdiff-backup.adoc[rdiff-backup] +* xref:requestforresources.adoc[Request for resources] +* xref:resultsdb.adoc[ResultsDB] +* xref:retrace.adoc[Retrace] +* xref:scmadmin.adoc[SCM Admin] +* xref:selinux.adoc[SELinux Infrastructure] +* xref:koji-builder-setup.adoc[Setup Koji Builder] +* xref:sigul-upgrade.adoc[Sigul servers upgrades/reboots] +* xref:simple_koji_ci.adoc[simple_koji_ci] +* xref:sshaccess.adoc[SSH Access Infrastructure] +* xref:sshknownhosts.adoc[SSH known hosts Infrastructure] +* xref:making-ssl-certificates.adoc[SSL Certificate Creation] +* xref:staging.adoc[Staging] +* xref:tag2distrepo.adoc[Tag2DistRepo Infrastructure] +* xref:hotness.adoc[The New Hotness] +* xref:2-factor.adoc[Two factor auth] +* xref:unbound.adoc[Unbound Notes] +* xref:virtio.adoc[Virtio Notes] +* xref:voting.adoc[Voting Infrastructure] +* xref:waiverdb.adoc[WaiverDB] +* xref:fedorawebsites.adoc[Websites Release] +* xref:fedmsg-websocket.adoc[WebSocket] +* xref:wcidff.adoc[What Can I Do For Fedora] +* xref:wiki.adoc[Wiki Infrastructure] +* xref:zodbot.adoc[Zodbot Infrastructure] + +== HOWTOs + +In this section is list of guides for common tasks that are done in Fedora Infrastructure. + +* xref:howtos:access_rabbitmq_ui.adoc[How to access the rabbitmq administrative UI] +* xref:howtos:archive-old-fedora.adoc[How to Archive Old Fedora Releases] +* xref:howtos:activate_a_spamcheck_account.adoc[How to activate a FAS account that was marked as spamcheck_awaiting] +* xref:howtos:add_a_package_to_infra_tag.adoc[How to add a package to an infra tag] +* xref:howtos:add_external_hardware_to_vpn.adoc[Add external servers to vpn] +* xref:howtos:build_against_infra_tags.adoc[How to build against an infra tag in koji] +* xref:howtos:check_robosignatory_production_logs.adoc[How to check robosignatory productions logs] +* xref:howtos:clean_2f_tokens.adoc[How to remove 2 factor authentication tokens in IPA] +* xref:howtos:clean_monitoring_sidetags.adoc[How to clean up the side-tags created by the monitor-gating project] +* xref:howtos:create_keytab.adoc[How to create a keytab for an user] +* xref:howtos:create_new_mailing_list.adoc[Creating a new mailing list] +* xref:howtos:creating_groups_distgit.adoc[How to create a group in dist-git] +* xref:howtos:delete_mailman_thread.adoc[How to delete a thread in mailman] +* xref:howtos:destroy_a_virt_instance.adoc[How to destroy a virt instance] +* xref:howtos:discourse_spam.adoc[How to deal with spam posts on discourse] +* xref:howtos:fedora_messaging_certificates.adoc[How to create TLS certificates for fedora-messaging] +* xref:howtos:fix_bugzilla_aws_saml_login.adoc[How to fix SAML login in bugzilla/AWS] +* xref:howtos:fix_robosignatory.adoc[How to check/fix robosignatory] +* xref:howtos:free_space_in_openshift.adoc[How to free some space in OpenShift] +* xref:howtos:generate_2fa_keys.adoc[How to generate 2 Factor Authentication key and certificate] +* xref:howtos:generate_openvpn_keys.adoc[How to generate private key and certificate for OpenVPN client] +* xref:howtos:get_logs_pod_openshift.adoc[How to get logs of a pod in OpenShift] +* xref:howtos:give_groups_dist_git.adoc[How to give a group from someone to someone else in dist-git] +* xref:howtos:groups_in_fedora.adoc[Groups in Fedora] +* xref:howtos:make_mailman_user_admin.adoc[Make mailman user an admin] +* xref:howtos:rebuild_osbs_buildroot.adoc[How to rebuild OSBS buildroot image] +* xref:howtos:refresh_osbs_odcs_oicd_token.adoc[How to refresh the ODCS OIDC token used by OSBS] +* xref:howtos:remove_meeting_minutes_meetbot.adoc[How to remove meeting minutes from meetbot] +* xref:howtos:remove_monitoring_rabbitmq_queue.adoc[How to remove the monitoring of a rabbitmq queue] +* xref:howtos:remove_branch_distgit.adoc[How to remove a git branch in a dist-git repository] +* xref:howtos:remove_fedora_user_at_launch_aws.adoc[How to add allow root ssh login and remove fedora user] +* xref:howtos:remove_user_from_watchlist_pagure.adoc[How to remove someone from a watch list on Pagure] +* xref:howtos:restart_sigul_bridge.adoc[How to restart the sigul bridge] +* xref:howtos:scale_up_or_down_deployment.adoc[How to scale up/down a deployment in OpenShift] +* xref:howtos:share_tmux_session.adoc[How to share a tmux session accross users] +* xref:howtos:unblock_bodhi_rawhide_updates.adoc[How to unblock Bodhi rawhide updates] +* xref:howtos:update_watch_dist_git.adoc[How to udpate the watch status of someone in dist-git]