Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Add a flatpak-indexer openshift service

Browse source 
flatpak-indexer replaces regindexer for creating an index of Fedora
Flatpaks. It adds an additional capability - creating "diffs" between image versions
allowing for incremental updates.

Add a new openshift namespace: flatpak-indexer, with three deploymentconfigs
in it:

 - flatpak-indexer: generates the index
 - flatpak-indexer-differ: worker(s) to run the expensive tardiff operation
 - redis: used for cache and communication between indexer and differ

The staging version of the indexer targets the *production* bodhi/koji/registry,
since we don't have useful Flatpak content in staging. This could be changed.

The registry reverse proxy configuration is updated to a slightly different
set of generated indexes (the 'annotations' indexes for F31 and older are
now suffixed with -annotations, and the 'labels' indexes unsuffixed.)

Signed-off-by: Owen W. Taylor <otaylor@fishsoup.net>
This commit is contained in:
Owen W. Taylor 2020-09-11 13:54:25 -04:00 • committed by kevin
parent c349a7c25e
commit 69e318a10a
14 changed files with 636 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

2
playbooks/groups/sundries.yml
View file

@ -27,8 +27,6 @@
- { role: geoip-city-wsgi/app, when: ansible_distribution_major_version|int <= 7 and ansible_distribution == 'RedHat' }
- role: easyfix/gather
when: master_sundries_node|bool
- role: regindexer/build
when: master_sundries_node|bool
- role: bz_review_report
when: master_sundries_node|bool and env != "staging"
- rsyncd

57
playbooks/openshift-apps/flatpak-indexer.yml Normal file
View file

@ -0,0 +1,57 @@
- name: make the app be real
hosts: os_masters_stg[0]
user: root
gather_facts: False
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
- /srv/web/infra/ansible/roles/openshift-apps/flatpak-indexer/vars/{{ env }}.yml
roles:
- role: openshift/project
app: flatpak-indexer
description: Fedora Flatpak Indexer apps
appowners:
- otaylor
- role: openshift/object
app: flatpak-indexer
template: secret.yml
objectname: secret.yml
- role: openshift/object
app: flatpak-indexer
file: imagestream.yml
objectname: imagestream.yml
- role: openshift/object
app: flatpak-indexer
template: buildconfig.yml
objectname: buildconfig.yml
- role: openshift/object
app: flatpak-indexer
file: storage.yml
objectname: storage.yml
- role: openshift/object
app: flatpak-indexer
template: configmap.yml
objectname: configmap.yml
- role: openshift/start-build
app: flatpak-indexer
# This will trigger the main build via a imageChange trigger
buildname: flatpak-indexer-tardiff-build
- role: openshift/object
app: flatpak-indexer
file: service.yml
objectname: service.yml
- role: openshift/object
app: flatpak-indexer
template: deploymentconfig.yml
objectname: deploymentconfig.yml

38
roles/httpd/reverseproxy/templates/reversepassproxy.registry-generic.conf
View file

@ -32,37 +32,37 @@ RewriteRule ^/v2/(.*)$ http://oci-registry01:5000/v2/$1 [P,L]
{% endif %}
RewriteRule ^/v2/(.*)$ http://localhost:10048/v2/$1 [P,L]
RewriteCond "&%{QUERY_STRING}" &annotation(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &label(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &tag=testing
RewriteCond "&%{QUERY_STRING}" &architecture=([^&]+)
RewriteRule "^/index/static$" /index/flatpak-testing-%1.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &label(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &annotation(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &tag=testing
RewriteCond "&%{QUERY_STRING}" &architecture=([^&]+)
RewriteRule "^/index/static$" /index/flatpak-testing-%1-labels.json [L,PT]
RewriteRule "^/index/static$" /index/flatpak-testing-%1-annotations.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &annotation(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &label(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &architecture=([^&]+)
RewriteRule "^/index/static$" /index/flatpak-%1.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &label(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &architecture=([^&]+)
RewriteRule "^/index/static$" /index/flatpak-%1-labels.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &annotation(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &architecture=([^&]+)
RewriteRule "^/index/static$" /index/flatpak-%1-annotations.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &label(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &tag=testing
RewriteRule "^/index/static$" /index/flatpak-testing.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &label(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &tag=testing
RewriteRule "^/index/static$" /index/flatpak-testing-labels.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &annotation(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteCond "&%{QUERY_STRING}" &tag=testing
RewriteRule "^/index/static$" /index/flatpak-testing-annotations.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &label(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteRule "^/index/static$" /index/flatpak.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &label(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteRule "^/index/static$" /index/flatpak-labels.json [L,PT]
RewriteCond "&%{QUERY_STRING}" &annotation(%3A|:)org.flatpak.ref(%3A|:)exists=1
RewriteRule "^/index/static$" /index/flatpak-annotations.json [L,PT]
DocumentRoot /srv/web/registry-index/
@ -87,6 +87,7 @@ SSLOptions +FakeBasicAuth
Alias "/index/" "/srv/web/registry-index/index/"
Alias "/app-icons/" "/srv/web/registry-index/icons/"
Alias "/deltas/" "/srv/web/registry-index/deltas/"
<Directory “/srv/web/registry-index/index/">
Options +FollowSymLinks
@ -106,3 +107,12 @@ Alias "/app-icons/" "/srv/web/registry-index/icons/"
Options +Indexes
Require all granted
</Directory>
<Directory "/srv/web/registry-index/deltas/">
ExpiresActive on
ExpiresDefault "access plus 1 year"
AllowOverride None
Options +Indexes
Require all granted
</Directory>

66
roles/openshift-apps/flatpak-indexer/files/imagestream.yml Normal file
View file

@ -0,0 +1,66 @@
---
apiVersion: v1
kind: ImageStream
metadata:
name: flatpak-indexer
---
apiVersion: v1
kind: ImageStream
metadata:
name: flatpak-indexer-tardiff
---
apiVersion: v1
kind: ImageStream
metadata:
name: redis
---
apiVersion: v1
kind: ImageStream
metadata:
name: fedora
spec:
lookupPolicy:
local: false
tags:
- name: latest
from:
kind: DockerImage
name: registry.fedoraproject.org/fedora:latest
importPolicy:
scheduled: true
referencePolicy:
type: Source
---
apiVersion: v1
kind: ImageStream
metadata:
name: ubi8-go-toolset
spec:
lookupPolicy:
local: false
tags:
- name: latest
from:
kind: DockerImage
name: registry.access.redhat.com/ubi8/go-toolset:latest
importPolicy:
scheduled: true
referencePolicy:
type: Source
---
apiVersion: v1
kind: ImageStream
metadata:
name: ubi8-python-38
spec:
lookupPolicy:
local: false
tags:
- name: latest
from:
kind: DockerImage
name: registry.access.redhat.com/ubi8/python-38:latest
importPolicy:
scheduled: true
referencePolicy:
type: Source

16
roles/openshift-apps/flatpak-indexer/files/service.yml Normal file
View file

@ -0,0 +1,16 @@
---
apiVersion: v1
kind: Service
metadata:
labels:
app: redis
name: redis
spec:
ports:
- name: 6379-tcp
port: 6379
protocol: TCP
targetPort: 6379
selector:
app: redis
deploymentconfig: redis

24
roles/openshift-apps/flatpak-indexer/files/storage.yml Normal file
View file

@ -0,0 +1,24 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: flatpak-indexer-storage
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Gi
storageClassName: ""
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: flatpak-indexer-redis-storage
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
storageClassName: ""

106
roles/openshift-apps/flatpak-indexer/templates/buildconfig.yml Normal file
View file

@ -0,0 +1,106 @@
---
apiVersion: build.openshift.io/v1
kind: BuildConfig
metadata:
labels:
app: flatpak-indexer
name: flatpak-indexer-tardiff-build
spec:
output:
to:
kind: ImageStreamTag
name: flatpak-indexer-tardiff:latest
runPolicy: Serial
source:
contextDir: differ
git:
uri: https://github.com/owtaylor/flatpak-indexer.git
ref: "{{ flatpak_indexer_git_ref }}"
type: Git
strategy:
dockerStrategy:
from:
kind: ImageStreamTag
name: ubi8-go-toolset:latest
noCache: true
type: Docker
triggers:
- type: ConfigChange
- type: ImageChange
imageChange: {}
---
apiVersion: build.openshift.io/v1
kind: BuildConfig
metadata:
labels:
app: flatpak-indexer
name: flatpak-indexer-build
spec:
output:
to:
kind: ImageStream
name: flatpak-indexer-tardiff:latest
postCommit:
command:
- tools/test.sh
runPolicy: Serial
source:
images:
- from:
kind: ImageStreamTag
name: flatpak-indexer-tardiff:latest
paths:
- destinationDir: bin
sourcePath: /opt/app-root/tar-diff
- destinationDir: bin
sourcePath: /usr/bin/time
git:
uri: https://pagure.io/flatpak-indexer.git
ref: "{{ flatpak_indexer_git_ref }}"
type: Git
strategy:
sourceStrategy:
env:
- name: FLATPAK_INDEXER_UPDATE_TEST_DATA
value: "1"
from:
kind: ImageStreamTag
name: ubi8-python-38:latest
type: Source
triggers:
- type: ConfigChange
- type: ImageChange
imageChange:
from:
kind: ImageStreamTag
name: flatpak-indexer-tardiff:latest
---
apiVersion: build.openshift.io/v1
kind: BuildConfig
metadata:
labels:
app: flatpak-indexer
name: redis-build
spec:
output:
to:
kind: ImageStreamTag
name: redis:latest
runPolicy: Serial
source:
contextDir: redis
git:
uri: https://github.com/owtaylor/flatpak-indexer.git
ref: "{{ flatpak_indexer_git_ref }}"
type: Git
strategy:
dockerStrategy:
from:
kind: ImageStreamTag
name: fedora:latest
noCache: true
type: Docker
triggers:
- type: ConfigChange
- type: ImageChange
imageChange: {}

98
roles/openshift-apps/flatpak-indexer/templates/configmap.yml Normal file
View file

@ -0,0 +1,98 @@
---
kind: ConfigMap
apiVersion: v1
metadata:
name: indexer-config
data:
config.yaml: |-
redis_url: ${REDIS_URL}
redis_password: ${REDIS_PASSWORD}
koji_config: fedora
deltas_dir: ${OUTPUT_DIR}/deltas/
icons_dir: ${OUTPUT_DIR}/icons/
{% if env == 'staging' %}
deltas_uri: https://registry{{ env_suffix }}fedoraproject.org/deltas/
icons_uri: https://registry{{ env_suffix }}fedoraproject.org/app-icons/
{% else %}
deltas_uri: https://cdn.registry.fedoraproject.org/deltas/
icons_uri: https://cdn.registry.fedoraproject.org/app-icons/
{% endif %}
daemon:
update_interval: 1800
registries:
fedora:
public_url: https://registry.fedoraproject.org/
datasource: fedora
force_flatpak_token: true
indexes:
latest:
output: ${OUTPUT_DIR}/fedora/flatpak-latest.json
registry: fedora
tag: latest
bodhi_status: stable
extract_icons: True
delta_keep_days: 30
latest-annotations:
output: ${OUTPUT_DIR}/fedora/flatpak-latest-annotations.json
registry: fedora
tag: latest
bodhi_status: stable
extract_icons: True
delta_keep_days: 30
flatpak_annotations: True
latest-amd64:
output: ${OUTPUT_DIR}/fedora/flatpak-latest-amd64.json
registry: fedora
architecture: amd64
tag: latest
bodhi_status: stable
extract_icons: True
delta_keep_days: 30
latest-amd64-annotations:
output: ${OUTPUT_DIR}/fedora/flatpak-latest-amd64-annotations.json
registry: fedora
architecture: amd64
tag: latest
bodhi_status: stable
extract_icons: True
delta_keep_days: 30
flatpak_annotations: True
testing:
output: ${OUTPUT_DIR}/fedora/flatpak-testing.json
registry: fedora
tag: testing
bodhi_status: testing
extract_icons: True
testing-annotations:
output: ${OUTPUT_DIR}/fedora/flatpak-testing-annotations.json
registry: fedora
tag: testing
bodhi_status: testing
extract_icons: True
flatpak_annotations: True
testing-amd64:
output: ${OUTPUT_DIR}/fedora/flatpak-testing-amd64.json
registry: fedora
architecture: amd64
tag: testing
bodhi_status: testing
extract_icons: True
testing-amd64-annotations:
output: ${OUTPUT_DIR}/fedora/flatpak-testing-amd64-annotations.json
registry: fedora
architecture: amd64
tag: testing
bodhi_status: testing
extract_icons: True
flatpak_annotations: True
---
kind: ConfigMap
apiVersion: v1
metadata:
name: koji-conf
data:
koji.conf: |-
[fedora]
server = https://koji.fedoraproject.org/kojihub
weburl = https://koji.fedoraproject.org/koji
topurl = https://kojipkgs.fedoraproject.org/

230
roles/openshift-apps/flatpak-indexer/templates/deploymentconfig.yml Normal file
View file

@ -0,0 +1,230 @@
---
kind: DeploymentConfig
apiVersion: apps.openshift.io/v1
metadata:
labels:
app: flatpak-indexer
name: flatpak-indexer
spec:
replicas: 1
selector:
app: flatpak-indexer
deploymentconfig: flatpak-indexer
strategy:
activeDeadlineSeconds: 21600
resources: {}
rollingParams:
intervalSeconds: 1
maxSurge: 25%
maxUnavailable: 25%
timeoutSeconds: 600
updatePeriodSeconds: 1
type: Rolling
template:
metadata:
labels:
app: flatpak-indexer
deploymentconfig: flatpak-indexer
spec:
containers:
- env:
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: redis-password
key: password
- name: REDIS_URL
value: redis://redis.flatpak.svc.cluster.local:6379
image: flatpak-indexer:latest
name: flatpak-indexer
ports:
- containerPort: 8080
protocol: TCP
resources: {}
volumeMounts:
- mountPath: /etc/flatpak-indexer
name: indexer-config-volume
readOnly: true
- mountPath: /etc/koji.conf
name: koji-conf-volume
readOnly: true
- mountPath: /var/www/flatpaks
name: index-output-volume
- mountPath: /etc/tls/flatpak-indexer
name: flatpak-indexer-crt-volume
dnsPolicy: ClusterFirst
restartPolicy: Always
volumes:
- configMap:
defaultMode: 420
name: indexer-config
name: indexer-config-volume
- configMap:
defaultMode: 420
name: koji-conf
name: koji-conf-volume
- name: index-output-volume
persistentVolumeClaim:
claimName: flatpak-indexer-storage
test: false
triggers:
- type: ConfigChange
- type: ImageChange
imageChangeParams:
automatic: true
containerNames:
- flatpak-indexer-differ
from:
kind: ImageStreamTag
name: flatpak-indexer:latest
---
kind: DeploymentConfig
apiVersion: apps.openshift.io/v1
metadata:
labels:
app: flatpak-indexer-differ
name: flatpak-indexer-differ
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
app: flatpak-indexer-differ
deploymentconfig: flatpak-indexer-differ
strategy:
activeDeadlineSeconds: 21600
resources: {}
rollingParams:
intervalSeconds: 1
maxSurge: 25%
maxUnavailable: 25%
timeoutSeconds: 600
updatePeriodSeconds: 1
type: Rolling
template:
metadata:
labels:
app: flatpak-indexer-differ
deploymentconfig: flatpak-indexer-differ
spec:
containers:
- env:
- name: FLATPAK_INDEXER_COMMAND
value: differ
- name: REDISCLI_AUTH
valueFrom:
secretKeyRef:
name: redis-password
key: password
- name: REDIS_URL
value: redis://redis.flatpak.svc.cluster.local:6379
image: flatpak-indexer:latest
name: flatpak-indexer-differ
ports:
- containerPort: 8080
protocol: TCP
resources:
requests:
cpu: 2
memory: 1Gi
limits:
cpu: 2
memory: 3Gi
volumeMounts:
- mountPath: /etc/flatpak-indexer
name: indexer-config-volume
readOnly: true
- mountPath: /var/www/flatpaks
name: index-output-volume
dnsPolicy: ClusterFirst
restartPolicy: Always
volumes:
- configMap:
defaultMode: 420
name: indexer-config
name: indexer-config-volume
- name: index-output-volume
persistentVolumeClaim:
claimName: flatpak-indexer-storage
test: false
triggers:
- type: ConfigChange
- type: ImageChange
imageChangeParams:
automatic: true
containerNames:
- flatpak-indexer-differ
from:
kind: ImageStreamTag
name: flatpak-indexer:latest
---
apiVersion: apps.openshift.io/v1
kind: DeploymentConfig
metadata:
labels:
app: redis
name: redis
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
app: redis
deploymentconfig: redis
strategy:
activeDeadlineSeconds: 21600
# Recreate so that the old pod is stopped before the new pod is created.
# Could also use a StatefulSet
type: Recreate
template:
metadata:
labels:
app: redis
deploymentconfig: redis
spec:
containers:
- env:
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: redis-password
key: password
image: redis:latest
livenessProbe:
exec:
command: ['redis-cli', 'ping']
failureThreshold: 3
initialDelaySeconds: 15
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
name: redis
ports:
- containerPort: 6379
protocol: TCP
readinessProbe:
exec:
command: ['redis-cli', 'ping']
failureThreshold: 3
periodSeconds: 60
successThreshold: 1
timeoutSeconds: 1
resources: {}
volumeMounts:
- mountPath: /var/lib/redis/data
name: redis-data-volume
dnsPolicy: ClusterFirst
restartPolicy: Always
volumes:
- name: redis-data-volume
persistentVolumeClaim:
claimName: flatpak-indexer-redis-storage
test: false
triggers:
- type: ConfigChange
- type: ImageChange
imageChangeParams:
automatic: true
containerNames:
- redis
from:
kind: ImageStreamTag
name: redis:latest

11
roles/openshift-apps/flatpak-indexer/templates/secret.yml Normal file
View file

@ -0,0 +1,11 @@
---
kind: Secret
apiVersion: v1
metadata:
name: redis-password
data:
{% if env == 'staging' %}
password: {{ stg_flatpak_indexer_redis_password }}
{% else %}
password: {{ prod_flatpak_indexer_redis_password }}
{% endif %}

2
roles/openshift-apps/flatpak-indexer/vars/production.yml Normal file
View file

@ -0,0 +1,2 @@
---
flatpak_indexer_git_ref: a2777e86e225b8ea436efdd33d6927dc1f13f66a

2
roles/openshift-apps/flatpak-indexer/vars/staging.yml Normal file
View file

@ -0,0 +1,2 @@
---
flatpak_indexer_git_ref: a2777e86e225b8ea436efdd33d6927dc1f13f66a

21
roles/regindexer/build/tasks/main.yml
View file

@ -1,21 +0,0 @@
---
- name: install needed packages
package: name=regindexer state=installed
tags:
- packages
- regindexer
- name: make sure regindexer owns it's own files
file: path=/var/lib/regindexer recurse=yes owner=regindexer group=regindexer
tags:
- regindexer
- name: Install config.yaml file
template: src=config.yaml dest=/etc/regindexer/config.yaml
tags:
- regindexer
- name: enable service
service: name=regindexer state=started enabled=true
tags:
- regindexer

74
roles/regindexer/build/templates/config.yaml
View file

@ -1,74 +0,0 @@
icons_dir: /var/lib/regindexer/icons/
icons_uri: /app-icons/
indexes:
flatpak:
output: /var/lib/regindexer/index/flatpak.json
registry: https://registry{{ env_suffix }}.fedoraproject.org
registry_public: /
tags: ['latest']
required_annotations: ['org.flatpak.ref']
skip_flatpak_labels: True
extract_icons: True
flatpak_labels:
output: /var/lib/regindexer/index/flatpak-labels.json
registry: https://registry{{ env_suffix }}.fedoraproject.org
registry_public: /
tags: ['latest']
required_labels: ['org.flatpak.ref']
skip_flatpak_annotations: True
extract_icons: True
flatpak_testing:
output: /var/lib/regindexer/index/flatpak-testing.json
registry: https://registry{{ env_suffix }}.fedoraproject.org
registry_public: /
tags: ['testing']
required_annotations: ['org.flatpak.ref']
skip_flatpak_labels: True
extract_icons: True
flatpak_testing_labels:
output: /var/lib/regindexer/index/flatpak-testing-labels.json
registry: https://registry{{ env_suffix }}.fedoraproject.org
registry_public: /
tags: ['testing']
required_labels: ['org.flatpak.ref']
skip_flatpak_annotations: True
extract_icons: True
flatpak_amd64:
output: /var/lib/regindexer/index/flatpak-amd64.json
registry: https://registry{{ env_suffix }}.fedoraproject.org
registry_public: /
tags: ['latest']
required_annotations: ['org.flatpak.ref']
architectures: ['amd64']
skip_flatpak_labels: True
extract_icons: True
flatpak_amd64_labels:
output: /var/lib/regindexer/index/flatpak-amd64-labels.json
registry: https://registry{{ env_suffix }}.fedoraproject.org
registry_public: /
tags: ['latest']
required_labels: ['org.flatpak.ref']
architectures: ['amd64']
skip_flatpak_annotations: True
extract_icons: True
flatpak_testing_amd64:
output: /var/lib/regindexer/index/flatpak-testing-amd64.json
registry: https://registry{{ env_suffix }}.fedoraproject.org
registry_public: /
tags: ['testing']
required_annotations: ['org.flatpak.ref']
architectures: ['amd64']
skip_flatpak_labels: True
extract_icons: True
flatpak_testing_amd64_labels:
output: /var/lib/regindexer/index/flatpak-testing-amd64-labels.json
registry: https://registry{{ env_suffix }}.fedoraproject.org
registry_public: /
tags: ['testing']
required_labels: ['org.flatpak.ref']
architectures: ['amd64']
skip_flatpak_annotations: True
extract_icons: True
daemon:
topic_prefix: org.fedoraproject
environment: "{{ fedmsg_env }}"