From 7e084f5140e02da2e1a724ffa5e4d8f2357fd1f5 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Thu, 9 Jun 2022 19:28:21 +0200 Subject: [PATCH] ocp4: setup VPN on worker nodes Deploy openvpn client on each node with a router. OpenVPN certs needs to be available for each node in /srv/private/ansible/files/vpn/pki/issued/$hostname.{crt,key} --- playbooks/openshift-apps/openvpn.yml | 72 ++++++++++++++ .../openvpn/templates/buildconfig.yml | 27 ++++++ .../openvpn/templates/client.conf | 25 +++++ .../openvpn/templates/configmap.yml | 8 ++ .../openvpn/templates/deployment.yml | 97 +++++++++++++++++++ .../openvpn/templates/imagestream.yml | 7 ++ .../openshift-apps/openvpn/templates/scc.yml | 43 ++++++++ .../openvpn/templates/scc_role.yml | 13 +++ .../openvpn/templates/scc_rolebinding.yml | 12 +++ .../openvpn/templates/secrets.yml | 11 +++ .../openvpn/templates/serviceaccount.yml | 4 + 11 files changed, 319 insertions(+) create mode 100644 playbooks/openshift-apps/openvpn.yml create mode 100644 roles/openshift-apps/openvpn/templates/buildconfig.yml create mode 100644 roles/openshift-apps/openvpn/templates/client.conf create mode 100644 roles/openshift-apps/openvpn/templates/configmap.yml create mode 100644 roles/openshift-apps/openvpn/templates/deployment.yml create mode 100644 roles/openshift-apps/openvpn/templates/imagestream.yml create mode 100644 roles/openshift-apps/openvpn/templates/scc.yml create mode 100644 roles/openshift-apps/openvpn/templates/scc_role.yml create mode 100644 roles/openshift-apps/openvpn/templates/scc_rolebinding.yml create mode 100644 roles/openshift-apps/openvpn/templates/secrets.yml create mode 100644 roles/openshift-apps/openvpn/templates/serviceaccount.yml diff --git a/playbooks/openshift-apps/openvpn.yml b/playbooks/openshift-apps/openvpn.yml new file mode 100644 index 0000000000..a35332cfaa --- /dev/null +++ b/playbooks/openshift-apps/openvpn.yml @@ -0,0 +1,72 @@ +- name: make the app be real + # We don't have any VPN set up on stg + hosts: os_masters[0] + user: root + gather_facts: False + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + vars: + app: openvpn + + roles: + - role: openshift/project + description: Openvpn Client + appowners: [] + + - role: openshift/object + objectname: imagestream.yml + template: imagestream.yml + + - role: openshift/object + objectname: buildconfig.yml + template: buildconfig.yml + + - role: openshift/object + template: serviceaccount.yml + objectname: serviceaccount.yml + + - role: openshift/object + template: scc.yml + objectname: scc.yml + + - role: openshift/object + template: scc_role.yml + objectname: scc_role.yml + + - role: openshift/object + template: scc_rolebinding.yml + objectname: scc_rolebinding.yml + + # load openVPN client certs for each node in ocp_nodes into ocp secret + # expect certs to be generated in {{private}}/files/vpn/pki/issued/ + - role: openshift/object + template: secrets.yml + objectname: secrets.yml + ocp_nodes: + - worker06.ocp.iad2.fedoraproject.org + + - role: openshift/object + template: configmap.yml + objectname: configmap.yml + + - role: openshift/object + template: deployment.yml + objectname: deployment.yml + + - role: openshift/start-build + buildname: openvpn + +############################################### +# actions to delete the project from OpenShift +############################################### +# to run: sudo rbac-playbook -l os_masters_stg[0] -t delete openshift-apps/openvpn.yml + - role: openshift/object-delete + objecttype: project + objectname: openvpn + tags: + - never + - delete + diff --git a/roles/openshift-apps/openvpn/templates/buildconfig.yml b/roles/openshift-apps/openvpn/templates/buildconfig.yml new file mode 100644 index 0000000000..6b4a01ee0c --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/buildconfig.yml @@ -0,0 +1,27 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: openvpn +spec: + failedBuildsHistoryLimit: 5 + nodeSelector: null + output: + to: + kind: ImageStreamTag + name: openvpn:latest + postCommit: {} + resources: {} + runPolicy: Serial + source: + dockerfile: |- + FROM registry.fedoraproject.org/fedora:latest + RUN dnf -y install \ + openvpn && \ + dnf clean all + WORKDIR /config + CMD openvpn --config openvpn.conf + type: Dockerfile + strategy: + dockerStrategy: {} + type: Docker + successfulBuildsHistoryLimit: 5 diff --git a/roles/openshift-apps/openvpn/templates/client.conf b/roles/openshift-apps/openvpn/templates/client.conf new file mode 100644 index 0000000000..e2f2b9f1ad --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/client.conf @@ -0,0 +1,25 @@ +client + +dev tun + +proto udp + +# Specify multiple vpn servers here +remote gateway +remote bastion02 +remote bastion-iad01 + +resolv-retry infinite + +nobind + +persist-key +persist-tun + +ca ssl/ca.crt +cert ssl/client.crt +key ssl/client.key + +comp-lzo + +keepalive 10 60 diff --git a/roles/openshift-apps/openvpn/templates/configmap.yml b/roles/openshift-apps/openvpn/templates/configmap.yml new file mode 100644 index 0000000000..8ea50b70d2 --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/configmap.yml @@ -0,0 +1,8 @@ +{% macro load_file(filename) %}{% include filename %}{%- endmacro -%} +apiVersion: v1 +kind: ConfigMap +metadata: + name: openvpn-config +data: + openvpn.conf: |- + {{ load_file('client.conf') | indent }} diff --git a/roles/openshift-apps/openvpn/templates/deployment.yml b/roles/openshift-apps/openvpn/templates/deployment.yml new file mode 100644 index 0000000000..c98752594d --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/deployment.yml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openvpn-client + annotations: + image.openshift.io/triggers: '[{"from":{"kind":"ImageStreamTag","name":"openvpn:latest"},"fieldPath":"spec.template.spec.containers[?(@.name==\"setup\")].image"},{"from":{"kind":"ImageStreamTag","name":"openvpn:latest"},"fieldPath":"spec.template.spec.containers[?(@.name==\"openvpn\")].image"}]' +spec: + replicas: {{ ocp_nodes | length }} + selector: + matchLabels: + app: openvpn-client + strategy: + type: Recreate + template: + metadata: + labels: + app: openvpn-client + spec: + initContainers: + - name: setup + image: image-registry.openshift-image-registry.svc:5000/openvpn-client/openvpn:latest + command: ["/bin/bash", "-c"] + args: + - > + set -e; + cp -v /ssl/${NODENAME}.crt /config/ssl/client.crt; + cp -v /ssl/${NODENAME}.key /config/ssl/client.key; + cp -v /ssl/ca.crt /config/ssl/; + env: + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /ssl + name: all-certs + - mountPath: /config/ssl + name: openvpn-ssl + containers: + - image: image-registry.openshift-image-registry.svc:5000/openvpn/openvpn:latest + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + add: ["NET_ADMIN"] + name: openvpn + volumeMounts: + - mountPath: /config/ssl + name: openvpn-ssl + - mountPath: /config/openvpn.conf + name: openvpn-config + readOnly: true + subPath: openvpn.conf + - mountPath: /dev/net/tun + readOnly: true + name: tun-device + hostNetwork: true + restartPolicy: Always + serviceAccount: openvpn + serviceAccountName: openvpn + terminationGracePeriodSeconds: 5 + volumes: + - name: all-certs + secret: + secretName: openvpn-certs + defaultMode: 0400 + - configMap: + name: openvpn-config + name: openvpn-config + - name: tun-device + hostPath: + path: /dev/net/tun + - name: openvpn-ssl + emptyDir: {} + nodeSelector: + kubernetes.io/hostname: worker06.ocp.iad2.fedoraproject.org + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - openvpn-client + topologyKey: "kubernetes.io/hostname" + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: ingresscontroller.operator.openshift.io/deployment-ingresscontroller + operator: In + values: + - default + topologyKey: "kubernetes.io/hostname" + namespaces: + - openshift-ingress + diff --git a/roles/openshift-apps/openvpn/templates/imagestream.yml b/roles/openshift-apps/openvpn/templates/imagestream.yml new file mode 100644 index 0000000000..8d62ffa5f4 --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/imagestream.yml @@ -0,0 +1,7 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: openvpn +spec: + lookupPolicy: + local: false diff --git a/roles/openshift-apps/openvpn/templates/scc.yml b/roles/openshift-apps/openvpn/templates/scc.yml new file mode 100644 index 0000000000..87cef8d84b --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/scc.yml @@ -0,0 +1,43 @@ +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: true +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: false +allowPrivilegedContainer: false +allowedCapabilities: +- NET_ADMIN +apiVersion: security.openshift.io/v1 +defaultAddCapabilities: null +fsGroup: + type: MustRunAs +groups: [] +kind: SecurityContextConstraints +metadata: + annotations: + kubernetes.io/description: openvpn specific security context constaints + generation: 1 + name: openvpn +priority: null +readOnlyRootFilesystem: false +requiredDropCapabilities: +- KILL +- MKNOD +- SETUID +- SETGID +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +users: [] +volumes: +- configMap +- downwardAPI +- emptyDir +- persistentVolumeClaim +- projected +- secret +- hostPath + diff --git a/roles/openshift-apps/openvpn/templates/scc_role.yml b/roles/openshift-apps/openvpn/templates/scc_role.yml new file mode 100644 index 0000000000..c73f7506ee --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/scc_role.yml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:openshift:scc:openvpn +rules: +- apiGroups: + - security.openshift.io + resourceNames: + - openvpn + resources: + - securitycontextconstraints + verbs: + - use diff --git a/roles/openshift-apps/openvpn/templates/scc_rolebinding.yml b/roles/openshift-apps/openvpn/templates/scc_rolebinding.yml new file mode 100644 index 0000000000..9554827a50 --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/scc_rolebinding.yml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:openshift:scc:openvpn +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openshift:scc:openvpn +subjects: +- kind: ServiceAccount + name: openvpn + namespace: openvpn diff --git a/roles/openshift-apps/openvpn/templates/secrets.yml b/roles/openshift-apps/openvpn/templates/secrets.yml new file mode 100644 index 0000000000..6084917ddb --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/secrets.yml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: openvpn-certs +data: + ca.crt: {{ lookup('file', private+'/files/vpn/pki/ca.crt') | b64encode }} +{% for node in ocp_nodes %} + {{node}}.crt: {{ lookup('file', private+'/files/vpn/pki/issued/'+node+'.crt') | b64encode }} + {{node}}.key: {{ lookup('file', private+'/files/vpn/pki/issued/'+node+'.key') | b64encode }} +{% endfor %} diff --git a/roles/openshift-apps/openvpn/templates/serviceaccount.yml b/roles/openshift-apps/openvpn/templates/serviceaccount.yml new file mode 100644 index 0000000000..52a016b230 --- /dev/null +++ b/roles/openshift-apps/openvpn/templates/serviceaccount.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openvpn