diff --git a/roles/fedora_nightlies/files/fedora_nightlies.conf.httpd b/roles/fedora_nightlies/files/fedora_nightlies.conf.httpd
deleted file mode 100644
index 8bcc8f8b47..0000000000
--- a/roles/fedora_nightlies/files/fedora_nightlies.conf.httpd
+++ /dev/null
@@ -1,3 +0,0 @@
-<VirtualHost *:80>
-    Alias /nightlies.html /var/www/fedora_nightlies/nightlies.html
-</VirtualHost>
diff --git a/roles/fedora_nightlies/files/openqa-extras-fedora_nightlies.inc.httpd b/roles/fedora_nightlies/files/openqa-extras-fedora_nightlies.inc.httpd
new file mode 100644
index 0000000000..35813b4540
--- /dev/null
+++ b/roles/fedora_nightlies/files/openqa-extras-fedora_nightlies.inc.httpd
@@ -0,0 +1 @@
+Alias /nightlies.html "/var/www/fedora_nightlies/nightlies.html"
diff --git a/roles/fedora_nightlies/tasks/main.yml b/roles/fedora_nightlies/tasks/main.yml
index 84f944f5e9..2038e9dc0d 100644
--- a/roles/fedora_nightlies/tasks/main.yml
+++ b/roles/fedora_nightlies/tasks/main.yml
@@ -148,8 +148,10 @@
   service: name=fm-consumer@fedora_nightlies enabled=yes state=started
   when: not fedora_nightlies_disabled|bool
 
+# This is also icky and assumes something is Including this config snippet,
+# i.e. a Fedora infra-y openQA server. But making it better is a bit hard
 - name: Set up Apache config
-  copy: src=fedora_nightlies.conf.httpd dest=/etc/httpd/conf.d/fedora_nightlies.conf owner=root group=root mode=0644
+  copy: src=openqa-extras-fedora_nightlies.inc.httpd dest=/etc/httpd/conf.d/openqa-extras-fedora_nightlies.inc owner=root group=root mode=0644
   notify:
   - reload httpd
   tags:
diff --git a/roles/openqa/server/templates/openqa.conf.httpd.j2 b/roles/openqa/server/templates/openqa.conf.httpd.j2
index 7557a2c446..9923111f79 100644
--- a/roles/openqa/server/templates/openqa.conf.httpd.j2
+++ b/roles/openqa/server/templates/openqa.conf.httpd.j2
@@ -7,4 +7,5 @@
     ProxyPass /apache-status !
 {% endif %}
     Include conf.d/openqa-common.inc
+    Include conf.d/openqa-extras-*.inc
 </VirtualHost>