diff --git a/roles/bodhi2/backend/files/koji-sync-listener.py b/roles/bodhi2/backend/files/koji-sync-listener.py
index 5adb7f6a93..dc4381e7c6 100644
--- a/roles/bodhi2/backend/files/koji-sync-listener.py
+++ b/roles/bodhi2/backend/files/koji-sync-listener.py
@@ -27,7 +27,10 @@ def handle(content):
         '--verbose',
     ] + taglist.split()
     print("Running %r" % cmd)
-    sp.Popen(cmd)
+    proc = sp.Popen(cmd)
+    status = proc.wait()
+    if status:
+        raise RuntimeError("%r gave return code %r" % (cmd, status))
 
 
 def main(fullname, fields, content):