diff --git a/roles/koji_builder/templates/osbuildapi-update.sh b/roles/koji_builder/templates/osbuildapi-update.sh
index e15cec371c..e3e665b60b 100644
--- a/roles/koji_builder/templates/osbuildapi-update.sh
+++ b/roles/koji_builder/templates/osbuildapi-update.sh
@@ -3,6 +3,11 @@
 # Make sure the ipset is created.
 /usr/sbin/ipset create osbuildapi hash:ip >& /dev/null
 
+# Prepare a temporary set to store the new IPs, so we can atomically swap them
+/usr/sbin/ipset create osbuildapi_tmp hash:ip >& /dev/null
+# Make sure the temporary set is empty
+/usr/sbin/ipset flush osbuildapi_tmp
+
 # in staging we need to allow api.stage and in prod api.
 {% if env == 'staging' %}
 RESOLVEQUERY=`resolvectl -4 --cache=no --legend=no query api.stage.openshift.com 2> /dev/null`
@@ -13,10 +18,9 @@ test $? -eq 0 || exit $?
 
 NEWIPS=`echo "$RESOLVEQUERY" | grep link | sed -E 's/.* ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+).*/\1/g' | sort -n`
 
-/usr/sbin/ipset flush osbuildapi
 for j in $NEWIPS
 do
-     /usr/sbin/ipset add osbuildapi $j
+     /usr/sbin/ipset add osbuildapi_tmp $j
 done
 
 # both stage and prod authenticate using sso.redhat.com
@@ -27,5 +31,9 @@ NEWIDENTITYIPS=`echo "$RESOLVEQUERY" | grep link | sed -E 's/.* ([0-9]+\.[0-9]+\
 
 for j in $NEWIDENTITYIPS
 do
-     /usr/sbin/ipset add osbuildapi $j
+     /usr/sbin/ipset add osbuildapi_tmp $j
 done
+
+# Swap the sets atomically
+/usr/sbin/ipset swap osbuildapi osbuildapi_tmp
+/usr/sbin/ipset destroy osbuildapi_tmp