check daily for left over VM of builders an remove them

files/copr/delete-forgotten-instances.cron

@@ -0,0 +1,5 @@
+#!/usr/bin/bash
+
+source /home/copr/cloud/ec2rc.sh
+/home/copr/delete-forgotten-instances.pl
+

files/copr/delete-forgotten-instances.pl

@@ -0,0 +1,28 @@
+#!/usr/bin/perl
+# this scrip query for all running VM and terminate those
+# which are not currently started by some ansible script
+
+while (chomp($a = qx(ps ax |grep -v 'sh -c ps ax'  |grep /home/copr/provision/builderpb.yml | grep -v grep))) {
+  # we are starting some VM and could not determine correct list of running VMs
+  sleep 5;
+}
+
+#print qx(ps ax |grep ' 172.16.3.' |awk '{ print \$32 }');
+@IPs = split('\s+', qx(ps ax |grep ' 172.16.3.' |awk '{ print \$32 }'));
+
+#print "Running instances\n";
+#print join(", ", @IPs), "\n";
+for my $i (@IPs) {
+  $check{$i} = 1;
+}
+
+@instances = split('\n', qx(/bin/euca-describe-instances));
+@TO_DELETE = ();
+for my $i (@instances) {
+  my @COLUMNS = split('\s+', $i);
+  next if $COLUMNS[0] eq 'RESERVATION';
+  #print $COLUMNS[1], ", ", $COLUMNS[15], "\n";
+  push(@TO_DELETE, $COLUMNS[1]) unless $check{$COLUMNS[15]};
+}
+$id_merged = join(" ", @TO_DELETE);
+qx|euca-terminate-instances $id_merged| if ($id_merged);

playbooks/hosts/copr-be.cloud.fedoraproject.org.yml

@@ -201,6 +201,12 @@
     - --service=https
     - --service=http
 
+  - name: copy delete-forgotten-instances.pl
+    action: copy src="{{ files }}/copr/delete-forgotten-instances.pl" dest=/home/copr/delete-forgotten-instances.pl mode=755
+
+  - name: copy delete-forgotten-instances.cron
+    action: copy src="{{ files }}/copr/delete-forgotten-instances.cron" dest=/etc/cron.daily/delete-forgotten-instances owner=root group=root mode=755
+
   handlers:
   - include: "{{ handlers }}/restart_services.yml"
   - name: concate ssl certs