copr-be: spot instance spawning, #1

roles/copr/backend/files/provision/builderpb-aws-spot-aarch64.yml

@@ -0,0 +1,52 @@
+---
+- name: create an aarch64 spot instance in aws
+  hosts: 127.0.0.1
+  gather_facts: false
+
+  roles:
+    - spawner
+
+  vars_files:
+    - vars.yml
+    - aws_cloud_vars.yml
+
+  vars:
+    keypair: copr-builder
+    instance_type: a1.xlarge
+    # We keep this around the on-demand price, as we don't want unnecessary
+    # interrupts.
+    spot_price: 0.102
+    security_group: ssh-only
+    max_spawn_time: 1100
+    spawning_vm_user: "fedora"
+    arch: aarch64
+    image_name: "{{ builder_images[arch] }}"
+    instance_volumes:
+      - device_name: sdb
+        delete_on_termination: True
+        volume_type: gp2
+        volume_size: 160
+
+  tasks:
+    - include: "spinup_aws_spot_task.yml"
+
+
+- name: provision builder
+  hosts: builder_temp_group
+  gather_facts: false
+  become: true
+  user: fedora
+
+  vars_files:
+    - vars.yml
+    - aws_cloud_vars.yml
+
+  vars:
+    ansible_python_interpreter: /usr/bin/python3
+
+  tasks:
+    - include: "create_swap_file.yml"
+      when:
+        - prepare_base_image is defined
+
+    - include: "provision_builder_tasks.yml"

roles/copr/backend/files/provision/builderpb-aws-spot-x86_64.yml

@@ -0,0 +1,47 @@
+---
+- name: create an x86_64 spot instance in aws
+  hosts: 127.0.0.1
+  gather_facts: false
+
+  roles:
+    - spawner
+
+  vars_files:
+    - vars.yml
+    - aws_cloud_vars.yml
+
+  vars:
+    keypair: copr-builder
+    instance_type: i3.large
+    # We keep this around the on-demand price, as we don't want unnecessary
+    # interrupts.
+    spot_price: 0.156
+    security_group: ssh-only
+    max_spawn_time: 1100
+    spawning_vm_user: "fedora"
+    arch: x86_64
+    image_name: "{{ builder_images[arch] }}"
+
+  tasks:
+    - include: "spinup_aws_spot_task.yml"
+
+
+- name: provision builder
+  hosts: builder_temp_group
+  gather_facts: false
+  become: true
+  user: fedora
+
+  vars_files:
+    - vars.yml
+    - aws_cloud_vars.yml
+
+  vars:
+    ansible_python_interpreter: /usr/bin/python3
+
+  tasks:
+    - include: "create_swap_file.yml"
+      when:
+        - prepare_base_image is defined
+
+    - include: "provision_builder_tasks.yml"

roles/copr/backend/files/provision/spinup_aws_spot_task.yml

@@ -0,0 +1,47 @@
+---
+- debug: msg="vm_name={{ vm_name }}"
+
+- name: random subnet to overcome datacenter failures
+  set_fact: subnet_id={{ item }}
+  with_random_choice: "{{ aws_arch_subnets[arch] }}"
+
+- name: Launch instance
+  ec2:
+    key_name: "{{ keypair }}"
+    group: "{{ security_group }}"
+    instance_type: "{{ instance_type }}"
+    image: "{{ image_name }}"
+    wait: true
+    region: "{{ aws_region }}"
+    # both x86_64 and aarch64 arches can be allocated in us-east-1c
+    vpc_subnet_id: "{{ subnet_id }}"
+    assign_public_ip: yes
+    instance_tags:
+      FedoraGroup: copr
+      CoprPurpose: builder
+      CoprInstance: "{% if devel %}devel{% else %}production{% endif %}"
+      Name: "{{ vm_name }}"
+      arch: "{{ arch }}"
+    aws_access_key: "{{ aws_access_key }}"
+    aws_secret_key: "{{ aws_secret_key }}"
+    volumes: "{% if instance_volumes is defined %}{{ instance_volumes }}{% else %}[]{% endif %}"
+    spot_price: "{{ spot_price }}"
+  register: ec2
+
+- name: Add new instance to host group
+  add_host:
+    hostname: "{{ item.public_ip }}"
+    groupname: builder_temp_group
+  loop: "{{ ec2.instances }}"
+
+- set_fact: builder_ip={{ ec2.instances[0].public_ip }}
+
+- debug: msg="VM_IP={{ builder_ip }}"
+
+- name: wait for he host to be hot
+  local_action: wait_for host={{ builder_ip }} port=22 delay=1 timeout={{ max_spawn_time }}
+
+- name: wait until ssh is available
+  local_action:  shell false; until [ "$?" -eq "0" ]; do sleep 2; ssh -o PasswordAuthentication=no {{ spawning_vm_user|default('fedora') }}@{{ builder_ip }} 'echo foobar' 2>/dev/null; done
+  async: 600
+  poll: 2

roles/copr/backend/templates/resalloc/vm-aws-new

@@ -15,6 +15,7 @@ Allocate AWS VM instance for Copr build system.
 Options:
   --arch                native architecture (required)
   -h, --help            show this help
+  --spot                start a spot instance
 
 Environment variables:
   \$RESALLOC_NAME       the name given to the allocated instance, will be
@@ -31,11 +32,12 @@ test $# -eq 0 && show_help 1
 
 test -n "$RESALLOC_NAME" || show_help 1
 
-ARGS=$(getopt -o "h" -l "arch:,help" -n "getopt" -- "$@") \
+ARGS=$(getopt -o "h" -l "arch:,spot,help" -n "getopt" -- "$@") \
     || show_help 1
 eval set -- "$ARGS"
 
 opt_arch=
+opt_spot=false
 
 while true; do
     # now the name is in $1 and argument in $2
@@ -47,6 +49,13 @@ while true; do
         eval "opt_$opt=\$2"
         shift 2
         ;;
+    --spot)
+        opt=${1##--}
+        opt=${opt##-}
+        opt=${opt//-/_}
+        eval "opt_$opt=:"
+        shift
+        ;;
     -h|--help) show_help 0;;
     --) # end!
         shift; break;;
@@ -72,10 +81,15 @@ cleanup_actions()
 }
 trap cleanup_actions EXIT
 
+spot_pb_part=
+if $opt_spot; then
+    spot_pb_part=-spot
+fi
+
 # TODO: We should call aws-cli directly here, instead of parsing output of
 # ansible playbook.  But at the time of writing this script we had the playbook
 # available, so parsing the output is the easiest way to start.
-playbook=/var/lib/resallocserver/provision/builderpb-aws-"$opt_arch".yml
+playbook=/var/lib/resallocserver/provision/builderpb-aws"$spot_pb_part"-"$opt_arch".yml
 
 {
     vm_ip=$(ansible-playbook "$playbook" \