diff --git a/roles/copr/backend/files/provision/builderpb-aws-spot-aarch64.yml b/roles/copr/backend/files/provision/builderpb-aws-spot-aarch64.yml new file mode 100644 index 0000000000..7d948c3e38 --- /dev/null +++ b/roles/copr/backend/files/provision/builderpb-aws-spot-aarch64.yml @@ -0,0 +1,52 @@ +--- +- name: create an aarch64 spot instance in aws + hosts: 127.0.0.1 + gather_facts: false + + roles: + - spawner + + vars_files: + - vars.yml + - aws_cloud_vars.yml + + vars: + keypair: copr-builder + instance_type: a1.xlarge + # We keep this around the on-demand price, as we don't want unnecessary + # interrupts. + spot_price: 0.102 + security_group: ssh-only + max_spawn_time: 1100 + spawning_vm_user: "fedora" + arch: aarch64 + image_name: "{{ builder_images[arch] }}" + instance_volumes: + - device_name: sdb + delete_on_termination: True + volume_type: gp2 + volume_size: 160 + + tasks: + - include: "spinup_aws_spot_task.yml" + + +- name: provision builder + hosts: builder_temp_group + gather_facts: false + become: true + user: fedora + + vars_files: + - vars.yml + - aws_cloud_vars.yml + + vars: + ansible_python_interpreter: /usr/bin/python3 + + tasks: + - include: "create_swap_file.yml" + when: + - prepare_base_image is defined + + - include: "provision_builder_tasks.yml" diff --git a/roles/copr/backend/files/provision/builderpb-aws-spot-x86_64.yml b/roles/copr/backend/files/provision/builderpb-aws-spot-x86_64.yml new file mode 100644 index 0000000000..d08aaa6fcd --- /dev/null +++ b/roles/copr/backend/files/provision/builderpb-aws-spot-x86_64.yml @@ -0,0 +1,47 @@ +--- +- name: create an x86_64 spot instance in aws + hosts: 127.0.0.1 + gather_facts: false + + roles: + - spawner + + vars_files: + - vars.yml + - aws_cloud_vars.yml + + vars: + keypair: copr-builder + instance_type: i3.large + # We keep this around the on-demand price, as we don't want unnecessary + # interrupts. + spot_price: 0.156 + security_group: ssh-only + max_spawn_time: 1100 + spawning_vm_user: "fedora" + arch: x86_64 + image_name: "{{ builder_images[arch] }}" + + tasks: + - include: "spinup_aws_spot_task.yml" + + +- name: provision builder + hosts: builder_temp_group + gather_facts: false + become: true + user: fedora + + vars_files: + - vars.yml + - aws_cloud_vars.yml + + vars: + ansible_python_interpreter: /usr/bin/python3 + + tasks: + - include: "create_swap_file.yml" + when: + - prepare_base_image is defined + + - include: "provision_builder_tasks.yml" diff --git a/roles/copr/backend/files/provision/spinup_aws_spot_task.yml b/roles/copr/backend/files/provision/spinup_aws_spot_task.yml new file mode 100644 index 0000000000..c1b94c63bb --- /dev/null +++ b/roles/copr/backend/files/provision/spinup_aws_spot_task.yml @@ -0,0 +1,47 @@ +--- +- debug: msg="vm_name={{ vm_name }}" + +- name: random subnet to overcome datacenter failures + set_fact: subnet_id={{ item }} + with_random_choice: "{{ aws_arch_subnets[arch] }}" + +- name: Launch instance + ec2: + key_name: "{{ keypair }}" + group: "{{ security_group }}" + instance_type: "{{ instance_type }}" + image: "{{ image_name }}" + wait: true + region: "{{ aws_region }}" + # both x86_64 and aarch64 arches can be allocated in us-east-1c + vpc_subnet_id: "{{ subnet_id }}" + assign_public_ip: yes + instance_tags: + FedoraGroup: copr + CoprPurpose: builder + CoprInstance: "{% if devel %}devel{% else %}production{% endif %}" + Name: "{{ vm_name }}" + arch: "{{ arch }}" + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + volumes: "{% if instance_volumes is defined %}{{ instance_volumes }}{% else %}[]{% endif %}" + spot_price: "{{ spot_price }}" + register: ec2 + +- name: Add new instance to host group + add_host: + hostname: "{{ item.public_ip }}" + groupname: builder_temp_group + loop: "{{ ec2.instances }}" + +- set_fact: builder_ip={{ ec2.instances[0].public_ip }} + +- debug: msg="VM_IP={{ builder_ip }}" + +- name: wait for he host to be hot + local_action: wait_for host={{ builder_ip }} port=22 delay=1 timeout={{ max_spawn_time }} + +- name: wait until ssh is available + local_action: shell false; until [ "$?" -eq "0" ]; do sleep 2; ssh -o PasswordAuthentication=no {{ spawning_vm_user|default('fedora') }}@{{ builder_ip }} 'echo foobar' 2>/dev/null; done + async: 600 + poll: 2 diff --git a/roles/copr/backend/templates/resalloc/vm-aws-new b/roles/copr/backend/templates/resalloc/vm-aws-new index 3777a1a4f0..d8a1e88be5 100755 --- a/roles/copr/backend/templates/resalloc/vm-aws-new +++ b/roles/copr/backend/templates/resalloc/vm-aws-new @@ -15,6 +15,7 @@ Allocate AWS VM instance for Copr build system. Options: --arch native architecture (required) -h, --help show this help + --spot start a spot instance Environment variables: \$RESALLOC_NAME the name given to the allocated instance, will be @@ -31,11 +32,12 @@ test $# -eq 0 && show_help 1 test -n "$RESALLOC_NAME" || show_help 1 -ARGS=$(getopt -o "h" -l "arch:,help" -n "getopt" -- "$@") \ +ARGS=$(getopt -o "h" -l "arch:,spot,help" -n "getopt" -- "$@") \ || show_help 1 eval set -- "$ARGS" opt_arch= +opt_spot=false while true; do # now the name is in $1 and argument in $2 @@ -47,6 +49,13 @@ while true; do eval "opt_$opt=\$2" shift 2 ;; + --spot) + opt=${1##--} + opt=${opt##-} + opt=${opt//-/_} + eval "opt_$opt=:" + shift + ;; -h|--help) show_help 0;; --) # end! shift; break;; @@ -72,10 +81,15 @@ cleanup_actions() } trap cleanup_actions EXIT +spot_pb_part= +if $opt_spot; then + spot_pb_part=-spot +fi + # TODO: We should call aws-cli directly here, instead of parsing output of # ansible playbook. But at the time of writing this script we had the playbook # available, so parsing the output is the easiest way to start. -playbook=/var/lib/resallocserver/provision/builderpb-aws-"$opt_arch".yml +playbook=/var/lib/resallocserver/provision/builderpb-aws"$spot_pb_part"-"$opt_arch".yml { vm_ip=$(ansible-playbook "$playbook" \