builders: initial pass at a ARMv7 virt builder

inventory/builders

@@ -48,7 +48,9 @@ buildvm-ppc64le-01.stg.ppc.fedoraproject.org
 
 [buildvm-aarch64-stg]
 buildvm-aarch64-01.stg.arm.fedoraproject.org
-# buildvm-armv7-01.stg.arm.fedoraproject.org
+
+[buildvm-armv7-stg]
+buildvm-armv7-01.stg.arm.fedoraproject.org
 
 [buildvm-aarch64]
 buildvm-aarch64-01.arm.fedoraproject.org
@@ -376,3 +378,4 @@ buildvm-stg
 buildvm-ppc64-stg
 buildvm-ppc64le-stg
 buildvm-aarch64-stg
+buildvm-armv7-stg

inventory/group_vars/all

@@ -109,6 +109,16 @@ virt_install_command_aarch64_two_nic: virt-install -n {{ inventory_hostname }}
                  --network bridge={{ main_bridge }},model=virtio --network=bridge={{ nfs_bridge }},model=virtio
                  --autostart --noautoconsole
 
+virt_install_command_armv7_one_nic: virt-install -n {{ inventory_hostname }} --arch armv7l
+                 --memory={{ mem_size }},maxmemory={{ max_mem_size }} --memballoon virtio
+                 --disk bus=virtio,path={{ volgroup }}/{{ inventory_hostname }}
+                 --vcpus={{ num_cpus }},maxvcpus={{ max_cpu }} -l {{ ks_repo }} -x
+                 'net.ifnames=0 ksdevice=eth0 ks={{ ks_url }} console=tty0 console=ttyAMA0
+                  hostname={{ inventory_hostname }} nameserver={{ dns }}
+                  ip={{ eth0_ip }}::{{ gw }}:{{ nm }}:{{ inventory_hostname }}:eth0:none'
+                 --network bridge={{ main_bridge }},model=virtio
+                 --autostart --noautoconsole
+
 virt_install_command_rhel6: virt-install -n {{ inventory_hostname }}
                  --memory={{ mem_size }},maxmemory={{ max_mem_size }}
                  --disk bus=virtio,path={{ volgroup }}/{{ inventory_hostname }}

inventory/host_vars/buildvm-armv7-01.stg.arm.fedoraproject.org

@@ -0,0 +1,15 @@
+---
+vmhost: aarch64-c21n1.arm.fedoraproject.org
+mem_size: 24576
+max_mem_size: "{{ mem_size }}"
+num_cpus: 4
+max_cpu: "{{ num_cpus }}"
+volgroup: /dev/vg_Server
+
+eth0_ip: 10.5.129.233
+gw: 10.5.129.254
+main_bridge: br0
+virt_install_command: "{{ virt_install_command_armv7_one_nic }}"
+
+ks_url: http://10.5.126.23/repo/rhel/ks/buildvm-fedora-25-armv7
+ks_repo: http://10.5.126.23/pub/fedora/linux/releases/25/Everything/armhfp/os/

library/virt_boot

@@ -0,0 +1,299 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2012, Jeroen Hoekx <jeroen@hoekx.be>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = '''
+---
+author: Jeroen Hoekx
+module: virt_boot
+short_description: Define libvirt boot parameters
+description:
+  - "This module configures the boot order or boot media of a libvirt virtual
+     machine. A guest can be configured to boot from network, hard disk, floppy,
+     cdrom or a direct kernel boot. Specific media can be attached for cdrom, 
+     floppy and direct kernel boot."
+  - This module requires the libvirt module.
+version_added: "0.8"
+options:
+  domain:
+    description:
+    - The name of the libvirt domain.
+    required: true
+  boot:
+    description:
+    - "Specify the boot order of the virtual machine. This is a comma-separated 
+      list of: I(fd), I(hd), I(cdrom) and I(network)."
+    required: false
+  bootmenu:
+    choices: [ "yes", "no" ]
+    description:
+    - Enable or disable the boot menu.
+    required: false
+  kernel:
+    description:
+    - The path of the kernel to boot.
+    required: false
+  initrd:
+    description:
+    - The path of the initrd to boot.
+    required: false
+  cmdline:
+    description:
+    - The command line to boot the kernel with.
+    required: false
+  device:
+    default: hdc
+    description:
+    - The libvirt device name of the cdrom/floppy.
+    required: false
+  image:
+    description:
+    - The image to connect to the cdrom/floppy device.
+    required: false
+  start:
+    choices: [ "yes", "no" ]
+    default: yes
+    description:
+    - Start the guest after configuration.
+    required: false
+examples:
+  - description: Boot from a cdrom image.
+    code: virt_boot domain=archrear image=/srv/rear/archrear/rear-archrear.iso boot=cdrom
+  - description: Boot from the local disk.
+    code: virt_boot domain=archrear boot=hd
+  - description: Boot a specific kernel with a special command line.
+    code: virt_boot domain=archrear kernel={{ storage }}/kernel-archrear initrd={{ storage }}/initramfs-archrear.img cmdline="root=/dev/ram0 vga=normal rw"
+  - description: Boot from the harddisk and if that fails from the network.
+    code: virt_boot domain=archrear boot=hd,network
+  - description: Enable the boot menu.
+    code: virt_boot domain=archrear bootmenu=yes
+requirements: [ "libvirt" ]
+notes:
+  - Run this on the libvirt host.
+  - I(kernel) and I(boot) are mutually exclusive.
+  - This module does not change a running system. A shutdown/restart is required.
+'''
+
+import sys
+
+try:
+    import xml.etree.ElementTree as ET
+    from xml.etree.ElementTree import SubElement
+except ImportError:
+    try:
+        import elementtree.ElementTree as ET
+        from elementtree.ElementTree import SubElement
+    except ImportError:
+        print "failed=True msg='ElementTree python module unavailable'"
+
+try:
+    import libvirt
+except ImportError:
+    print "failed=True msg='libvirt python module unavailable'"
+    sys.exit(1)
+
+from ansible.module_utils.basic import AnsibleModule
+
+def get_disk(doc, device):
+    for disk in doc.findall('.//disk'):
+        target = disk.find('target')
+        if target is not None:
+            if target.get('dev','') == device:
+                return disk
+
+def attach_disk(domain, doc, device, image):
+    disk = get_disk(doc, device)
+    if disk is not None:
+        source = disk.find('source')
+        if source is not None and source.get('file') == image:
+            return False
+
+    xml = '''<disk type="file" device="cdrom">
+                <driver name="qemu" type="raw"/>
+                <source file="{path}"/>
+                <target bus="ide" dev="{dev}"/>
+             </disk>'''.format(path=image, dev=device)
+    domain.updateDeviceFlags(xml, libvirt.VIR_DOMAIN_AFFECT_CONFIG)
+    return True
+
+def detach_disk(domain, doc, device):
+    disk = get_disk(doc, device)
+    if disk is not None:
+        source = disk.find('source')
+        if source is not None and 'file' in source.attrib:
+            del source.attrib['file']
+            domain.updateDeviceFlags(ET.tostring(disk), libvirt.VIR_DOMAIN_AFFECT_CONFIG)
+            return True
+    return False
+
+def main():
+
+    module = AnsibleModule(
+        argument_spec = dict(
+            domain=dict(required=True, aliases=['guest']),
+            boot=dict(),
+            bootmenu=dict(type='bool'),
+            kernel=dict(),
+            initrd=dict(),
+            cmdline=dict(),
+            device=dict(default='hdc'),
+            image=dict(),
+            start=dict(type='bool', default='yes'),
+        ),
+        required_one_of = [['boot','kernel','image','bootmenu']],
+        mutually_exclusive = [['boot','kernel']]
+    )
+
+    params = module.params
+
+    domain_name = params['domain']
+
+    bootmenu = module.boolean(params['bootmenu'])
+
+    boot = params['boot']
+    kernel = params['kernel']
+    initrd = params['initrd']
+    cmdline = params['cmdline']
+
+    device = params['device']
+    image = params['image']
+
+    start = module.boolean(params['start'])
+
+    changed = False
+
+    conn = libvirt.open("qemu:///system")
+    domain = conn.lookupByName(domain_name)
+
+    doc = ET.fromstring( domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE) )
+
+    ### Connect image
+    if image:
+        changed = changed or attach_disk(domain, doc, device, image)
+        if not boot and not kernel:
+            module.exit_json(changed=changed, image=image, device=device)
+    else:
+        changed = changed or detach_disk(domain, doc, device)
+
+    if changed:
+        doc = ET.fromstring( domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE) )
+
+    ### Boot ordering
+    os = doc.find('os')
+    boot_list = os.findall('boot')
+    kernel_el = os.find('kernel')
+    initrd_el = os.find('initrd')
+    cmdline_el = os.find('cmdline')
+
+    ### traditional boot
+    if boot:
+        if kernel_el is not None:
+            changed = True
+            os.remove(kernel_el)
+        if initrd_el is not None:
+            changed = True
+            os.remove(initrd_el)
+        if cmdline_el is not None:
+            changed = True
+            os.remove(cmdline_el)
+
+        items = boot.split(',')
+        if boot_list:
+            needs_change = False
+            if len(items) == len(boot_list):
+                for (boot_el, dev) in zip(boot_list, items):
+                    if boot_el.get('dev') != dev:
+                        needs_change = True
+            else:
+                needs_change = True
+
+            if needs_change:
+                changed = True
+                for boot_el in boot_list:
+                    os.remove(boot_el)
+                for item in items:
+                    boot_el = SubElement(os, 'boot')
+                    boot_el.set('dev', item)
+        else:
+            changed = True
+            for item in items:
+                boot_el = SubElement(os, 'boot')
+                boot_el.set('dev', item)
+    ### direct kernel boot
+    elif kernel:
+        if boot_list:
+            ### libvirt alwas adds boot=hd using direct kernel boot
+            if not (len(boot_list)==1 and boot_list[0].get('dev')=='hd'):
+                changed = True
+                for boot_el in boot_list:
+                    os.remove(boot_el)
+
+        if kernel_el is not None:
+            if kernel_el.text != kernel:
+                changed = True
+                kernel_el.text = kernel
+        else:
+            changed = True
+            kernel_el = SubElement(os, 'kernel')
+            kernel_el.text = kernel
+
+        if initrd_el is not None:
+            if initrd_el.text != initrd:
+                changed = True
+                initrd_el.text = initrd
+        else:
+            changed = True
+            initrd_el = SubElement(os, 'initrd')
+            initrd_el.text = initrd
+
+        if cmdline_el is not None:
+            if cmdline_el.text != cmdline:
+                changed = True
+                cmdline_el.text = cmdline
+        else:
+            changed = True
+            cmdline_el = SubElement(os, 'cmdline')
+            cmdline_el.text = cmdline
+
+    ### Enable/disable bootmenu
+    bootmenu_el = os.find('bootmenu')
+    if bootmenu and bootmenu_el is not None:
+        bootmenu_enabled = bootmenu_el.get('enable')
+        if bootmenu_enabled != 'yes':
+            changed = True
+            bootmenu_el.set('enable', 'yes')
+    elif bootmenu:
+        bootmenu_el = SubElement(os, 'bootmenu')
+        bootmenu_el.set('enable', 'yes')
+        changed = True
+    elif bootmenu_el is not None:
+        os.remove(bootmenu_el)
+        changed = True
+
+    ### save back
+    conn.defineXML( ET.tostring(doc) )
+
+    if start and not domain.isActive():
+        changed = True
+        domain.create()
+
+    module.exit_json(changed=changed)
+
+if __name__ == '__main__':
+    main()

playbooks/groups/buildvm.yml

@@ -3,7 +3,7 @@
 # NOTE: make sure there is room/space for this builder on the buildvmhost
 # NOTE: most of these vars_path come from group_vars/buildvm or from hostvars
 
-- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=buildvm:buildvm-stg:buildvm-aarch64:buildvm-ppc64:buildvm-ppc64le:buildppcle:buildppc:buildvm-s390:buildvm-ppc64-stg:buildvm-ppc64le-stg:buildvm-aarch64-stg"
+- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=buildvm:buildvm-stg:buildvm-aarch64:buildvm-ppc64:buildvm-ppc64le:buildppcle:buildppc:buildvm-s390:buildvm-ppc64-stg:buildvm-ppc64le-stg:buildvm-aarch64-stg:buildvm-armv7-stg"
 
 - name: make koji builder(s)
   hosts: buildvm:buildvm-stg:buildvm-aarch64:buildvm-ppc64:buildvm-ppc64le:buildppcle:buildppc:buildvm-s390:buildvm-ppc64-stg:buildvm-ppc64le-stg:buildvm-aarch64-stg

roles/virthost/tasks/main.yml

@@ -55,6 +55,14 @@
   - packages
   when: ansible_distribution == 'RedHat' and ansible_distribution_major_version|int == 7 and ansible_architecture == 'aarch64'
 
+- name: install libguestfs-tools for ARMv7 VMs on aarch64 rhel7 virthosts
+  yum: pkg={{ item }} state=present
+  with_items:
+  - libguestfs-tools-c
+  tags:
+  - packages
+  when: ansible_distribution == 'RedHat' and ansible_distribution_major_version|int == 7 and ansible_architecture == 'aarch64'
+
 # install libvirtd.conf
 #
 # This provides us with the ability to use virt-manager from non root accounts.

tasks/virt_instance_create.yml

@@ -27,6 +27,19 @@
   delay: 10
   when: inventory_hostname not in result.list_vms
 
+- name: ARMv7 copy the kernel out
+  command: virt-builder --get-kernel {{ volgroup }}/{{ inventory_hostname }} --output /var/lib/libvirt/images/  | awk -F/ '{print $NF}' > /var/lib/libvirt/images/{{ inventory_hostname }}-details.txt
+  when: inventory_hostname.startswith('buildvm-armv7')
+
+- name: ARMv7 copy the cmdline out
+  command: virt-cat -a {{ volgroup }}/{{ inventory_hostname }} /boot/extlinux/extlinux.conf | grep -m1 append | sed -e 's/append //'
+  register: host_cmdline
+  when: inventory_hostname.startswith('buildvm-armv7')
+
+- name: ARMv7 update the virt parameters
+  virt_boot: kernel="/var/lib/libvirt/images/vmlinuz-4.10.8-200.fc25.armv7hl+lpae" initrd="/var/lib/libvirt/images/initramfs-4.10.8-200.fc25.armv7hl+lpae.img" cmdline={{ host_cmdline }}
+  when: inventory_hostname.startswith('buildvm-armv7')
+
 - name: start the vm up
   virt: state=started name={{ inventory_hostname }}
   delegate_to: "{{ vmhost }}"