gpu: GPU passthrough test suite

libvirt/tests/cfg/gpu/default_smmuv3_attrs.cfg

@@ -0,0 +1,20 @@
+- gpu.default_smmuv3_attrs:
+    only aarch64
+    type = single_passthrough_numa_pci_topo
+    start_vm = "no"
+    mem_value = 32
+    vcpu = 32
+    test_devices = '["3D"]'
+    cmdqv_on_num = 1
+
+    numa_attrs = "'cpu':{'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+    acpi_nodeset = {'nodeset':'1-8'}
+    pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+    pxb_ctrl_attrs = {'type':'pci','model':'pcie-expander-bus','index': '%s',${pxb_addr}}
+    pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+    gpu_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    gpu_hostdev_dict = {${gpu_driver},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}
+    iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on'}}

libvirt/tests/cfg/gpu/emulated_nic_with_gpu.cfg

@@ -0,0 +1,35 @@
+- gpu.emulated_nic_with_gpu:
+    only aarch64
+    type = emulated_nic_with_gpu
+    start_vm = "no"
+    mem_value = 24
+    vcpu = 32
+    test_devices = '["3D"]'
+
+    variants nic_topology:
+        - pcie_root:
+            nic_bus_type = "pcie_root"
+        - pxb_pcie:
+            nic_bus_type = "pxb_pcie"
+            pxb_ctrl_attrs_2 = {'type':'pci','model':'pcie-expander-bus','index':'%s',${pxb_addr}}
+            pcie_root_port_attrs_2 = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+            iommu_dict_2 = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'off'}}
+
+    variants attach_method:
+        - coldplug:
+            attach_method = "coldplug"
+        - hotplug:
+            attach_method = "hotplug"
+
+    ping_target = "8.8.8.8"
+    numa_attrs = "'cpu':{'numa_cell':[{'id':'0','cpus':'0-31','memory':'24','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+    acpi_nodeset = {'nodeset':'1-8'}
+    pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+    pxb_ctrl_attrs = {'type':'pci','model':'pcie-expander-bus','index':'%s',${pxb_addr}}
+    pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+    gpu_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    gpu_hostdev_dict = {${gpu_driver},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}
+    iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidsize':'20','oas':'48'}}

libvirt/tests/cfg/gpu/interconnect_stress.cfg

@@ -0,0 +1,35 @@
+- gpu.interconnect_stress:
+    only aarch64
+    type = interconnect_stress
+    start_vm = "no"
+    mem_value = 32
+    vcpu = 32
+    test_devices = '["3D"]'
+    cmdqv_on_num = 0
+
+    nccl_test_name = "all_gather_perf"
+    nccl_min_bytes = 8
+    nccl_max_bytes = "4G"
+    nccl_step_factor = 2
+    nccl_num_gpus = 1
+    nccl_test_timeout = 600
+    min_nccl_bandwidth = 10
+    nvbandwidth_tests = ["host_to_device_memcpy_ce","device_to_host_memcpy_ce","device_to_device_memcpy_read_ce","device_to_device_memcpy_write_ce"]
+    nvbandwidth_test_timeout = 300
+    min_host_to_device_bandwidth = 100
+    min_device_to_host_bandwidth = 100
+    min_device_to_device_bandwidth = 700
+    git_clone_timeout = 600
+    build_timeout = 600
+
+    numa_attrs = "'cpu':{'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+    acpi_nodeset = {'nodeset':'1-8'}
+    pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+    pxb_ctrl_attrs = {'type':'pci','model':'pcie-expander-bus','index':'%s',${pxb_addr}}
+    pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+    gpu_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    gpu_hostdev_dict = {${gpu_driver},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}
+    iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','cmdqv':'off','ats':'on','ril':'off','ssidsize':'20','oas':'48'}}

libvirt/tests/cfg/gpu/invalid_numa_config.cfg

@@ -0,0 +1,22 @@
+- gpu.invalid_numa_config:
+    only aarch64
+    type = invalid_xml_config
+    start_vm = "no"
+    mem_value = 32
+    vcpu = 32
+
+    variants:
+        - numa_nonexistent_node:
+            numa_attrs = "'cpu':{'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+            acpi_nodeset = {'nodeset':'1-9'}
+            expected_error = "is invalid"
+
+    pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+    pxb_ctrl_attrs = {'type':'pci','model':'pcie-expander-bus','index':'%s',${pxb_addr}}
+    pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+    gpu_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    gpu_hostdev_dict = {${gpu_driver},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}
+    iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','ssidsize':'20','oas':'48'}}

libvirt/tests/cfg/gpu/invalid_smmuv3_config.cfg

@@ -0,0 +1,46 @@
+- gpu.invalid_smmuv3_config:
+    only aarch64
+    type = invalid_xml_config
+    start_vm = "no"
+    mem_value = 32
+    vcpu = 32
+
+    variants:
+        - invalid_ssidsize_accel_on:
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','pasid':'on','ssidsize':'22','oas':'48'}}
+            expected_error = "'ssidsize' does not accept value"
+        - invalid_ril_on_accel_on:
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'on','pasid':'on','ssidsize':'20','oas':'48'}}
+            expected_error = "Range Invalidation"
+        - invalid_oas_accel_on:
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','pasid':'on','ssidsize':'20','oas':'52'}}
+            expected_error = "QEMU SMMUv3 model only implements auto, 44 bit, or 48 bit OAS"
+        - invalid_ril_off_accel_off:
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'off','ats':'on','ril':'off','pasid':'on','ssidsize':'20','oas':'48'}}
+            expected_error = "accel must be enabled for iommu model"
+        - invalid_ats_on_accel_off:
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'off','ats':'on','ril':'on','pasid':'on','ssidsize':'20','oas':'48'}}
+            expected_error = "accel must be enabled for iommu model"
+        - invalid_oas_accel_off:
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'off','ats':'off','ril':'on','pasid':'on','ssidsize':'20','oas':'48'}}
+            expected_error = "accel must be enabled for iommu model"
+        - invalid_ssidsize_accel_off:
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'off','ats':'off','ril':'on','pasid':'on','ssidsize':'20','oas':'44'}}
+            expected_error = "accel must be enabled for iommu model"
+        - vfio_non_accel_unsupported:
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'off','ats':'off','ril':'on','oas':'44'}}
+            expected_error = "accel must be enabled for iommu model"
+        - invalid_vfio_no_iommufd_accel_on:
+            gpu_hostdev_dict = {'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}
+            iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','ats':'on','ril':'off','oas':'48'}}
+            expected_error = "vfio-pci endpoint devices without an iommufd backend"
+
+    numa_attrs = "'cpu':{'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+    acpi_nodeset = {'nodeset':'1-8'}
+    pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+    pxb_ctrl_attrs = {'type':'pci','model':'pcie-expander-bus','index':'%s',${pxb_addr}}
+    pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    gpu_hostdev_dict = {'driver':{'driver_attr':{'iommufd':'yes'}},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}

libvirt/tests/cfg/gpu/lifecycle_vm_with_gpu.cfg

@@ -1,9 +1,21 @@
 - gpu.lifecycle_vm_with_gpu:
+    only aarch64
     type = lifecycle_vm_with_gpu
     start_vm = "no"
     unsupported_error = "VFIO migration is not supported in kernel"
+    mem_value = 32
+    vcpu = 32
+    test_devices = '["3D"]'
+    cmdqv_on_num = 0
 
-    only aarch64
-    variants:
-        - gpu_address:
-            hostdev_dict = {'mode': 'subsystem', 'type': 'pci', 'source': {'untyped_address': gpu_pci_addr}, 'managed': 'yes'}
+    numa_attrs = "'cpu': {'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+    acpi_nodeset = {'nodeset':'1-8'}
+    pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+    pxb_ctrl_attrs = {'type':'pci','model': 'pcie-expander-bus','index': '%s',${pxb_addr}}
+    pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+    gpu_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    gpu_hostdev_dict = {${gpu_driver},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}
+    iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','cmdqv':'off','ats':'on','ril':'off','ssidsize':'20','oas':'48'}}

libvirt/tests/cfg/gpu/llm_workload.cfg

@@ -0,0 +1,27 @@
+- gpu.llm_workload:
+    only aarch64
+    type = llm_workload
+    start_vm = "no"
+    mem_value = 32
+    vcpu = 32
+    test_devices = '["3D"]'
+    cmdqv_on_num = 0
+
+    modelfile_content = "FROM llama3.2\nPARAMETER temperature 1\nPARAMETER num_ctx 4096\nSYSTEM You are a helpful assistant."
+    ollama_test_prompt = "What is a GPU?"
+    max_ollama_create_retries = 3
+    min_gpu_utilization_percent = 50
+    model_create_timeout = 600
+    inference_timeout = 300
+
+    numa_attrs = "'cpu': {'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+    acpi_nodeset = {'nodeset':'1-8'}
+    pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+    pxb_ctrl_attrs = {'type':'pci','model':'pcie-expander-bus','index':'%s',${pxb_addr}}
+    pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${mem_value},'vcpu':${vcpu},${numa_attrs}}
+    gpu_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+    gpu_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    gpu_hostdev_dict = {${gpu_driver},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}
+    iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','cmdqv':'off','ats':'on','ril':'off','ssidsize':'20','oas':'48'}}

libvirt/tests/cfg/gpu/memory_hotplug_with_gpu_passthrough.cfg

@@ -0,0 +1,31 @@
+- gpu.memory_hotplug_with_gpu_passthrough:
+    only aarch64
+    type = memory_hotplug_with_gpu_passthrough
+    start_vm = "no"
+    mem_value = 36
+    guest_mem = 32
+    vcpu = 32
+    test_devices = '["3D"]'
+    cmdqv_on_num = 1
+    kernel_hp_file = '/sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages'
+
+    variants:
+        - hugepage_512M:
+            hugepage_size = 512
+            memory_backing = {'source_type':'memfd','hugepages':{'pages': [{'size': '512', 'unit':'MiB'}]},'access_mode':'shared','allocation':{'mode':'immediate'}}
+        - hugepage_2M:
+            hugepage_size = 2
+            memory_backing = {'source_type':'memfd','hugepages':{'pages': [{'size': '2', 'unit':'MiB'}]},'access_mode':'shared','allocation':{'mode':'immediate'}}
+
+    numa_attrs = "'cpu':{'numa_cell':[{'id':'0','cpus':'0-31','memory':'32','unit':'GiB'},{'id':'1','memory':'0'},{'id':'2','memory':'0'},{'id':'3','memory':'0'},{'id':'4','memory':'0'},{'id':'5','memory':'0'},{'id':'6','memory':'0'},{'id':'7','memory':'0'},{'id':'8','memory':'0'}]}"
+    acpi_nodeset = {'nodeset':'1-8'}
+    pxb_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':0x00,'slot':'%s','function':'0x0'}}"
+    pxb_ctrl_attrs = {'type':'pci','model':'pcie-expander-bus','index':'%s',${pxb_addr}}
+    pcie_root_port_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    pcie_root_port_attrs = {'type':'pci','model':'pcie-root-port','index':'%s',${pcie_root_port_addr}}
+    vm_attrs = {'memory_unit':'GiB','memory':${guest_mem},'vcpu':${vcpu},${numa_attrs}, 'max_mem_rt':37748736,'max_mem_rt_slots':16,'max_mem_rt_unit':'KiB'}
+    gpu_driver = "'driver':{'driver_attr':{'iommufd':'yes'}}"
+    gpu_addr = "'address':{'attrs':{'type':'pci','domain':'0x0000','bus':'%s','slot':'0x00','function':'0x0'}}"
+    gpu_hostdev_dict = {${gpu_driver},'acpi':${acpi_nodeset},'mode':'subsystem','type':'pci','source':{'untyped_address':gpu_pci_addr},'managed':'no',${gpu_addr}}
+    iommu_dict = {'model':'smmuv3','driver':{'pciBus':'%s','accel':'on','cmdqv':'on','ats':'on','ril':'off','ssidsize':'20','oas':'48'}}
+    mem_dict = {'mem_model':'dimm','target':{'size':2,'size_unit':'GiB','node':0}}