config for new cpu/setup

playbook.yml

@@ -4,39 +4,30 @@
     audio:
       source: EasyEffects Source
       sink: EasyEffects Sink
+    # for vfio-pcie
     vfio_pci_ids:
-      - '8086:15b8' # Intel Corporation Ethernet Connection (2) I219-V
       - '1002:731f' # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 [Radeon RX 5700 / 5700 XT]
       - '1002:ab38' # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 HDMI Audio
+    # for passthrough
     pci_device_ids:
-      # Intel Corporation Ethernet Connection (2) I219-V
-      - domain: '0000'
-        bus: '00'
-        slot: '1f'
-        function: '6'
       # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 [Radeon RX 5700 / 5700 XT]
       - domain: '0000'
         bus: '03'
         slot: '00'
         function: '0'
+        fix_reset_method: yes
       # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 HDMI Audio
       - domain: '0000'
         bus: '03'
         slot: '00'
         function: '1'
-      # Micron/Crucial Technology P1 NVMe PCIe SSD
+      # ASMedia Technology Inc. ASM2142/ASM3142 USB 3.1 Host Controller
-      # - alias: ua-nvme1
+      - domain: '0000'
-      #   domain: '0000'
+        bus: '07'
-      #   bus: '0b'
+        slot: '00'
-      #   slot: '00'
+        function: '0'
-      #   function: '0'
+    usb_device_ids: []
-    usb_device_ids:
-      - '2dc8:3106' # 8BitDo Ultimate Wireless Controller
-      - '054c:05c4' # Sony Corp. DualShock 4 [CUH-ZCT1x]
-      - '046d:c539' # Logitech, Inc. USB Receiver
-      - '046d:c08d' # Logitech, Inc. G502 LIGHTSPEED Wireless Gaming Mouse
     keyboards:
-      # - '/dev/input/by-id/usb-0d3d_USBPS2-event-kbd'
       - '/dev/input/by-path/platform-i8042-serio-0-event-kbd'
     mice: []
   roles:

roles/win10/templates/archlinux_iommu.conf

@@ -2,4 +2,4 @@ title    Archlinux w/ VFIO
 linux    /vmlinuz-linux
 initrd   /intel-ucode.img
 initrd   /initramfs-linux.img
-options  mitigations=off vfio-pci.ids={{ ','.join(vfio_pci_ids) }} intel_iommu=on iommu=pt intel_iommu=igfx_off experimental_zcopytx=1 kvm.ignore_msrs=1 acpi_backlight=none root=LABEL=ARCHLINUX rw
+options  mitigations=off vfio-pci.ids={{ ','.join(vfio_pci_ids) }} intel_iommu=on iommu=pt intel_iommu=igfx_off hugepages=8192 rcu_nocbs=0-15 split_lock_detect=off kvm.ignore_msrs=1 kvm.report_ignored_msrs=0 root=LABEL=ARCHLINUX rw

roles/win10/templates/hooks/qemu

@@ -5,22 +5,24 @@ prepare() {
 
     # https://github.com/gnif/vendor-reset/issues/46#issuecomment-992282166
     {% for pci_device_id in pci_device_ids %}
+    {% if pci_device_id.fix_reset_method|default(False) %}
     echo 'device_specific' >'/sys/bus/pci/devices/{{ pci_device_id.domain }}:{{ pci_device_id.bus }}:{{ pci_device_id.slot }}.{{ pci_device_id.function }}/reset_method'
+    {% endif %}
     {% endfor %}
         
     vfio-isolate -u /tmp/win10.undo \
-        drop-caches \
+        irq-affinity mask C0-15 \
-        compact-memory \
+        cpuset-modify --cpus C16-19 /system.slice \
-        irq-affinity mask C1-3,5-7 \
+        cpuset-modify --cpus C16-19 /user.slice \
-        cpuset-modify --cpus C0,4 /system.slice \
+        cpu-governor performance C0-19
-        cpuset-modify --cpus C0,4 /user.slice \
+        # drop-caches \
-        cpu-governor performance C0-7
+        # compact-memory \
 
-    for _ in $(seq 10); do
+    # for _ in $(seq 10); do
         # assign hugepages
-        sysctl -w vm.nr_overcommit_hugepages=8192 && break
+        # sysctl -w vm.nr_overcommit_hugepages=8192 && break
-        sleep 1
+        # sleep 1
-    done
+    # done
     # sleep 10
 }
 

roles/win10/templates/win10.xml

@@ -12,23 +12,47 @@
     <hugepages/>
     <locked/>
   </memoryBacking>
-  <vcpu placement='static'>6</vcpu>
+  <vcpu placement='static'>16</vcpu>
   <iothreads>1</iothreads>
   <cputune>
-    <vcpupin vcpu='0' cpuset='1'/>
+    <vcpupin vcpu='0' cpuset='0'/>
-    <vcpupin vcpu='1' cpuset='5'/>
+    <vcpupin vcpu='1' cpuset='1'/>
     <vcpupin vcpu='2' cpuset='2'/>
-    <vcpupin vcpu='3' cpuset='6'/>
+    <vcpupin vcpu='3' cpuset='3'/>
-    <vcpupin vcpu='4' cpuset='3'/>
+    <vcpupin vcpu='4' cpuset='4'/>
+    <vcpupin vcpu='5' cpuset='5'/>
+    <vcpupin vcpu='6' cpuset='6'/>
+    <vcpupin vcpu='7' cpuset='7'/>
+    <vcpupin vcpu='8' cpuset='8'/>
+    <vcpupin vcpu='9' cpuset='9'/>
+    <vcpupin vcpu='10' cpuset='10'/>
+    <vcpupin vcpu='11' cpuset='11'/>
+    <vcpupin vcpu='12' cpuset='12'/>
+    <vcpupin vcpu='13' cpuset='13'/>
+    <vcpupin vcpu='14' cpuset='14'/>
+    <vcpupin vcpu='15' cpuset='15'/>
+    <!-- <vcpupin vcpu='0' cpuset='2'/>
+    <vcpupin vcpu='1' cpuset='3'/>
+    <vcpupin vcpu='2' cpuset='4'/>
+    <vcpupin vcpu='3' cpuset='5'/>
+    <vcpupin vcpu='4' cpuset='6'/>
     <vcpupin vcpu='5' cpuset='7'/>
-    <emulatorpin cpuset='0,4'/>
+    <vcpupin vcpu='6' cpuset='8'/>
-    <iothreadpin iothread='1' cpuset='0,4'/>
+    <vcpupin vcpu='7' cpuset='9'/>
-    <vcpusched vcpus='0-5' scheduler='fifo' priority='99'/>
+    <vcpupin vcpu='8' cpuset='10'/>
+    <vcpupin vcpu='9' cpuset='11'/>
+    <vcpupin vcpu='10' cpuset='12'/>
+    <vcpupin vcpu='11' cpuset='13'/>
+    <vcpupin vcpu='12' cpuset='14'/>
+    <vcpupin vcpu='13' cpuset='15'/> -->
+    <emulatorpin cpuset='16'/>
+    <iothreadpin cpuset='17' iothread='1'/>
+    <vcpusched vcpus='0-15' scheduler='fifo' priority='99'/>
     <emulatorsched scheduler='fifo' priority='99'/>
-    <iothreadsched iothreads='1' scheduler='fifo' priority='99'/>
+    <iothreadsched scheduler='fifo' priority='99' iothreads='1'/>
   </cputune>
   <os>
-    <type arch='x86_64' machine='pc-q35-4.1'>hvm</type>
+    <type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
     <smbios mode="host"/>
     <loader readonly='yes' type='pflash'>/usr/share/ovmf/x64/OVMF_CODE.fd</loader>
     <nvram>/var/lib/libvirt/qemu/nvram/win10_VARS.fd</nvram>
@@ -37,6 +61,24 @@
     <acpi/>
     <apic/>
     <hyperv mode='passthrough'>
+      <!-- hyper-v enlightments, the more the better -->
+      <!-- https://archive.fosdem.org/2019/schedule/event/vai_enlightening_kvm/ -->
+      <!-- <evmcs state='off'/>
+      <frequencies state='on'/>
+      <ipi state='on'/>
+      <reenlightenment state='on'/>
+      <relaxed state='on'/>
+      <reset state='on'/>
+      <runtime state='on'/>
+      <spinlocks state='on' retries='8191'/>
+      <stimer state="on">
+        <direct state="on"/>
+      </stimer>
+      <synic state='on'/>
+      <tlbflush state='on'/>
+      <vapic state='on'/>
+      <vendor_id state='on' value='other'/>
+      <vpindex state='on'/> -->
     </hyperv>
     <kvm>
         <hidden state='on'/>
@@ -44,23 +86,23 @@
     <vmport state='off'/>
     <ioapic driver='kvm'/>
   </features>
-  <cpu mode='host-passthrough' check='partial'>
+  <cpu mode='host-passthrough' check='none' migratable='off'>
-    <topology sockets='1' cores='3' threads='2'/>
+    <topology sockets='1' cores='8' threads='2'/>
     <cache mode='passthrough'/>
     <feature policy='require' name='invtsc'/>
-    <!-- required for Genshin Impact, come with a performance hit -->
+    <!-- required for some games, comes with a performance hit -->
     <!-- <feature policy="disable" name="hypervisor"/> -->
   </cpu>
   <clock offset='localtime'>
     <timer name='tsc' present='yes' mode='native'/>
     <timer name='hypervclock' present='yes'/>
-    <timer name='rtc' tickpolicy='catchup'/>
+    <timer name='rtc' present='no' tickpolicy='catchup'/>
-    <timer name='pit' tickpolicy='delay'/>
+    <timer name='pit' present='no' tickpolicy='delay'/>
     <timer name='hpet' present='no'/>
+    <timer name='kvmclock' present='no'/>
   </clock>
   <on_poweroff>destroy</on_poweroff>
-  <!-- Because of the AMD reset bug, we cannot actually reboot normally -->
+  <on_reboot>restart</on_reboot>
-  <on_reboot>destroy</on_reboot>
   <on_crash>destroy</on_crash>
   <pm>
     <suspend-to-mem enabled='no'/>
@@ -70,7 +112,7 @@
     <emulator>/usr/bin/qemu-system-x86_64</emulator>
     <memballoon model='none'/>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw'/>
+      <driver name='qemu' type='raw' cache='none' io='native' discard='unmap'/>
       <source file='/data/marchambault/VirtualMachines/img/win10.img'/>
       <target dev='sda' bus='scsi' rotation_rate="1"/>
       <boot order='2'/>
@@ -91,7 +133,7 @@
       <address type='drive' controller='0' bus='0' target='0' unit='1'/>
     </disk>
     <controller type='scsi' index='0' model='virtio-scsi'>
-      <driver queues='6' iothread='1'/>
+      <driver queues='8' iothread='1'/>
       <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
     </controller>
     <controller type='sata' index='0'>
@@ -110,7 +152,7 @@
     <controller type='pci' index='3' model='pcie-root-port'>
       <model name='pcie-root-port'/>
       <target chassis='3' port='0x10'/>
-      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
     </controller>
     <controller type='pci' index='4' model='pcie-root-port'>
       <model name='pcie-root-port'/>
@@ -211,6 +253,8 @@
       {% if 'alias' in pci_device_id %}
         <alias name='{{ pci_device_id.alias }}'/>
       {% endif %}
+      <!-- connect to virtual pcie controller -->
+      <address type='pci' domain='0x{{ pci_device_id.domain }}' bus='0x{{ pci_device_id.bus }}' slot='0x{{ pci_device_id.slot }}' function='0x{{ pci_device_id.function }}' multifunction='on'/>
     </hostdev>
     {% endfor %}