switch to atlas, new keyboard

playbook.yml

@@ -1,16 +1,34 @@
 - hosts: all, localhost
   become: yes
   vars:
-    pcie_device_ids:
-      - '8086:15b8' # Intel Corporation Ethernet Connection (2) I219-V
+    audio:
+      source: EasyEffects Source
+      sink: EasyEffects Sink
+    # for vfio-pcie
+    vfio_pci_ids:
       - '1002:731f' # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 [Radeon RX 5700 / 5700 XT]
       - '1002:ab38' # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 HDMI Audio
-    usb_device_ids:
-      - '054c:05c4' # Sony Corp. DualShock 4 [CUH-ZCT1x]
-      - '046d:c539' # Logitech, Inc. USB Receiver
-      - '046d:c08d' # Logitech, Inc. G502 LIGHTSPEED Wireless Gaming Mouse
+    # for passthrough
+    pci_device_ids:
+      # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 [Radeon RX 5700 / 5700 XT]
+      - domain: '0000'
+        bus: '03'
+        slot: '00'
+        function: '0'
+        fix_reset_method: yes
+      # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 HDMI Audio
+      - domain: '0000'
+        bus: '03'
+        slot: '00'
+        function: '1'
+      # ASMedia Technology Inc. ASM2142/ASM3142 USB 3.1 Host Controller
+      - domain: '0000'
+        bus: '07'
+        slot: '00'
+        function: '0'
+    usb_device_ids: []
     keyboards:
-      - '/dev/input/by-id/usb-0d3d_USBPS2-event-kbd'
+      - '/dev/input/by-id/usb-Keychron_K4_Keychron_K4-event-kbd'
     mice: []
   roles:
     - win10

roles/win10/files/archlinux_iommu.conf

@@ -1,6 +0,0 @@
-title    Archlinux w/ VFIO
-linux    /vmlinuz-linux
-initrd   /intel-ucode.img
-initrd   /initramfs-linux.img
-options  mitigations=off intel_iommu=on iommu=pt intel_iommu=igfx_off experimental_zcopytx=1 kvm.ignore_msrs=1 acpi_backlight=none root=LABEL=ARCHLINUX rw
-

roles/win10/files/hooks/qemu

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-prepare() {
-    systemctl start win10-usb
-
-    # setup cgroupv2
-    systemctl set-property --runtime -- user.slice AllowedCPUs=0,4
-    systemctl set-property --runtime -- system.slice AllowedCPUs=0,4
-    systemctl set-property --runtime -- init.scope AllowedCPUs=0,4
-
-
-    vfio-isolate -u /tmp/win10.undo \
-        drop-caches \
-        compact-memory \
-        irq-affinity mask C1-3,5-7 \
-        cpu-governor performance C0-7
-        
-    # vfio-isolate -u /tmp/win10.undo \
-    #     drop-caches \
-    #     compact-memory \
-    #     irq-affinity mask C1-3,5-7 \
-    #     cpuset-create --cpus C0,4 /host.slice \
-    #     cpuset-create --cpus C1-3,5-7 -nlb /win10.slice \
-    #     move-tasks / /host.slice \
-    #     cpu-governor performance C0-7
-
-    for _ in $(seq 5); do
-        sleep 3
-        # assign hugepages
-        sysctl -w vm.nr_hugepages=8192 && break
-    done
-    sleep 10
-}
-
-release() {
-    echo "Restore system"
-
-    # restore cgroupv2
-    systemctl set-property --runtime -- user.slice AllowedCPUs=0,1,2,3,4,5,6,7
-    systemctl set-property --runtime -- system.slice AllowedCPUs=0,1,2,3,4,5,6,7
-    systemctl set-property --runtime -- init.scope AllowedCPUs=0,1,2,3,4,5,6,7
-
-    vfio-isolate restore /tmp/win10.undo
-    systemctl stop win10-usb
-    sysctl -w vm.nr_hugepages=0
-}
-
-if [ "$1" == "win10" ]; then
-    if [ "$2" == "prepare" ]; then
-        prepare
-    fi
-    if [ "$2" == "release" ]; then
-        release
-    fi
-fi

roles/win10/files/modprobe/kvm.conf

@@ -0,0 +1 @@
+options kvm_intel nested=1

roles/win10/tasks/main.yml

@@ -2,7 +2,10 @@
   pacman:
     name:
       - qemu
+      # - qemu-audio-jack
       - libvirt
+      - dmidecode
+      - bridge-utils
       - ovmf
       - virt-manager
       - python-evdev
@@ -16,6 +19,7 @@
     name:
       - vfio-isolate
       - persistent-evdev-git
+      - vendor-reset-dkms-git
 
 - name: create hooks directory
   file:
@@ -23,7 +27,7 @@
     state: directory
 
 - name: install qemu hook
-  copy:
+  template:
     src: hooks/qemu
     dest: /etc/libvirt/hooks/qemu
     mode: '755'
@@ -47,14 +51,14 @@
     line: 'MODULES=(vfio_pci vfio vfio_iommu_type1 vfio_virqfd i915)'
   notify: regenerate linux initramfs
 
-- name: configure vfio-pci device ids
-  template:
-    src: vfio.conf.j2
-    dest: /etc/modprobe.d/vfio.conf
+- name: configure kvm kernel module
+  copy:
+    src: modprobe/kvm.conf
+    dest: /etc/modprobe.d/kvm.conf
   notify: regenerate linux initramfs
 
 - name: install boot entry
-  copy:
+  template:
     src: archlinux_iommu.conf
     dest: /boot/loader/entries/archlinux_iommu.conf
 
@@ -95,10 +99,10 @@
     block: |
       cgroup_device_acl = [
           {% for item in keyboards %}
-          "/dev/input/by-id/uinput-persist-keyboard{{ loop.index0 }}",
+          "{{ item }}",
           {% endfor %}
           {% for item in mice %}
-          "/dev/input/by-id/uinput-persist-mouse{{ loop.index0 }}",
+          "{ item }}",
           {% endfor %}
           "/dev/kvm",
           "/dev/null", "/dev/full", "/dev/zero",

roles/win10/templates/archlinux_iommu.conf

@@ -0,0 +1,5 @@
+title    Archlinux w/ VFIO
+linux    /vmlinuz-linux
+initrd   /intel-ucode.img
+initrd   /initramfs-linux.img
+options  mitigations=off vfio-pci.ids={{ ','.join(vfio_pci_ids) }} intel_iommu=on iommu=pt intel_iommu=igfx_off hugepages=8192 rcu_nocbs=0-15 split_lock_detect=off kvm.ignore_msrs=1 kvm.report_ignored_msrs=0 root=LABEL=ARCHLINUX rw

roles/win10/templates/hooks/qemu

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+prepare() {
+    systemctl start win10-usb
+
+    # https://github.com/gnif/vendor-reset/issues/46#issuecomment-992282166
+    {% for pci_device_id in pci_device_ids %}
+    {% if pci_device_id.fix_reset_method|default(False) %}
+    echo 'device_specific' >'/sys/bus/pci/devices/{{ pci_device_id.domain }}:{{ pci_device_id.bus }}:{{ pci_device_id.slot }}.{{ pci_device_id.function }}/reset_method'
+    {% endif %}
+    {% endfor %}
+        
+    vfio-isolate -u /tmp/win10.undo \
+        irq-affinity mask C0-15 \
+        cpuset-modify --cpus C16-19 /system.slice \
+        cpuset-modify --cpus C16-19 /user.slice \
+        cpu-governor performance C0-19
+        # drop-caches \
+        # compact-memory \
+
+    # for _ in $(seq 10); do
+        # assign hugepages
+        # sysctl -w vm.nr_overcommit_hugepages=8192 && break
+        # sleep 1
+    # done
+    # sleep 10
+}
+
+release() {
+    systemctl stop win10-usb
+    vfio-isolate restore /tmp/win10.undo
+    # sysctl -w vm.nr_overcommit_hugepages=0
+}
+
+if [ "$1" == "win10" ]; then
+    if [ "$2" == "prepare" ]; then
+        prepare
+    fi
+    if [ "$2" == "release" ]; then
+        release
+    fi
+fi

roles/win10/templates/persistent-evdev/config.json.j2

@@ -1,12 +1,13 @@
 {
     "cache": "/opt/persistent-evdev/cache",
     "devices": {
+        {% if keyboards or mice %}
         {% for item in keyboards %}
-        "persist-keyboard{{loop.index0}}": "{{ item }}",
+        "persist-keyboard{{loop.index0}}": "{{ item }}"{%if not loop.last or mice %},{% endif %}
         {% endfor %}
         {% for item in mice %}
-        "persist-mouse{{loop.index0}}": "{{ item }}",
+        "persist-mouse{{loop.index0}}": "{{ item }}"{%if not loop.last %},{% endif %}
         {% endfor %}
-        "dummy" : "/dev/null"
+        {% endif %}
     }
 }

roles/win10/templates/vfio.conf.j2

@@ -1 +0,0 @@
-options vfio-pci ids={{ ','.join(pcie_device_ids) }}

roles/win10/templates/win10.xml

@@ -12,69 +12,83 @@
     <hugepages/>
     <locked/>
   </memoryBacking>
-  <vcpu placement='static'>6</vcpu>
+  <vcpu placement='static'>16</vcpu>
   <iothreads>1</iothreads>
   <cputune>
-    <vcpupin vcpu='0' cpuset='1'/>
-    <vcpupin vcpu='1' cpuset='5'/>
+    <vcpupin vcpu='0' cpuset='0'/>
+    <vcpupin vcpu='1' cpuset='1'/>
     <vcpupin vcpu='2' cpuset='2'/>
-    <vcpupin vcpu='3' cpuset='6'/>
-    <vcpupin vcpu='4' cpuset='3'/>
-    <vcpupin vcpu='5' cpuset='7'/>
-    <emulatorpin cpuset='0,4'/>
-    <iothreadpin iothread='1' cpuset='0,4'/>
-    <vcpusched vcpus='0-5' scheduler='fifo' priority='99'/>
+    <vcpupin vcpu='3' cpuset='3'/>
+    <vcpupin vcpu='4' cpuset='4'/>
+    <vcpupin vcpu='5' cpuset='5'/>
+    <vcpupin vcpu='6' cpuset='6'/>
+    <vcpupin vcpu='7' cpuset='7'/>
+    <vcpupin vcpu='8' cpuset='8'/>
+    <vcpupin vcpu='9' cpuset='9'/>
+    <vcpupin vcpu='10' cpuset='10'/>
+    <vcpupin vcpu='11' cpuset='11'/>
+    <vcpupin vcpu='12' cpuset='12'/>
+    <vcpupin vcpu='13' cpuset='13'/>
+    <vcpupin vcpu='14' cpuset='14'/>
+    <vcpupin vcpu='15' cpuset='15'/>
+    <emulatorpin cpuset='16'/>
+    <iothreadpin cpuset='17' iothread='1'/>
+    <vcpusched vcpus='0-15' scheduler='fifo' priority='99'/>
     <emulatorsched scheduler='fifo' priority='99'/>
-    <iothreadsched iothreads='1' scheduler='fifo' priority='99'/>
+    <iothreadsched scheduler='fifo' priority='99' iothreads='1'/>
   </cputune>
   <os>
-    <type arch='x86_64' machine='pc-q35-4.1'>hvm</type>
+    <type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
+    <smbios mode="host"/>
     <loader readonly='yes' type='pflash'>/usr/share/ovmf/x64/OVMF_CODE.fd</loader>
     <nvram>/var/lib/libvirt/qemu/nvram/win10_VARS.fd</nvram>
   </os>
   <features>
     <acpi/>
     <apic/>
-    <hyperv>
+    <hyperv mode='passthrough'>
       <!-- hyper-v enlightments, the more the better -->
       <!-- https://archive.fosdem.org/2019/schedule/event/vai_enlightening_kvm/ -->
-      <relaxed state='on'/>
-      <vapic state='on'/>
-      <spinlocks state='on' retries='8191'/>
-      <vpindex state='on'/>
-      <synic state='on'/>
-      <stimer state='on'/>
-      <tlbflush state='on'/>
-      <ipi state='on'/>
-      <reset state='on'/>
+      <!-- <evmcs state='off'/>
       <frequencies state='on'/>
+      <ipi state='on'/>
       <reenlightenment state='on'/>
-      <!-- nvidia code 43 fix -->
-      <vendor_id state='on' value='fNvidia'/>
+      <relaxed state='on'/>
+      <reset state='on'/>
+      <runtime state='on'/>
+      <spinlocks state='on' retries='8191'/>
+      <stimer state="on">
+        <direct state="on"/>
+      </stimer>
+      <synic state='on'/>
+      <tlbflush state='on'/>
+      <vapic state='on'/>
+      <vendor_id state='on' value='other'/>
+      <vpindex state='on'/> -->
     </hyperv>
     <kvm>
-        <!-- nvidia code 43 fix -->
         <hidden state='on'/>
     </kvm>
     <vmport state='off'/>
     <ioapic driver='kvm'/>
   </features>
-  <cpu mode='host-passthrough' check='none'>
-    <topology sockets='1' cores='3' threads='2'/>
+  <cpu mode='host-passthrough' check='none' migratable='off'>
+    <topology sockets='1' cores='8' threads='2'/>
     <cache mode='passthrough'/>
     <feature policy='require' name='invtsc'/>
+    <!-- required for some games, comes with a performance hit -->
+    <!-- <feature policy="disable" name="hypervisor"/> -->
   </cpu>
   <clock offset='localtime'>
+    <timer name='tsc' present='yes' mode='native'/>
+    <timer name='hypervclock' present='yes'/>
     <timer name='rtc' present='no' tickpolicy='catchup'/>
     <timer name='pit' present='no' tickpolicy='delay'/>
     <timer name='hpet' present='no'/>
     <timer name='kvmclock' present='no'/>
-    <timer name='hypervclock' present='yes'/>
-    <timer name='tsc' present='yes' mode='native'/>
   </clock>
   <on_poweroff>destroy</on_poweroff>
-  <!-- Because of the reset bug, we cannot actually restart without restarting the host -->
-  <on_reboot>destroy</on_reboot>
+  <on_reboot>restart</on_reboot>
   <on_crash>destroy</on_crash>
   <pm>
     <suspend-to-mem enabled='no'/>
@@ -82,28 +96,30 @@
   </pm>
   <devices>
     <emulator>/usr/bin/qemu-system-x86_64</emulator>
+    <memballoon model='none'/>
     <disk type='file' device='disk'>
-      <driver name='qemu' type='raw'/>
-      <source file='/home/marchambault/Documents/virtualmachines/img/win10.img'/>
-      <target dev='sda' bus='scsi'/>
-      <boot order='1'/>
+      <driver name='qemu' type='raw' cache='none' io='native' discard='unmap'/>
+      <source file='/data/marchambault/VirtualMachines/img/atlas.img'/>
+      <target dev='sda' bus='scsi' rotation_rate="1"/>
+      <boot order='2'/>
       <address type='drive' controller='0' bus='0' target='0' unit='0'/>
     </disk>
     <disk type='block' device='disk'>
-      <driver name='qemu' type='raw' cache='none' io='native' discard='unmap' detect_zeroes='unmap'/>
+      <driver name='qemu' type='raw' cache='none' io='native' discard='unmap'/>
       <source dev='/dev/disk/by-id/nvme-CT1000P1SSD8_1910E1F03D8E'/>
-      <target dev='sdc' bus='scsi'/>
+      <target dev='sdc' bus='scsi' rotation_rate="1"/>
       <address type='drive' controller='0' bus='0' target='0' unit='2'/>
     </disk>
     <disk type='file' device='cdrom'>
       <driver name='qemu' type='raw'/>
-      <source file='/home/marchambault/Documents/virtualmachines/iso/virtio-win-0.1.171.iso'/>
+      <!-- <source file=''/> -->
       <target dev='sdb' bus='sata'/>
       <readonly/>
+      <boot order='1'/>
       <address type='drive' controller='0' bus='0' target='0' unit='1'/>
     </disk>
     <controller type='scsi' index='0' model='virtio-scsi'>
-      <driver queues='6' iothread='1'/>
+      <driver queues='8' iothread='1'/>
       <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
     </controller>
     <controller type='sata' index='0'>
@@ -122,7 +138,7 @@
     <controller type='pci' index='3' model='pcie-root-port'>
       <model name='pcie-root-port'/>
       <target chassis='3' port='0x10'/>
-      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
     </controller>
     <controller type='pci' index='4' model='pcie-root-port'>
       <model name='pcie-root-port'/>
@@ -137,7 +153,6 @@
     <controller type='pci' index='6' model='pcie-root-port'>
       <model name='pcie-root-port'/>
       <target chassis='6' port='0x13'/>
-      <alias name='ua-gfx0'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
     </controller>
     <controller type='pci' index='7' model='pcie-root-port'>
@@ -163,6 +178,8 @@
     <controller type='usb' index='0' model='qemu-xhci' ports='15'>
       <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
     </controller>
+
+    <!-- networking -->
     <interface type='bridge'>
       <mac address='52:54:00:f5:2c:df'/>
       <source bridge='virbr0'/>
@@ -170,63 +187,97 @@
       <link state='up'/>
       <address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
     </interface>
-    <input type='mouse' bus='ps2'/>
-    <input type='keyboard' bus='ps2'/>
-    <graphics type='spice'>
-        <listen type='socket' socket='/tmp/win10-spice.sock'/>
+
+    <!-- HID -->
+    <!-- <input type='mouse' bus='ps2'/> -->
+    <input type='mouse' bus='virtio'/>
+    <!-- <input type='keyboard' bus='ps2'/> -->
+    <input type='keyboard' bus='virtio'/>
+    <!-- mouse evdev passthrough -->
+    {% for item in mice %}
+    <input type='evdev'>
+      <source dev='/dev/input/by-id/uinput-persist-mouse{{ loop.index0 }}'/>
+    </input>
+    {% endfor %}
+    <!-- keyboard evdev passthrough -->
+    {% for item in keyboards %}
+    <input type='evdev'>
+      <source dev='/dev/input/by-id/uinput-persist-keyboard{{ loop.index0 }}' grab='all' repeat='on'/>
+    </input>
+    {% endfor %}
+
+    <!-- audio -->
+    <sound model='ich9'>
+      <address type='pci' domain='0x0000' bus='0x02' slot='0x02' function='0x0'/>
+      <codec type='micro'/>
+      <audio id='1'/>
+    </sound>
+    <!-- <audio id='1' type='pulseaudio' serverName='/run/user/1000/pulse/native'>
+      <input mixingEngine='yes' fixedSettings='yes' voices='1' bufferLength='100'/>
+      <output mixingEngine='yes' fixedSettings='yes' voices='1' bufferLength='100'>
+        <settings frequency='48000' channels='2' format='s16'/>
+      </output>
+    </audio> -->
+
+    <audio id='1' type='pulseaudio' serverName='/run/user/1000/pulse/native'>
+      <input mixingEngine='no'/>
+      <output mixingEngine='no'/>
+    </audio>
+    
+    <!-- <audio id='1' type='jack'>
+      <input clientName="win10" connectPorts="{{ audio.source }}"/>
+      <output clientName="win10" connectPorts="{{ audio.sink }}"/>
+    </audio> -->
+  
+    <!-- pci passthrough -->
+    {% for pci_device_id in pci_device_ids %}
+    <hostdev mode='subsystem' type='pci' managed='yes'>
+      <source>
+        <address domain='0x{{ pci_device_id.domain }}' bus='0x{{ pci_device_id.bus }}' slot='0x{{ pci_device_id.slot }}' function='0x{{ pci_device_id.function }}'/>
+      </source>
+      {% if 'alias' in pci_device_id %}
+        <alias name='{{ pci_device_id.alias }}'/>
+      {% endif %}
+      <!-- connect to virtual pcie controller -->
+      <address type='pci' domain='0x{{ pci_device_id.domain }}' bus='0x{{ pci_device_id.bus }}' slot='0x{{ pci_device_id.slot }}' function='0x{{ pci_device_id.function }}' multifunction='on'/>
+    </hostdev>
+    {% endfor %}
+
+    <!-- spice -->
+    <!-- <graphics type='spice' listen='127.0.0.1' port='5900' defaultMode='insecure'>
+        <listen type='address' address='127.0.0.1'/>
         <image compression='off' />
         <mouse mode='server'/>
         <filetransfer enable='no'/>
     </graphics>
-    <sound model='ich9'>
-      <address type='pci' domain='0x0000' bus='0x02' slot='0x02' function='0x0'/>
-      <audio id='1'/>
-    </sound>
-    <audio id='1' type='pulseaudio' serverName='/run/user/1000/pulse/native'/>
-    <hostdev mode='subsystem' type='pci' managed='yes'>
-      <source>
-        <address domain='0x0000' bus='0x00' slot='0x1f' function='0x6'/>
-      </source>
-      <address type='pci' domain='0x0000' bus='0x02' slot='0x03' function='0x0'/>
-    </hostdev>
-    <hostdev mode='subsystem' type='pci' managed='yes'>
-      <source>
-        <address domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
-      </source>
-      <address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
-    </hostdev>
-    <hostdev mode='subsystem' type='pci' managed='yes'>
-      <source>
-        <address domain='0x0000' bus='0x03' slot='0x00' function='0x1'/>
-      </source>
-      <address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
-    </hostdev>
     <redirdev bus='usb' type='spicevmc'>
       <address type='usb' bus='0' port='2'/>
     </redirdev>
     <redirdev bus='usb' type='spicevmc'>
       <address type='usb' bus='0' port='3'/>
-    </redirdev>
-    <memballoon model='none'/>
-    <!-- <memballoon model='virtio'>
-      <address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
-    </memballoon> -->
-    <shmem name='looking-glass'>
+    </redirdev> -->
+
+    <!-- looking-glass -->
+    <!-- <shmem name='looking-glass'>
       <model type='ivshmem-plain'/>
       <size unit='M'>32</size>
       <address type='pci' domain='0x0000' bus='0x02' slot='0x04' function='0x0'/>
-    </shmem>
+    </shmem> -->
   </devices>
+
+  <qemu:override>
+    <!-- workaround for NVMe passthrough with SMI SM2262 controller -->
+    <!-- this breaks with >15 vCPUs! -->
+    <!-- https://bugzilla.kernel.org/show_bug.cgi?id=202055#c42 -->
+    <!-- <qemu:device alias='ua-nvme1'>
+      <qemu:frontend>
+        <qemu:property name='x-msix-relocation' type='string' value='bar2'/>
+      </qemu:frontend>
+    </qemu:device> -->
+  </qemu:override>
+
   <qemu:commandline>
-    <!-- keyboard evdev passthrough -->
-    {% for item in keyboards %}
-    <qemu:arg value='-object'/>
-    <qemu:arg value='input-linux,id=kbd1,evdev=/dev/input/by-id/uinput-persist-keyboard{{ loop.index0 }},grab_all=off,repeat=on'/>
-    {% endfor %}
-    <!-- mouse evdev passthrough -->
-    {% for item in mice %}
-    <qemu:arg value='-object'/>
-    <qemu:arg value='input-linux,id=input5,evdev=/dev/input/by-id/uinput-persist-mouse{{ loop.index0 }}'/>
-    {% endfor %}
+    <!-- <qemu:env name="PIPEWIRE_RUNTIME_DIR" value="/run/user/1000"/>
+    <qemu:env name="PIPEWIRE_LATENCY" value="512/48000"/> -->
   </qemu:commandline>
 </domain>