badjware
/
vfio-win10
1
0
Fork 0
Code Releases Activity

Compare commits

base: badjware:master
Branches Tags
badjware:master
..
compare: badjware:513ab94563ac8bdf6d00707c2e1b1cc7f4f470e0
Branches Tags
badjware:master

No commits in common. "master" and "513ab94563ac8bdf6d00707c2e1b1cc7f4f470e0" have entirely different histories.
master ... 513ab94563

10 changed files with 164 additions and 224 deletions
Show Stats Expand all files Collapse all files

32
playbook.yml
View File

@ -1,34 +1,16 @@
- hosts: all, localhost
become: yes
vars:
audio:
source: EasyEffects Source
sink: EasyEffects Sink
# for vfio-pcie
vfio_pci_ids:
pcie_device_ids:
- '8086:15b8' # Intel Corporation Ethernet Connection (2) I219-V
- '1002:731f' # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 [Radeon RX 5700 / 5700 XT]
- '1002:ab38' # Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 HDMI Audio
# for passthrough
pci_device_ids:
# Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 [Radeon RX 5700 / 5700 XT]
- domain: '0000'
bus: '03'
slot: '00'
function: '0'
fix_reset_method: yes
# Advanced Micro Devices, Inc. [AMD/ATI] Navi 10 HDMI Audio
- domain: '0000'
bus: '03'
slot: '00'
function: '1'
# ASMedia Technology Inc. ASM2142/ASM3142 USB 3.1 Host Controller
- domain: '0000'
bus: '07'
slot: '00'
function: '0'
usb_device_ids: []
usb_device_ids:
- '054c:05c4' # Sony Corp. DualShock 4 [CUH-ZCT1x]
- '046d:c539' # Logitech, Inc. USB Receiver
- '046d:c08d' # Logitech, Inc. G502 LIGHTSPEED Wireless Gaming Mouse
keyboards:
- '/dev/input/by-id/usb-Keychron_K4_Keychron_K4-event-kbd'
- '/dev/input/by-id/usb-0d3d_USBPS2-event-kbd'
mice: []
roles:
- win10

6
roles/win10/files/archlinux_iommu.conf Normal file
View File

@ -0,0 +1,6 @@
title Archlinux w/ VFIO
linux /vmlinuz-linux
initrd /intel-ucode.img
initrd /initramfs-linux.img
options mitigations=off intel_iommu=on iommu=pt intel_iommu=igfx_off experimental_zcopytx=1 kvm.ignore_msrs=1 acpi_backlight=none root=LABEL=ARCHLINUX rw

55
roles/win10/files/hooks/qemu Executable file
View File

@ -0,0 +1,55 @@
#!/bin/bash
prepare() {
systemctl start win10-usb
# setup cgroupv2
systemctl set-property --runtime -- user.slice AllowedCPUs=0,4
systemctl set-property --runtime -- system.slice AllowedCPUs=0,4
systemctl set-property --runtime -- init.scope AllowedCPUs=0,4
vfio-isolate -u /tmp/win10.undo \
drop-caches \
compact-memory \
irq-affinity mask C1-3,5-7 \
cpu-governor performance C0-7
# vfio-isolate -u /tmp/win10.undo \
# drop-caches \
# compact-memory \
# irq-affinity mask C1-3,5-7 \
# cpuset-create --cpus C0,4 /host.slice \
# cpuset-create --cpus C1-3,5-7 -nlb /win10.slice \
# move-tasks / /host.slice \
# cpu-governor performance C0-7
for _ in $(seq 5); do
sleep 3
# assign hugepages
sysctl -w vm.nr_hugepages=8192 && break
done
sleep 10
}
release() {
echo "Restore system"
# restore cgroupv2
systemctl set-property --runtime -- user.slice AllowedCPUs=0,1,2,3,4,5,6,7
systemctl set-property --runtime -- system.slice AllowedCPUs=0,1,2,3,4,5,6,7
systemctl set-property --runtime -- init.scope AllowedCPUs=0,1,2,3,4,5,6,7
vfio-isolate restore /tmp/win10.undo
systemctl stop win10-usb
sysctl -w vm.nr_hugepages=0
}
if [ "$1" == "win10" ]; then
if [ "$2" == "prepare" ]; then
prepare
fi
if [ "$2" == "release" ]; then
release
fi
fi

1
roles/win10/files/modprobe/kvm.conf
View File

@ -1 +0,0 @@
options kvm_intel nested=1

20
roles/win10/tasks/main.yml
View File

@ -2,10 +2,7 @@
pacman:
name:
- qemu
# - qemu-audio-jack
- libvirt
- dmidecode
- bridge-utils
- ovmf
- virt-manager
- python-evdev
@ -19,7 +16,6 @@
name:
- vfio-isolate
- persistent-evdev-git
- vendor-reset-dkms-git
- name: create hooks directory
file:
@ -27,7 +23,7 @@
state: directory
- name: install qemu hook
template:
copy:
src: hooks/qemu
dest: /etc/libvirt/hooks/qemu
mode: '755'
@ -51,14 +47,14 @@
line: 'MODULES=(vfio_pci vfio vfio_iommu_type1 vfio_virqfd i915)'
notify: regenerate linux initramfs
- name: configure kvm kernel module
copy:
src: modprobe/kvm.conf
dest: /etc/modprobe.d/kvm.conf
- name: configure vfio-pci device ids
template:
src: vfio.conf.j2
dest: /etc/modprobe.d/vfio.conf
notify: regenerate linux initramfs
- name: install boot entry
template:
copy:
src: archlinux_iommu.conf
dest: /boot/loader/entries/archlinux_iommu.conf
@ -99,10 +95,10 @@
block: |
cgroup_device_acl = [
{% for item in keyboards %}
"{{ item }}",
"/dev/input/by-id/uinput-persist-keyboard{{ loop.index0 }}",
{% endfor %}
{% for item in mice %}
"{ item }}",
"/dev/input/by-id/uinput-persist-mouse{{ loop.index0 }}",
{% endfor %}
"/dev/kvm",
"/dev/null", "/dev/full", "/dev/zero",

5
roles/win10/templates/archlinux_iommu.conf
View File

@ -1,5 +0,0 @@
title Archlinux w/ VFIO
linux /vmlinuz-linux
initrd /intel-ucode.img
initrd /initramfs-linux.img
options mitigations=off vfio-pci.ids={{ ','.join(vfio_pci_ids) }} intel_iommu=on iommu=pt intel_iommu=igfx_off hugepages=8192 rcu_nocbs=0-15 split_lock_detect=off kvm.ignore_msrs=1 kvm.report_ignored_msrs=0 root=LABEL=ARCHLINUX rw

42
roles/win10/templates/hooks/qemu
View File

@ -1,42 +0,0 @@
#!/bin/bash
prepare() {
systemctl start win10-usb
# https://github.com/gnif/vendor-reset/issues/46#issuecomment-992282166
{% for pci_device_id in pci_device_ids %}
{% if pci_device_id.fix_reset_method|default(False) %}
echo 'device_specific' >'/sys/bus/pci/devices/{{ pci_device_id.domain }}:{{ pci_device_id.bus }}:{{ pci_device_id.slot }}.{{ pci_device_id.function }}/reset_method'
{% endif %}
{% endfor %}
vfio-isolate -u /tmp/win10.undo \
irq-affinity mask C0-15 \
cpuset-modify --cpus C16-19 /system.slice \
cpuset-modify --cpus C16-19 /user.slice \
cpu-governor performance C0-19
# drop-caches \
# compact-memory \
# for _ in $(seq 10); do
# assign hugepages
# sysctl -w vm.nr_overcommit_hugepages=8192 && break
# sleep 1
# done
# sleep 10
}
release() {
systemctl stop win10-usb
vfio-isolate restore /tmp/win10.undo
# sysctl -w vm.nr_overcommit_hugepages=0
}
if [ "$1" == "win10" ]; then
if [ "$2" == "prepare" ]; then
prepare
fi
if [ "$2" == "release" ]; then
release
fi
fi

7
roles/win10/templates/persistent-evdev/config.json.j2
View File

@ -1,13 +1,12 @@
{
"cache": "/opt/persistent-evdev/cache",
"devices": {
{% if keyboards or mice %}
{% for item in keyboards %}
"persist-keyboard{{loop.index0}}": "{{ item }}"{%if not loop.last or mice %},{% endif %}
"persist-keyboard{{loop.index0}}": "{{ item }}",
{% endfor %}
{% for item in mice %}
"persist-mouse{{loop.index0}}": "{{ item }}"{%if not loop.last %},{% endif %}
"persist-mouse{{loop.index0}}": "{{ item }}",
{% endfor %}
{% endif %}
"dummy" : "/dev/null"
}
}

1
roles/win10/templates/vfio.conf.j2 Normal file
View File

@ -0,0 +1 @@
options vfio-pci ids={{ ','.join(pcie_device_ids) }}

219
roles/win10/templates/win10.xml
View File

@ -12,83 +12,69 @@
<hugepages/>
<locked/>
</memoryBacking>
<vcpu placement='static'>16</vcpu>
<vcpu placement='static'>6</vcpu>
<iothreads>1</iothreads>
<cputune>
<vcpupin vcpu='0' cpuset='0'/>
<vcpupin vcpu='1' cpuset='1'/>
<vcpupin vcpu='0' cpuset='1'/>
<vcpupin vcpu='1' cpuset='5'/>
<vcpupin vcpu='2' cpuset='2'/>
<vcpupin vcpu='3' cpuset='3'/>
<vcpupin vcpu='4' cpuset='4'/>
<vcpupin vcpu='5' cpuset='5'/>
<vcpupin vcpu='6' cpuset='6'/>
<vcpupin vcpu='7' cpuset='7'/>
<vcpupin vcpu='8' cpuset='8'/>
<vcpupin vcpu='9' cpuset='9'/>
<vcpupin vcpu='10' cpuset='10'/>
<vcpupin vcpu='11' cpuset='11'/>
<vcpupin vcpu='12' cpuset='12'/>
<vcpupin vcpu='13' cpuset='13'/>
<vcpupin vcpu='14' cpuset='14'/>
<vcpupin vcpu='15' cpuset='15'/>
<emulatorpin cpuset='16'/>
<iothreadpin cpuset='17' iothread='1'/>
<vcpusched vcpus='0-15' scheduler='fifo' priority='99'/>
<vcpupin vcpu='3' cpuset='6'/>
<vcpupin vcpu='4' cpuset='3'/>
<vcpupin vcpu='5' cpuset='7'/>
<emulatorpin cpuset='0,4'/>
<iothreadpin iothread='1' cpuset='0,4'/>
<vcpusched vcpus='0-5' scheduler='fifo' priority='99'/>
<emulatorsched scheduler='fifo' priority='99'/>
<iothreadsched scheduler='fifo' priority='99' iothreads='1'/>
<iothreadsched iothreads='1' scheduler='fifo' priority='99'/>
</cputune>
<os>
<type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
<smbios mode="host"/>
<type arch='x86_64' machine='pc-q35-4.1'>hvm</type>
<loader readonly='yes' type='pflash'>/usr/share/ovmf/x64/OVMF_CODE.fd</loader>
<nvram>/var/lib/libvirt/qemu/nvram/win10_VARS.fd</nvram>
</os>
<features>
<acpi/>
<apic/>
<hyperv mode='passthrough'>
<hyperv>
<!-- hyper-v enlightments, the more the better -->
<!-- https://archive.fosdem.org/2019/schedule/event/vai_enlightening_kvm/ -->
<!-- <evmcs state='off'/>
<frequencies state='on'/>
<ipi state='on'/>
<reenlightenment state='on'/>
<relaxed state='on'/>
<reset state='on'/>
<runtime state='on'/>
<spinlocks state='on' retries='8191'/>
<stimer state="on">
<direct state="on"/>
</stimer>
<synic state='on'/>
<tlbflush state='on'/>
<vapic state='on'/>
<vendor_id state='on' value='other'/>
<vpindex state='on'/> -->
<spinlocks state='on' retries='8191'/>
<vpindex state='on'/>
<synic state='on'/>
<stimer state='on'/>
<tlbflush state='on'/>
<ipi state='on'/>
<reset state='on'/>
<frequencies state='on'/>
<reenlightenment state='on'/>
<!-- nvidia code 43 fix -->
<vendor_id state='on' value='fNvidia'/>
</hyperv>
<kvm>
<!-- nvidia code 43 fix -->
<hidden state='on'/>
</kvm>
<vmport state='off'/>
<ioapic driver='kvm'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='off'>
<topology sockets='1' cores='8' threads='2'/>
<cpu mode='host-passthrough' check='none'>
<topology sockets='1' cores='3' threads='2'/>
<cache mode='passthrough'/>
<feature policy='require' name='invtsc'/>
<!-- required for some games, comes with a performance hit -->
<!-- <feature policy="disable" name="hypervisor"/> -->
</cpu>
<clock offset='localtime'>
<timer name='tsc' present='yes' mode='native'/>
<timer name='hypervclock' present='yes'/>
<timer name='rtc' present='no' tickpolicy='catchup'/>
<timer name='pit' present='no' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
<timer name='kvmclock' present='no'/>
<timer name='hypervclock' present='yes'/>
<timer name='tsc' present='yes' mode='native'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<!-- Because of the reset bug, we cannot actually restart without restarting the host -->
<on_reboot>destroy</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
@ -96,30 +82,28 @@
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<memballoon model='none'/>
<disk type='file' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native' discard='unmap'/>
<source file='/data/marchambault/VirtualMachines/img/atlas.img'/>
<target dev='sda' bus='scsi' rotation_rate="1"/>
<boot order='2'/>
<driver name='qemu' type='raw'/>
<source file='/home/marchambault/Documents/virtualmachines/img/win10.img'/>
<target dev='sda' bus='scsi'/>
<boot order='1'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native' discard='unmap'/>
<driver name='qemu' type='raw' cache='none' io='native' discard='unmap' detect_zeroes='unmap'/>
<source dev='/dev/disk/by-id/nvme-CT1000P1SSD8_1910E1F03D8E'/>
<target dev='sdc' bus='scsi' rotation_rate="1"/>
<target dev='sdc' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='2'/>
</disk>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<!-- <source file=''/> -->
<source file='/home/marchambault/Documents/virtualmachines/iso/virtio-win-0.1.171.iso'/>
<target dev='sdb' bus='sata'/>
<readonly/>
<boot order='1'/>
<address type='drive' controller='0' bus='0' target='0' unit='1'/>
</disk>
<controller type='scsi' index='0' model='virtio-scsi'>
<driver queues='8' iothread='1'/>
<driver queues='6' iothread='1'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</controller>
<controller type='sata' index='0'>
@ -138,7 +122,7 @@
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
@ -153,6 +137,7 @@
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0x13'/>
<alias name='ua-gfx0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
@ -178,8 +163,6 @@
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</controller>
<!-- networking -->
<interface type='bridge'>
<mac address='52:54:00:f5:2c:df'/>
<source bridge='virbr0'/>
@ -187,97 +170,63 @@
<link state='up'/>
<address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
</interface>
<!-- HID -->
<!-- <input type='mouse' bus='ps2'/> -->
<input type='mouse' bus='virtio'/>
<!-- <input type='keyboard' bus='ps2'/> -->
<input type='keyboard' bus='virtio'/>
<!-- mouse evdev passthrough -->
{% for item in mice %}
<input type='evdev'>
<source dev='/dev/input/by-id/uinput-persist-mouse{{ loop.index0 }}'/>
</input>
{% endfor %}
<!-- keyboard evdev passthrough -->
{% for item in keyboards %}
<input type='evdev'>
<source dev='/dev/input/by-id/uinput-persist-keyboard{{ loop.index0 }}' grab='all' repeat='on'/>
</input>
{% endfor %}
<!-- audio -->
<sound model='ich9'>
<address type='pci' domain='0x0000' bus='0x02' slot='0x02' function='0x0'/>
<codec type='micro'/>
<audio id='1'/>
</sound>
<!-- <audio id='1' type='pulseaudio' serverName='/run/user/1000/pulse/native'>
<input mixingEngine='yes' fixedSettings='yes' voices='1' bufferLength='100'/>
<output mixingEngine='yes' fixedSettings='yes' voices='1' bufferLength='100'>
<settings frequency='48000' channels='2' format='s16'/>
</output>
</audio> -->
<audio id='1' type='pulseaudio' serverName='/run/user/1000/pulse/native'>
<input mixingEngine='no'/>
<output mixingEngine='no'/>
</audio>
<!-- <audio id='1' type='jack'>
<input clientName="win10" connectPorts="{{ audio.source }}"/>
<output clientName="win10" connectPorts="{{ audio.sink }}"/>
</audio> -->
<!-- pci passthrough -->
{% for pci_device_id in pci_device_ids %}
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x{{ pci_device_id.domain }}' bus='0x{{ pci_device_id.bus }}' slot='0x{{ pci_device_id.slot }}' function='0x{{ pci_device_id.function }}'/>
</source>
{% if 'alias' in pci_device_id %}
<alias name='{{ pci_device_id.alias }}'/>
{% endif %}
<!-- connect to virtual pcie controller -->
<address type='pci' domain='0x{{ pci_device_id.domain }}' bus='0x{{ pci_device_id.bus }}' slot='0x{{ pci_device_id.slot }}' function='0x{{ pci_device_id.function }}' multifunction='on'/>
</hostdev>
{% endfor %}
<!-- spice -->
<!-- <graphics type='spice' listen='127.0.0.1' port='5900' defaultMode='insecure'>
<listen type='address' address='127.0.0.1'/>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='spice'>
<listen type='socket' socket='/tmp/win10-spice.sock'/>
<image compression='off' />
<mouse mode='server'/>
<filetransfer enable='no'/>
</graphics>
<sound model='ich9'>
<address type='pci' domain='0x0000' bus='0x02' slot='0x02' function='0x0'/>
<audio id='1'/>
</sound>
<audio id='1' type='pulseaudio' serverName='/run/user/1000/pulse/native'/>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x00' slot='0x1f' function='0x6'/>
</source>
<address type='pci' domain='0x0000' bus='0x02' slot='0x03' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x03' slot='0x00' function='0x1'/>
</source>
<address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
</hostdev>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='2'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='3'/>
</redirdev> -->
<!-- looking-glass -->
<!-- <shmem name='looking-glass'>
</redirdev>
<memballoon model='none'/>
<!-- <memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
</memballoon> -->
<shmem name='looking-glass'>
<model type='ivshmem-plain'/>
<size unit='M'>32</size>
<address type='pci' domain='0x0000' bus='0x02' slot='0x04' function='0x0'/>
</shmem> -->
</shmem>
</devices>
<qemu:override>
<!-- workaround for NVMe passthrough with SMI SM2262 controller -->
<!-- this breaks with >15 vCPUs! -->
<!-- https://bugzilla.kernel.org/show_bug.cgi?id=202055#c42 -->
<!-- <qemu:device alias='ua-nvme1'>
<qemu:frontend>
<qemu:property name='x-msix-relocation' type='string' value='bar2'/>
</qemu:frontend>
</qemu:device> -->
</qemu:override>
<qemu:commandline>
<!-- <qemu:env name="PIPEWIRE_RUNTIME_DIR" value="/run/user/1000"/>
<qemu:env name="PIPEWIRE_LATENCY" value="512/48000"/> -->
<!-- keyboard evdev passthrough -->
{% for item in keyboards %}
<qemu:arg value='-object'/>
<qemu:arg value='input-linux,id=kbd1,evdev=/dev/input/by-id/uinput-persist-keyboard{{ loop.index0 }},grab_all=off,repeat=on'/>
{% endfor %}
<!-- mouse evdev passthrough -->
{% for item in mice %}
<qemu:arg value='-object'/>
<qemu:arg value='input-linux,id=input5,evdev=/dev/input/by-id/uinput-persist-mouse{{ loop.index0 }}'/>
{% endfor %}
</qemu:commandline>
</domain>