使用 Qemu 搭建 aarch64 虚拟机

发表于 更新于 阅读次数：

qemu 手动搭建记录

安装qemu

  1. 在Arch Linux下

执行以下命令。qemu 安装虚拟机本体， qemu 安装非x86_64架构的其他架构。

sudo pacman -S qemu qemu-arch-extra

  1. 在CentOS下。

epel库只带qemu2.0，而要qemu只有3.0以上才支持aarch64，因此需要编译。
从官网下载源码。

1
2
3
4
5
yum install git glib2-devel libfdt-devel pixman-devel zlib-devel
tar xvf qemu-x.tar.xz  
cd qemu-xxxx  
./configure  
make && make install

虚拟机镜像部署

创建虚拟磁盘用来存储系统

1
qemu-img create -f qcow2 centos-aarch64.qcow2 100G

下载aarch64下ovmf的UEFI boot，可以从这里得到，保存为 QEMU_EFI-aarch64.fd。

确认以下文件都存在，且在同一目录下。

  • aarch64的CentOS镜像 CentOS-7-aarch64-Everything-xxxx.iso
  • aarch64的UEFI boot QEMU_EFI-aarch64.fd
  • aarch64的CentOS虚拟机存储文件 centos-aarch64.qcow2

安装系统

1
2
3
4
5
6
7
8
9
10
11
12
qemu-system-aarch64 \
-m 4096 \
-cpu cortex-a72 \
-smp 4 \
-M virt \
-bios QEMU_EFI-aarch64.fd \
-nographic \
-drive if=none,file=CentOS-7-aarch64-Everything-xxxx.iso,id=cdrom,media=cdrom \
-device virtio-scsi-device \
-device scsi-cd,drive=cdrom \
-drive if=none,file=centos-aarch64.qcow2,id=hd0 \
-device virtio-blk-device,drive=hd0

安装完成后运行虚拟机

1
2
3
4
5
6
7
8
9
10
qemu-system-aarch64 \
-m 4096 \
-cpu cortex-a72 \
-smp 4 \
-M virt \
-bios QEMU_EFI-aarch64.fd \
-nographic \
-device virtio-scsi-device \
-drive if=none,file=centos-aarch64.qcow2,id=hd0 \
-device virtio-blk-device,drive=hd0

此时的系统因为没有配置可用的网卡因此无法连接网络。

虚拟机网络配置

NAT

参考qemu文档

安装依赖

1
yum install bridge-utils iptables dnsmasq

将以下脚本保存为 /etc/qemu-ifup 并赋予可执行权限

1
chmod 755 /etc/qemu-ifup

脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#!/bin/sh
#
# Copyright IBM, Corp. 2010  
#
# Authors:
#  Anthony Liguori <aliguori@us.ibm.com>
#
# This work is licensed under the terms of the GNU GPL, version 2.  See
# the COPYING file in the top-level directory.

# Set to the name of your bridge
BRIDGE=br0

# Network information
NETWORK=192.168.53.0
NETMASK=255.255.255.0
GATEWAY=192.168.53.1
DHCPRANGE=192.168.53.2,192.168.53.254

# Optionally parameters to enable PXE support
TFTPROOT=
BOOTP=

do_brctl() {
    brctl "$@"
}

do_ifconfig() {
    ifconfig "$@"
}

do_dd() {
    dd "$@"
}

do_iptables_restore() {
    iptables-restore "$@"
}

do_dnsmasq() {
    dnsmasq "$@"
}

check_bridge() {
    if do_brctl show | grep "^$1" > /dev/null 2> /dev/null; then
	return 1
    else
	return 0
    fi
}

create_bridge() {
    do_brctl addbr "$1"
    do_brctl stp "$1" off
    do_brctl setfd "$1" 0
    do_ifconfig "$1" "$GATEWAY" netmask "$NETMASK" up
}

enable_ip_forward() {
    echo 1 | do_dd of=/proc/sys/net/ipv4/ip_forward > /dev/null
}

add_filter_rules() {
do_iptables_restore <<EOF
# Generated by iptables-save v1.3.6 on Fri Aug 24 15:20:25 2007
*nat
:PREROUTING ACCEPT [61:9671]
:POSTROUTING ACCEPT [121:7499]
:OUTPUT ACCEPT [132:8691]
-A POSTROUTING -s $NETWORK/$NETMASK -j MASQUERADE 
COMMIT
# Completed on Fri Aug 24 15:20:25 2007
# Generated by iptables-save v1.3.6 on Fri Aug 24 15:20:25 2007
*filter
:INPUT ACCEPT [1453:976046]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [1605:194911]
-A INPUT -i $BRIDGE -p tcp -m tcp --dport 67 -j ACCEPT 
-A INPUT -i $BRIDGE -p udp -m udp --dport 67 -j ACCEPT 
-A INPUT -i $BRIDGE -p tcp -m tcp --dport 53 -j ACCEPT 
-A INPUT -i $BRIDGE -p udp -m udp --dport 53 -j ACCEPT 
-A FORWARD -i $1 -o $1 -j ACCEPT 
-A FORWARD -s $NETWORK/$NETMASK -i $BRIDGE -j ACCEPT 
-A FORWARD -d $NETWORK/$NETMASK -o $BRIDGE -m state --state RELATED,ESTABLISHED -j ACCEPT 
-A FORWARD -o $BRIDGE -j REJECT --reject-with icmp-port-unreachable 
-A FORWARD -i $BRIDGE -j REJECT --reject-with icmp-port-unreachable 
COMMIT
# Completed on Fri Aug 24 15:20:25 2007
EOF
}

start_dnsmasq() {
    do_dnsmasq \
	--strict-order \
	--except-interface=lo \
	--interface=$BRIDGE \
	--listen-address=$GATEWAY \
	--bind-interfaces \
	--dhcp-range=$DHCPRANGE \
	--conf-file="" \
	--pid-file=/var/run/qemu-dnsmasq-$BRIDGE.pid \
	--dhcp-leasefile=/var/run/qemu-dnsmasq-$BRIDGE.leases \
	--dhcp-no-override \
	${TFTPROOT:+"--enable-tftp"} \
	${TFTPROOT:+"--tftp-root=$TFTPROOT"} \
	${BOOTP:+"--dhcp-boot=$BOOTP"}
}

setup_bridge_nat() {
    if check_bridge "$1" ; then
	create_bridge "$1"
	enable_ip_forward
	add_filter_rules "$1"
	start_dnsmasq "$1"
    fi
}

setup_bridge_vlan() {
    if check_bridge "$1" ; then
	create_bridge "$1"
	start_dnsmasq "$1"
    fi
}

setup_bridge_nat "$BRIDGE"

if test "$1" ; then
    do_ifconfig "$1" 0.0.0.0 up
    do_brctl addif "$BRIDGE" "$1"
fi

运行虚拟机

1
2
3
4
5
6
7
8
9
10
11
12
qemu-system-aarch64 \
-m 4096 \
-cpu cortex-a72 \
-smp 4 \
-M virt \
-bios QEMU_EFI-aarch64.fd \
-nographic \
-device virtio-scsi-device \
-drive if=none,file=centos-aarch64.qcow2,id=hd0 \
-device virtio-blk-device,drive=hd0 \
-net tap \
-net nic

桥接

创建br0虚拟网卡并挂载到可连接外网的网卡上，以eth0举例

1
2
3
4
sudo ip link add name br0 type bridge
sudo ip link set dev br0 up
sudo ip link set eth0 promisc on
sudo ip link set dev eth0 master br0

此时Host原ip会不可连，需给br0赋予ip

1
sudo systemctl start dhcpcd@br0

允许qemu通过br0连接网络

1
2
echo "allow br0" >> /etc/qemu/bridge.conf
echo "allow br0" >> /usr/local/etc/qemu/bridge.conf

运行虚拟机

1
2
3
4
5
6
7
8
9
10
11
12
qemu-system-aarch64 \
-m 4096 \
-cpu cortex-a72 \
-smp 4 \
-M virt \
-bios QEMU_EFI-aarch64.fd \
-nographic \
-device virtio-scsi-device \
-drive if=none,file=centos-aarch64.qcow2,id=hd0 \
-device virtio-blk-device,drive=hd0 \
-net nic \
-net bridge,br=br0

SR-IOV

准备 SR-IOV 环境

参考另一篇文章

确认 IOMMU 功能确实打开

1
2
3
4
5
dmesg | grep -e DMAR -e IOMMU

...
DMAR: IOMMU enabled
...

确认 Virtual Functional 已经开启

1
2
3
4
5
6
lspci -vv  | grep Eth

18:00.0 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01)
18:00.1 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01)
18:10.0 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)
18:10.2 Ethernet controller: Intel Corporation 82599 Ethernet Controller Virtual Function (rev 01)

给设备挂载 vfio 驱动

挂载 vfio-pci 驱动可以通过手动挂载，也可以通过 DPDK 的 usertools 下的脚本进行挂载。

加载 vfio 相关模块

1
2
modprobe vfio
modprobe vfio-pci

确认加载是否成功

1
2
lsmod | grep vfio
...

解绑原有设备驱动

1
2
3
4
5
# 查看目标网卡设备锁在 iommu_group
ls /sys/bus/pci/devices/0000:18:10.0/iommu_group/devices

# 解绑驱动
echo 0000:18:10.0 > /sys/bus/pci/devices/0000:18:10.0/driver/unbind

获取设备厂商号和型号

1
2
lspci -ns 0000:18:10.0
18:10.0 0300: 8086:370d (rev 06)

绑定 vfio-pci 驱动

1
echo 8086 370d > /sys/bus/pci/drivers/vfio-pci/new_id

运行虚拟机

1
2
3
4
5
6
7
8
9
10
11
12
13
qemu-system-aarch64 \
-m 4096 \
-cpu cortex-a72 \
-smp 4 \
-M virt \
-bios QEMU_EFI-aarch64.fd \
-nographic \
-device virtio-scsi-device \
-drive if=none,file=centos-aarch64.qcow2,id=hd0 \
-device virtio-blk-device,drive=hd0 \
-net tap \
-net nic \
-device vfio-pci,host=0000:18:10.0