Building a Minimal Virtual Machine
Foreword
KVM (Kernel-based Virtual Machine) on the ARM architecture is a virtualization technology based on the Linux kernel. It allows user-space programs to control hardware directly through system calls, enabling virtual machine management. KVM is already very mature on x86, and on ARM it has also received broad support and development.
PS: This article was originally published on Youlan Wiki - Building a Minimal Virtual Machine.
KVM Characteristics on ARM
Hardware support: In recent years, ARM processors have added support for virtualization, including the Virtualization Extensions in the ARMv8-A architecture, which makes KVM on ARM possible.
- Performance advantage: KVM delivers near-bare-metal performance because it runs directly on top of hardware, reducing the overhead of traditional virtualization solutions.
- Compatibility: KVM supports a wide range of guest operating systems, including but not limited to Linux, Windows, and various Unix-like operating systems.
- Management tools: KVM can be used together with QEMU (Quick EMUlator). QEMU provides the user-space components needed for virtual machine management and device emulation. In addition, tools such as libvirt provide advanced virtual machine management capabilities.
How ARM KVM Is Implemented
Implementing KVM virtualization on the ARM platform mainly relies on the following components:
- Linux kernel: The ARM version of the Linux kernel includes KVM support, allowing user-space applications to directly access virtualization hardware resources.
- QEMU: An open-source machine emulator that can be used to start virtual machines and provide them with an emulated hardware environment.
- libvirt: A software suite for managing virtualization; it can simplify KVM management and deployment.
ARM KVM Use Cases
ARM KVM is mainly used in the following scenarios:
- Data centers: As ARM server chips continue to evolve, more and more data centers are adopting ARM-based servers, and KVM helps them achieve efficient virtualization.
- Embedded systems: ARM chips are widely used in embedded systems, and KVM provides a flexible way to test and develop embedded systems.
- Cloud computing platforms: ARM KVM can be used to build cloud platforms that provide high-performance virtualization services.
Writing a Minimal Virtual Machine in C
This article will try to write a minimal virtual machine in C that can run simple bare-metal programs on the ARM AArch64 instruction set on a Youlan laptop.
Environment Setup
The Youlan laptop already includes the GCC development toolchain, so there is no need to set up an additional development environment. We can start coding directly.
Experimental Code
- First we need to include the necessary header files:
1// Include basic libc headers
2#include <err.h>
3#include <stdint.h>
4#include <stdio.h>
5#include <string.h>
6#include <stddef.h>
7#include <unistd.h>
8
9// Include the mmap header for allocating guest memory
10#include <sys/mman.h>
11
12// Include I/O-related headers for accessing files
13#include <fcntl.h>
14#include <sys/ioctl.h>
15
16// Include KVM-related headers for configuring the virtual machine
17#include <linux/kvm.h>
- Then perform KVM initialization:
1int main(void)
2{
3 ...
4
5 /* Open the kvm file and obtain its fd */
6 int kvmfd = open("/dev/kvm", O_RDWR);
7 if (kvmfd == -1)
8 err(1, "/dev/kvm");
9 else
10 printf("[%d] Open kvm succesfuly, fd is %d\n", ++step, kvmfd);
11
12 /* Make sure the KVM API version is 12 */
13 ret = ioctl(kvmfd, KVM_GET_API_VERSION, NULL);
14 if (ret < 0)
15 err(1, "KVM_GET_API_VERSION");
16 if (ret != 12)
17 errx(1, "KVM_GET_API_VERSION %d, expected 12", ret);
18 else
19 printf("[%d] Get kvm version %d\n", ++step, ret);
20
21 /* 1. Get the VM ID */
22 int vmfd = ioctl(kvmfd, KVM_CREATE_VM, (unsigned long)0);
23 if (vmfd < 0)
24 err(1, "KVM_CREATE_VM");
25 else
26 printf("[%d] Create VM succesfuly, fd is %d\n", ++step, vmfd);
27 ...
28}
- Then configure the VM, which means configuring the vCPU:
1int main()
2{
3 ...
4 /* 2. Create the vCPU */
5 int vcpufd = ioctl(vmfd, KVM_CREATE_VCPU, (unsigned long)0);
6 if (vcpufd < 0)
7 err(1, "KVM_CREATE_VCPU");
8 else
9 printf("[%d] Create vCPU succesfuly, fd is %d\n", ++step, vcpufd);
10
11 /* 3. Set the vCPU type; here it is ARMv8 */
12 // sample code can check qemu/target/arm/kvm64.c
13 memset(&init, 0, sizeof(init));
14 init.target = KVM_ARM_TARGET_GENERIC_V8;
15 ret = ioctl(vcpufd, KVM_ARM_VCPU_INIT, &init);
16 if (ret < 0)
17 err(1, "init vcpu type failed\n");
18 else
19 printf("[%d] Set vCPU type is Aarch64 (ARMv8)\n", ++step);
20
21 /* 4. Allocate memory for kvm_run */
22 mmap_size = ioctl(kvmfd, KVM_GET_VCPU_MMAP_SIZE, NULL);
23 if (mmap_size < 0)
24 err(1, "KVM_GET_VCPU_MMAP_SIZE");
25 if (mmap_size < sizeof(*run))
26 errx(1, "KVM_GET_VCPU_MMAP_SIZE unexpectedly small");
27 run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, vcpufd, 0);
28 if (run == MAP_FAILED)
29 err(1, "mmap vcpu");
30 else
31 printf("[%d] Init kvm_run successfuly!\n", ++step);
32}
- Write a simple guest program and copy it into guest memory:
1const unsigned code[] = {
2 // Write "Hello" to port 0x996
3 0xd28132c4, // mov x4, #0x996 // #2454
4 0xd2800905, // mov x5, #0x48 // H
5 0x39000085, // strb w5, [x4]
6 0xd2800ca5, // mov x5, #0x65 // e
7 0x39000085, // strb w5, [x4]
8 0xd2800d85, // mov x5, #0x6c // ll
9 0x39000085, // strb w5, [x4]
10 0x39000085, // strb w5, [x4]
11 0xd2800de5, // mov x5, #0x6f // o
12 0x39000085, // strb w5, [x4]
13 0xd2800145, // mov x5, #0xa // \n
14 0x39000085, // strb w5, [x4]
15};
16#define MEM_SIZE 0x1000
17...
18int main()
19{
20 ...
21 /* 5. Copy the program into guest memory */
22 ram = mmap(NULL, MEM_SIZE, PROT_READ | PROT_WRITE,
23 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
24 if (!ram)
25 err(1, "allocating guest memory");
26 memcpy(ram, code, sizeof(code));
27 printf("[%d] Load the vm running program to buffer 'ram'\n", ++step);
28 ...
29}
- Initialize userspace_memory_region and set vCPU registers:
1int main()
2{
3 ...
4
5 /* 6. Set the VM userspace memory region and bind it to vmfd */
6 struct kvm_userspace_memory_region region = {
7 .slot = 0,
8 .flags = 0,
9 .memory_size = MEM_SIZE,
10 .guest_phys_addr = PHY_ADDR,
11 .userspace_addr = (unsigned long)ram,
12 };
13 ret = ioctl(vmfd, KVM_SET_USER_MEMORY_REGION, ®ion);
14 if (ret < 0)
15 err(1, "KVM_SET_USER_MEMORY_REGION");
16 else
17 printf("[%d] Set the vm userspace program ram to vm fd handler\n", ++step);
18
19 /* 7. Set the vCPU PC register to the guest's first instruction address */
20 reg.id = ARM64_CORE_REG(regs.pc);
21 reg.addr = (__u64)&guest_entry;
22 ret = ioctl(vcpufd, KVM_SET_ONE_REG, ®);
23 if (ret < 0)
24 err(1,"KVM_SET_ONE_REG failed (pc)");
25 else
26 printf("[%d] Set vCPU PC, is 0x%x\n", ++step, (unsigned)guest_entry);
27 ...
28}
- Handle VM runtime logic and add I/O emulation:
1#define PHY_ADDR 0x10000
2int main()
3{
4 ...
5 /* 8. VM runtime handling */
6 printf("[%d] Run vCPU and print message:\n", ++step);
7
8 while (1) {
9 ret = ioctl(vcpufd, KVM_RUN, NULL);
10 if (ret < 0)
11 err(1, "KVM_RUN");
12
13 switch (run->exit_reason) {
14 case KVM_EXIT_MMIO:
15 if (run->mmio.is_write && run->mmio.len == 1) {
16 printf("%c", run->mmio.data[0]);
17 }
18 if (run->mmio.data[0] == '\n')
19 return 0;
20 else
21 break;
22 case KVM_EXIT_FAIL_ENTRY:
23 errx(1, "KVM_EXIT_FAIL_ENTRY: hardware_entry_failure_reason = 0x%llx",
24 (unsigned long long)run->fail_entry.hardware_entry_failure_reason);
25 case KVM_EXIT_INTERNAL_ERROR:
26 errx(1, "KVM_EXIT_INTERNAL_ERROR: suberror = 0x%x", run->internal.suberror);
27 default:
28 errx(1, "exit_reason = 0x%x", run->exit_reason);
29 }
30 }
31
32 return 0;
33}
The runtime result is as follows:
1geduer@ulan:~/gevico/kvm$ gcc main.c && ./a.out
2[1] Open kvm succesfuly, fd is 3
3[2] Get kvm version 12
4[3] Create VM succesfuly, fd is 4
5[4] Create vCPU succesfuly, fd is 5
6[5] Set vCPU type is Aarch64 (ARMv8)
7[6] Init kvm_run successfuly!
8[7] Load the vm running program to buffer 'ram'
9[8] Set the vm userspace program ram to vm fd handler
10[9] Set vCPU PC, is 0x10000
11[10] Run vCPU and print message:
12Hello
Full Code
The complete code is as follows:
1#include <err.h>
2#include <stdint.h>
3#include <stdio.h>
4#include <string.h>
5#include <stddef.h>
6#include <unistd.h>
7#include <sys/mman.h>
8#include <fcntl.h>
9#include <sys/ioctl.h>
10#include <linux/kvm.h>
11
12#define MEM_SIZE 0x1000
13#define PHY_ADDR 0x10000
14
15static __u64 __core_reg_id(__u64 offset)
16{
17 __u64 id = KVM_REG_ARM64 | KVM_REG_ARM_CORE | offset;
18
19 if (offset < KVM_REG_ARM_CORE_REG(fp_regs))
20 id |= KVM_REG_SIZE_U64;
21 else if (offset < KVM_REG_ARM_CORE_REG(fp_regs.fpsr))
22 id |= KVM_REG_SIZE_U128;
23 else
24 id |= KVM_REG_SIZE_U32;
25
26 return id;
27}
28
29#define ARM64_CORE_REG(x) __core_reg_id(KVM_REG_ARM_CORE_REG(x))
30
31const unsigned code[] = {
32 // Write "Hello" to port 0x996
33 0xd28132c4, // mov x4, #0x996 // #2454
34 0xd2800905, // mov x5, #0x48 // H
35 0x39000085, // strb w5, [x4]
36 0xd2800ca5, // mov x5, #0x65 // e
37 0x39000085, // strb w5, [x4]
38 0xd2800d85, // mov x5, #0x6c // ll
39 0x39000085, // strb w5, [x4]
40 0x39000085, // strb w5, [x4]
41 0xd2800de5, // mov x5, #0x6f // o
42 0x39000085, // strb w5, [x4]
43 0xd2800145, // mov x5, #0xa // \n
44 0x39000085, // strb w5, [x4]
45};
46
47int main(void)
48{
49 /* Initialize registers: the instruction pointer for our code, addends, and
50 * the initial flags required by the AArch64 architecture. */
51 struct kvm_one_reg reg;
52 struct kvm_vcpu_init init; // use init to set the vcpu type
53 struct kvm_vcpu_init preferred;
54 __u64 guest_entry = PHY_ADDR;
55 int ret;
56 int step = 0;
57
58 uint8_t *ram;
59 size_t mmap_size;
60 struct kvm_run *run;
61
62 int kvmfd = open("/dev/kvm", O_RDWR);
63 if (kvmfd == -1)
64 err(1, "/dev/kvm");
65 else
66 printf("[%d] Open kvm succesfuly, fd is %d\n", ++step, kvmfd);
67
68 /* Make sure we have the stable version of the API */
69 ret = ioctl(kvmfd, KVM_GET_API_VERSION, NULL);
70 if (ret < 0)
71 err(1, "KVM_GET_API_VERSION");
72 if (ret != 12)
73 errx(1, "KVM_GET_API_VERSION %d, expected 12", ret);
74 else
75 printf("[%d] Get kvm version %d\n", ++step, ret);
76
77 /* 1. Create the VM and get the vm fd handler */
78 int vmfd = ioctl(kvmfd, KVM_CREATE_VM, (unsigned long)0);
79 if (vmfd < 0)
80 err(1, "KVM_CREATE_VM");
81 else
82 printf("[%d] Create VM succesfuly, fd is %d\n", ++step, vmfd);
83
84 /* 2. Create the vCPU */
85 int vcpufd = ioctl(vmfd, KVM_CREATE_VCPU, (unsigned long)0);
86 if (vcpufd < 0)
87 err(1, "KVM_CREATE_VCPU");
88 else
89 printf("[%d] Create vCPU succesfuly, fd is %d\n", ++step, vcpufd);
90
91 /* 3. Initialize the arm64 vCPU type */
92 // sample code can check qemu/target/arm/kvm64.c
93 memset(&init, 0, sizeof(init));
94 init.target = KVM_ARM_TARGET_GENERIC_V8;
95 ret = ioctl(vcpufd, KVM_ARM_VCPU_INIT, &init);
96 if (ret < 0)
97 err(1, "init vcpu type failed\n");
98 else
99 printf("[%d] Set vCPU type is Aarch64 (ARMv8)\n", ++step);
100
101 /* 4. Map the shared kvm_run structure and following data. */
102 mmap_size = ioctl(kvmfd, KVM_GET_VCPU_MMAP_SIZE, NULL);
103 if (mmap_size < 0)
104 err(1, "KVM_GET_VCPU_MMAP_SIZE");
105 if (mmap_size < sizeof(*run))
106 errx(1, "KVM_GET_VCPU_MMAP_SIZE unexpectedly small");
107 run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, vcpufd, 0);
108 if (run == MAP_FAILED)
109 err(1, "mmap vcpu");
110 else
111 printf("[%d] Init kvm_run successfuly!\n", ++step);
112
113 /* 5. Load the program into buffer 'ram' */
114 ram = mmap(NULL, MEM_SIZE, PROT_READ | PROT_WRITE,
115 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
116 if (!ram)
117 err(1, "allocating guest memory");
118 memcpy(ram, code, sizeof(code));
119 printf("[%d] Load the vm running program to buffer 'ram'\n", ++step);
120
121 /* 6. Set the VM userspace memory region and bind it to vmfd */
122 struct kvm_userspace_memory_region region = {
123 .slot = 0,
124 .flags = 0,
125 .memory_size = MEM_SIZE,
126 .guest_phys_addr = PHY_ADDR,
127 .userspace_addr = (unsigned long)ram,
128 };
129 ret = ioctl(vmfd, KVM_SET_USER_MEMORY_REGION, ®ion);
130 if (ret < 0)
131 err(1, "KVM_SET_USER_MEMORY_REGION");
132 else
133 printf("[%d] Set the vm userspace program ram to vm fd handler\n", ++step);
134
135 /* 7. Set the vCPU PC register to the guest's first instruction address */
136 reg.id = ARM64_CORE_REG(regs.pc);
137 reg.addr = (__u64)&guest_entry;
138 ret = ioctl(vcpufd, KVM_SET_ONE_REG, ®);
139 if (ret < 0)
140 err(1,"KVM_SET_ONE_REG failed (pc)");
141 else
142 printf("[%d] Set vCPU PC, is 0x%x\n", ++step, (unsigned)guest_entry);
143
144 /* 8. Run the VM and handle exits at runtime. */
145 printf("[%d] Run vCPU and print message:\n", ++step);
146
147 while (1) {
148 ret = ioctl(vcpufd, KVM_RUN, NULL);
149 if (ret < 0)
150 err(1, "KVM_RUN");
151
152 switch (run->exit_reason) {
153 case KVM_EXIT_MMIO:
154 if (run->mmio.is_write && run->mmio.len == 1) {
155 printf("%c", run->mmio.data[0]);
156 }
157 if (run->mmio.data[0] == '\n')
158 return 0;
159 else
160 break;
161 case KVM_EXIT_FAIL_ENTRY:
162 errx(1, "KVM_EXIT_FAIL_ENTRY: hardware_entry_failure_reason = 0x%llx",
163 (unsigned long long)run->fail_entry.hardware_entry_failure_reason);
164 case KVM_EXIT_INTERNAL_ERROR:
165 errx(1, "KVM_EXIT_INTERNAL_ERROR: suberror = 0x%x", run->internal.suberror);
166 default:
167 errx(1, "exit_reason = 0x%x", run->exit_reason);
168 }
169 }
170
171 return 0;
172}
References: