RISC 内存虚拟化在 KVM 及 kvmtool 中的实现

/*
 * Address types:
 *
 *  gva - guest virtual address
 *  gpa - guest physical address
 *  gfn - guest frame number
 *  hva - host virtual address
 *  hpa - host physical address
 *  hfn - host frame number
 */

typedef unsigned long  gva_t;
typedef u64            gpa_t;
typedef u64            gfn_t;

#define GPA_INVALID	(~(gpa_t)0)

typedef unsigned long  hva_t;
typedef u64            hpa_t;
typedef u64            hfn_t;

// arch/riscv/kvm/mmu.c: line 712

int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm)
{
	struct page *pgd_page;
	// 是否已经为 VM 分配了目录页号
	if (kvm->arch.pgd != NULL) {
		kvm_err("kvm_arch already initialized?\n");
		return -EINVAL;
	}
	// 分配
	pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
				get_order(gstage_pgd_size));
	if (!pgd_page)
		return -ENOMEM;
	kvm->arch.pgd = page_to_virt(pgd_page);
	kvm->arch.pgd_phys = page_to_phys(pgd_page);

	return 0;
}

// include/linux/gfp_types.h: line 333
#define GFP_KERNEL	(__GFP_RECLAIM | __GFP_IO | __GFP_FS)
// include/linux/gfp_types.h: line 249
#define __GFP_ZERO	((__force gfp_t)___GFP_ZERO)

// arch/riscv/include/asm/csr.h: line 139
#define HGATP_PAGE_SHIFT	12
// include/linux/gfp_types.h: line 32
#define gstage_pgd_xbits	2
#define gstage_pgd_size	(1UL << (HGATP_PAGE_SHIFT + gstage_pgd_xbits))

// include/linux/gfp.h: line 275
static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order)
{
	return alloc_pages_node(numa_node_id(), gfp_mask, order);
}

// include/linux/gfp.h: line 260
/*
 * Allocate pages, preferring the node given as nid. When nid == NUMA_NO_NODE,
 * prefer the current CPU's closest node. Otherwise node must be valid and
 * online.
 */
static inline struct page *alloc_pages_node(int nid, gfp_t gfp_mask,
						unsigned int order)
{
	if (nid == NUMA_NO_NODE)
		nid = numa_mem_id();

	return __alloc_pages_node(nid, gfp_mask, order);
}

// include/linux/gfp.h: line 237
/*
 * Allocate pages, preferring the node given as nid. The node must be valid and
 * online. For more general interface, see alloc_pages_node().
 */
static inline struct page *
__alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order)
{
	VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);
	VM_WARN_ON((gfp_mask & __GFP_THISNODE) && !node_online(nid));

	return __alloc_pages(gfp_mask, order, nid, NULL);
}

// mm/page_alloc.c: line 5513
/*
 * This is the 'heart' of the zoned buddy allocator.
 */
struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,
							nodemask_t *nodemask)
{
	struct page *page;

	/* 页分配之前的准备工作，此处代码略去 */
	/* Preparation code, omitted here */
	// ...

	/* 先从 freelist 中申请内存 */
	/* First allocation attempt */
	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
	if (likely(page))
		goto out;

	// ...

	/* 如果从 freelist 中申请失败，则需要在内存回收后进行分配，即 slow path */
	page = __alloc_pages_slowpath(alloc_gfp, order, &ac);

out:

	// ...

	return page;
}

// arch/riscv/kvm/mmu.c: line 731
void kvm_riscv_gstage_free_pgd(struct kvm *kvm)
{
	void *pgd = NULL;

	spin_lock(&kvm->mmu_lock);
	if (kvm->arch.pgd) {
		gstage_unmap_range(kvm, 0UL, gstage_gpa_size, false);
		pgd = READ_ONCE(kvm->arch.pgd);
		kvm->arch.pgd = NULL;
		kvm->arch.pgd_phys = 0;
	}
	spin_unlock(&kvm->mmu_lock);

	if (pgd)
		free_pages((unsigned long)pgd, get_order(gstage_pgd_size));
}

// mm/page_alloc.c: 5641
void free_pages(unsigned long addr, unsigned int order)
{
	if (addr != 0) {
		VM_BUG_ON(!virt_addr_valid((void *)addr));
		__free_pages(virt_to_page((void *)addr), order);
	}
}

// mm/page_alloc.c: 5631
void __free_pages(struct page *page, unsigned int order)
{
	if (put_page_testzero(page))
		free_the_page(page, order);
	else if (!PageHead(page))
		while (order-- > 0)
			free_the_page(page + (1 << order), order);
}

// mm/page_alloc.c: 764
static inline void free_the_page(struct page *page, unsigned int order)
{
	if (pcp_allowed_order(order))		/* Via pcp? */
		free_unref_page(page, order);
	else
		__free_pages_ok(page, order, FPI_NONE);
}

// arch/riscv/kvm/vcpu_exit.c: line 12
static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run,
			     struct kvm_cpu_trap *trap)
{
	struct kvm_memory_slot *memslot;
	unsigned long hva, fault_addr;
	bool writable;
	gfn_t gfn;
	int ret;

	// 从 trap 信息中获取页错误的地址 gpa (Guest Physical Address)
	// get page fault address from trap information
	fault_addr = (trap->htval << 2) | (trap->stval & 0x3);

	// 将发生错误的地址先转换为 Guest 页号（Guest Frame Number）、再转换为可以在 hypervisor 中进行处理的 hva（Hypervisor Virtual Address）
	gfn = fault_addr >> PAGE_SHIFT;
	memslot = gfn_to_memslot(vcpu->kvm, gfn);
	// 返回 gfn 对应的 hva 及其读写属性
	// Return the hva of a @gfn and the R/W attribute if possible
	hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);

	// 依据 scause CSR 的值判定当前的 page fault 是何类型（Load/Store）并进行相应处理
	if (kvm_is_error_hva(hva) ||
	    (trap->scause == EXC_STORE_GUEST_PAGE_FAULT && !writable)) {
		switch (trap->scause) {
		case EXC_LOAD_GUEST_PAGE_FAULT:
			return kvm_riscv_vcpu_mmio_load(vcpu, run,
							fault_addr,
							trap->htinst);
		case EXC_STORE_GUEST_PAGE_FAULT:
			return kvm_riscv_vcpu_mmio_store(vcpu, run,
							 fault_addr,
							 trap->htinst);
		default:
			return -EOPNOTSUPP;
		};
	}

	// 进行 G-Stage 的地址映射
	ret = kvm_riscv_gstage_map(vcpu, memslot, fault_addr, hva,
		(trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false);
	if (ret < 0)
		return ret;

	return 1;
}

// arch/riscv/include/asm/kvm_host.h
struct kvm_cpu_trap {
	unsigned long sepc;
	unsigned long scause;
	unsigned long stval;
	unsigned long htval;
	unsigned long htinst;
};

// arch/riscv/kvm/mmu.c: line 617
int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu,
			 struct kvm_memory_slot *memslot,
			 gpa_t gpa, unsigned long hva, bool is_write)
{
	int ret;
	kvm_pfn_t hfn;
	bool writable;
	short vma_pageshift;
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct vm_area_struct *vma;
	struct kvm *kvm = vcpu->kvm;
	struct kvm_mmu_memory_cache *pcache = &vcpu->arch.mmu_page_cache;
	bool logging = (memslot->dirty_bitmap &&
			!(memslot->flags & KVM_MEM_READONLY)) ? true : false;
	unsigned long vma_pagesize, mmu_seq;

	mmap_read_lock(current->mm);

	vma = find_vma_intersection(current->mm, hva, hva + 1);
	if (unlikely(!vma)) {
		kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
		mmap_read_unlock(current->mm);
		return -EFAULT;
	}

	if (is_vm_hugetlb_page(vma))
		vma_pageshift = huge_page_shift(hstate_vma(vma));
	else
		vma_pageshift = PAGE_SHIFT;
	vma_pagesize = 1ULL << vma_pageshift;
	if (logging || (vma->vm_flags & VM_PFNMAP))
		vma_pagesize = PAGE_SIZE;

	if (vma_pagesize == PMD_SIZE || vma_pagesize == PGDIR_SIZE)
		gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT;

	mmap_read_unlock(current->mm);

	if (vma_pagesize != PGDIR_SIZE &&
	    vma_pagesize != PMD_SIZE &&
	    vma_pagesize != PAGE_SIZE) {
		kvm_err("Invalid VMA page size 0x%lx\n", vma_pagesize);
		return -EFAULT;
	}

	/* We need minimum second+third level pages */
	ret = kvm_mmu_topup_memory_cache(pcache, gstage_pgd_levels);
	if (ret) {
		kvm_err("Failed to topup G-stage cache\n");
		return ret;
	}

	mmu_seq = kvm->mmu_invalidate_seq;

	hfn = gfn_to_pfn_prot(kvm, gfn, is_write, &writable);
	if (hfn == KVM_PFN_ERR_HWPOISON) {
		send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva,
				vma_pageshift, current);
		return 0;
	}
	if (is_error_noslot_pfn(hfn))
		return -EFAULT;

	/*
	 * If logging is active then we allow writable pages only
	 * for write faults.
	 */
	if (logging && !is_write)
		writable = false;

	spin_lock(&kvm->mmu_lock);

	if (mmu_invalidate_retry(kvm, mmu_seq))
		goto out_unlock;

	if (writable) {
		kvm_set_pfn_dirty(hfn);
		mark_page_dirty(kvm, gfn);
		ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT,
				      vma_pagesize, false, true);
	} else {
		ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT,
				      vma_pagesize, true, true);
	}

	if (ret)
		kvm_err("Failed to map in G-stage\n");

out_unlock:
	spin_unlock(&kvm->mmu_lock);
	kvm_set_pfn_accessed(hfn);
	kvm_release_pfn_clean(hfn);
	return ret;
}

// arch/riscv/kvm/mmu.c: line 177
static int gstage_map_page(struct kvm *kvm,
			   struct kvm_mmu_memory_cache *pcache,
			   gpa_t gpa, phys_addr_t hpa,
			   unsigned long page_size,
			   bool page_rdonly, bool page_exec)
{
	int ret;
	u32 level = 0;
	pte_t new_pte;
	pgprot_t prot;

	// 根据 page_size 确定页所在层级（level），如果没有对应大小的页，则返回非 0 值
	ret = gstage_page_size_to_level(page_size, &level);
	if (ret)
		return ret;

	/*
	 * A RISC-V implementation can choose to either:
	 * 1) Update 'A' and 'D' PTE bits in hardware
	 * 2) Generate page fault when 'A' and/or 'D' bits are not set
	 *    PTE so that software can update these bits.
	 *
	 * We support both options mentioned above. To achieve this, we
	 * always set 'A' and 'D' PTE bits at time of creating G-stage
	 * mapping. To support KVM dirty page logging with both options
	 * mentioned above, we will write-protect G-stage PTEs to track
	 * dirty pages.
	 */

	/* 基于 RISC-V 指令集手册的 PTE 更新机制的实现可以有两种选择，
	 * 即在硬件中更新页表项（PTE）的 A（Access）/D（Dirty）位，
	 * 或默认 PTE 的 A/D 位不设置，当访问未初始化的 PTE 时产生 page-fault
	 * 从而使软件更新上述标志位。
	 *
	 * KVM 的实现中同时支持了上述两种机制：在进行 G-Stage 地址映射时就初始化
	 * 对应标志位，同时通过对 G-Stage PTEs 的写保护达成了软件层面的脏页追踪机制。
	 */

	// 获取要执行操作的页的访问权限
	if (page_exec) {
		if (page_rdonly)
			prot = PAGE_READ_EXEC;
		else
			prot = PAGE_WRITE_EXEC;
	} else {
		if (page_rdonly)
			prot = PAGE_READ;
		else
			prot = PAGE_WRITE;
	}
	// 设置 hpa 对应页的权限位并标记为 dirty
	new_pte = pfn_pte(PFN_DOWN(hpa), prot);
	new_pte = pte_mkdirty(new_pte);

	// 设置此次 page-fault 处理的页的内容
	return gstage_set_pte(kvm, level, pcache, gpa, &new_pte);
}

// arch/riscv/kvm/mmu.c: line 137
static int gstage_set_pte(struct kvm *kvm, u32 level,
			   struct kvm_mmu_memory_cache *pcache,
			   gpa_t addr, const pte_t *new_pte)
{
	u32 current_level = gstage_pgd_levels - 1;
	pte_t *next_ptep = (pte_t *)kvm->arch.pgd;
	pte_t *ptep = &next_ptep[gstage_pte_index(addr, current_level)];

	if (current_level < level)
		return -EINVAL;

	while (current_level != level) {
		if (gstage_pte_leaf(ptep))
			return -EEXIST;

		// 若当前页表项无效，则根据 pcache 的内容有效性选择申请页表项（kvm_mmu_memory_cache_alloc）或直接返回错误代码
		if (!pte_val(*ptep)) {
			if (!pcache)
				return -ENOMEM;
			next_ptep = kvm_mmu_memory_cache_alloc(pcache);
			if (!next_ptep)
				return -ENOMEM;
			*ptep = pfn_pte(PFN_DOWN(__pa(next_ptep)),
					__pgprot(_PAGE_TABLE));
		} else {
			if (gstage_pte_leaf(ptep))
				return -EEXIST;
			next_ptep = (pte_t *)gstage_pte_page_vaddr(*ptep);
		}

		current_level--;
		ptep = &next_ptep[gstage_pte_index(addr, current_level)];
	}

	// 为找到的页表项赋值（保存之前设置的权限位和脏页标志位）
	*ptep = *new_pte;
	// 倘若为叶子页表，刷新 TLB 对应项
	if (gstage_pte_leaf(ptep))
		gstage_remote_tlb_flush(kvm, current_level, addr);

	return 0;
}

// arch/riscv/kvm/mmu.c: line 42
static inline unsigned long gstage_pte_index(gpa_t addr, u32 level)
{
	unsigned long mask;
	unsigned long shift = HGATP_PAGE_SHIFT + (gstage_index_bits * level);

	if (level == (gstage_pgd_levels - 1))
		mask = (PTRS_PER_PTE * (1UL << gstage_pgd_xbits)) - 1;
	else
		mask = PTRS_PER_PTE - 1;

	return (addr >> shift) & mask;
}

// arch/riscv/kvm/mmu.c: line 126
static void gstage_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr)
{
	unsigned long order = PAGE_SHIFT;

	if (gstage_level_to_page_order(level, &order))
		return;
	addr &= ~(BIT(order) - 1);

	kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0, addr, BIT(order), order);
}

// arch/riscv/kvm/mmu.c: line 748
void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu)
{
	// 设置当前地址系统（SV32，SV39，etc.）对应的 hgatp 初始值
	unsigned long hgatp = gstage_mode;
	// 获取指向当前 vcpu 的架构信息的指针
	struct kvm_arch *k = &vcpu->kvm->arch;

	// 设置 hgatp.VMID 位
	hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) &
		 HGATP_VMID_MASK;
	// 设置 hgatp.PPN 位
	hgatp |= (k->pgd_phys >> PAGE_SHIFT) & HGATP_PPN;

	// 更新 hgatp 的值
	csr_write(CSR_HGATP, hgatp);

	// 若当前 vmid 无效，使用 hfence.gvma 指令刷新全部 TLB 的项
	if (!kvm_riscv_gstage_vmid_bits())
		kvm_riscv_local_hfence_gvma_all();
}

#ifdef CONFIG_64BIT
static unsigned long gstage_mode = (HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT);
static unsigned long gstage_pgd_levels = 3;
#define gstage_index_bits	9
#else
static unsigned long gstage_mode = (HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT);
static unsigned long gstage_pgd_levels = 2;
#define gstage_index_bits	10
#endif

// arch/riscv/kvm/vcpu.c: line 915
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
{
	int ret;

	// ...

	vcpu_load(vcpu);

	kvm_sigset_activate(vcpu);

	ret = 1;
	run->exit_reason = KVM_EXIT_UNKNOWN;
	/* 处理 vCPU 内部请求的循环 */
	while (ret > 0) {
		// ...

		/* 更新 VMID（内部将根据 VM 做出更新 hgatp 等请求）*/
		kvm_riscv_gstage_vmid_update(vcpu);
		/* 处理各个 vCPU 内部的请求 */
		kvm_riscv_check_vcpu_requests(vcpu);

		// ...

		ret = kvm_riscv_vcpu_exit(vcpu, run, &trap);
	}

	kvm_sigset_deactivate(vcpu);

	vcpu_put(vcpu);

	kvm_vcpu_srcu_read_unlock(vcpu);

	return ret;
}

// arch/riscv/kvm/vmid.c: line 71
void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu)
{
	unsigned long i;
	struct kvm_vcpu *v;
	struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid;

	/* 视情况更新 vmid 版本并刷新 TLB */
	if (!kvm_riscv_gstage_vmid_ver_changed(vmid))
		return;
	spin_lock(&vmid_lock);
	// ...
	spin_unlock(&vmid_lock);

	/* 为每一个 vCPU 更新页表的刷新请求 */
	/* Request G-stage page table update for all VCPUs */
	kvm_for_each_vcpu(i, v, vcpu->kvm)
		kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
}

// arch/riscv/kvm/vcpu.c: line 848
static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
{
	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);

	if (kvm_request_pending(vcpu)) {
		/* sleep, reset requests handling */
		// ...

		if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
			kvm_riscv_gstage_update_hgatp(vcpu);

		/* Memory management requests (fence.i, hfence.gvma, hfence.vvma, etc.) handling */
		// ...
	}
}

// arch/riscv/kvm/tlb.c: line 388
void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
				    unsigned long hbase, unsigned long hmask,
				    gpa_t gpa, gpa_t gpsz,
				    unsigned long order)
{
	struct kvm_riscv_hfence data;

	data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA;
	data.asid = 0;
	data.addr = gpa;
	data.size = gpsz;
	data.order = order;
	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
			    KVM_REQ_HFENCE_GVMA_VMID_ALL, &data);
}

// arch/riscv/kvm/tlb.c: line 345
static void make_xfence_request(struct kvm *kvm,
				unsigned long hbase, unsigned long hmask,
				unsigned int req, unsigned int fallback_req,
				const struct kvm_riscv_hfence *data)
{
	// ...

	/* 将每个 vCPU 的 hfence 信息入队 */
	kvm_for_each_vcpu(i, vcpu, kvm) {
		// ...
		if (!vcpu_hfence_enqueue(vcpu, data))
			actual_req = fallback_req;
	}

	/* 若队满无法全部入队，则直接提交名为 fallback_req 的请求，对于指定了 VMID 和 GPA 的 hfence.gvma 指令而言，其 fallback_req 为刷新对应 VMID 的全部 TLB 项的 `KVM_REQ_HFENCE_GVMA_VMID_ALL`，以此保证即便不能做到精细化的内存管理，也可以通过粗粒度的指令达成所需的效果。*/
	kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask);
}

// riscv/kvm.c: line 64
void kvm__arch_init(struct kvm *kvm)
{
	/* 申请 Guest 内存。Buffer 做 64K 对齐，如使用了 THP（Transparent Huge Page）则按 2M 对齐 */

	/* 确定 Guest 内存的起始位置与大小 */
	kvm->ram_size = min(kvm->cfg.ram_size, (u64)RISCV_MAX_MEMORY(kvm));
	kvm->arch.ram_alloc_size = kvm->ram_size + SZ_2M;
	kvm->arch.ram_alloc_start = mmap_anon_or_hugetlbfs(kvm,
						kvm->cfg.hugetlbfs_path,
						kvm->arch.ram_alloc_size);
	if (kvm->arch.ram_alloc_start == MAP_FAILED)
		die("Failed to map %lld bytes for guest memory (%d)",
		    kvm->arch.ram_alloc_size, errno);
	kvm->ram_start = (void *)ALIGN((unsigned long)kvm->arch.ram_alloc_start,
					SZ_2M);

	/* 为 Guest 申请特定类型的内存 */
	madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
		MADV_MERGEABLE);
	madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
		MADV_HUGEPAGE);
}