RISC-V 缺页异常处理程序分析（1）：do_page_fault() 和 handle_mm_fault()

asmlinkage void do_page_fault(struct pt_regs *regs)
{

	unsigned long addr, cause;

	...
	/*
	 * 异常处理开始前，会将寄存器值压入内核栈，regs 即为栈内存有寄存器值的结构体
	 * cause 寄存器存有页异常原因有关信息，badaddr 寄存器存有异常对应的虚拟地址
	 */
	cause = regs->cause;
	addr = regs->badaddr;
	/* 获取获取当前进程的内存上下文 mm */
	tsk = current;
	mm = tsk->mm;

	/*
	 * Fault-in kernel-space virtual memory on-demand.
	 * The 'reference' page table is init_mm.pgd.
	 *
	 * NOTE! We MUST NOT take any locks for this case. We may
	 * be in an interrupt or a critical region, and should
	 * only copy the information from the master page table,
	 * nothing more.
	 */
        /* 在 vmalloc 区域的错误，由 vmalloc_fault() 处理 */
	if (unlikely((addr >= VMALLOC_START) && (addr < VMALLOC_END))) {
		vmalloc_fault(regs, code, addr);
		return;
	}

	...

	/*
	 * If we're in an interrupt, have no user context, or are running
	 * in an atomic region, then we must not take the fault.
	 */
        /* 内核线程、中断上下文、原子上下文或缺页处理句柄关闭，交由 no_context() 处理 */
	if (unlikely(faulthandler_disabled() || !mm)) {
		tsk->thread.bad_cause = cause;
		no_context(regs, addr);
		return;
	}

        /*
         * 同时满足缺页发生在内核态，地址在进程地址范围内，且缺页不是内核访问用户态页表里存在的地址（status 中 SUM 位未置位，在处理器处于
         * 内核态时访问用户空间地址会由处理器对该位置位），那就是一个 bug
	 */
	if (!user_mode(regs) && addr < TASK_SIZE &&
			unlikely(!(regs->status & SR_SUM)))
		die_kernel_fault("access to user memory without uaccess routines",
				addr, regs);

retry:
	mmap_read_lock(mm);
	/* 找到第一个末尾大于等于 addr 的虚拟地址空间 */
	vma = find_vma(mm, addr);
	/* 没有找到，说明该地址一定没有被包含在任何地址空间内（注：栈向下生长，也不可能由于栈没有分配），为用户态编程错误，由 bad_area() 处理 */
	if (unlikely(!vma)) {
		tsk->thread.bad_cause = cause;
		bad_area(regs, mm, code, addr);
		return;
	}
	/* 地址被该虚拟地址空间覆盖，跳到 good_area 处理，此标号表明可以抢救一下 */
	if (likely(vma->vm_start <= addr))
		goto good_area;
	/* 如果不是向下增长的虚拟地址空间（栈对应的），走到这说明错误地址不在进程地址空间，也去 bad_area() 里处理 */
	if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
		tsk->thread.bad_cause = cause;
		bad_area(regs, mm, code, addr);
		return;
	}
	/*
         * 如果是栈区，有可能是压栈操作栈不够导致（给栈分配虚拟地址空间机制是用完了继续扩），进一步判断后扩大一下栈对应的虚拟地址空间，
         * 如果判断地址离得的太远等则不是压栈操作，也去 bad_area() 处理
         */
	if (unlikely(expand_stack(vma, addr))) {
		tsk->thread.bad_cause = cause;
		bad_area(regs, mm, code, addr);
		return;
	}

	/*
	 * Ok, we have a good vm_area for this memory access, so
	 * we can handle it.
	 */
	/* 到这里说明缺页的地址是属于该进程 */
good_area:
	code = SEGV_ACCERR;
	/* access_error() 为真说明权限和访问操作不对应，访问不合法，也到 bad_area() 处理 */
	if (unlikely(access_error(cause, vma))) {
		tsk->thread.bad_cause = cause;
		bad_area(regs, mm, code, addr);
		return;
	}

	/*
	 * If for any reason at all we could not handle the fault,
	 * make sure we exit gracefully rather than endlessly redo
	 * the fault.
	 */
        /* 走到这里说明页异常发生在处理器处于用户态时且地址处于合法地址空间，将交由 handle_mm_fault() 进一步处理 */
	fault = handle_mm_fault(vma, addr, flags, regs);

	...
}

// mm/memory.c:4619
static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
		unsigned long address, unsigned int flags)
{
	struct vm_fault vmf = {
		.vma = vma,
		.address = address & PAGE_MASK,
		.flags = flags,
		.pgoff = linear_page_index(vma, address),
		.gfp_mask = __get_fault_gfp_mask(vma),
	};
	/* 获取虚拟地址空间对应的内存描述符 */
	struct mm_struct *mm = vma->vm_mm;
	pgd_t *pgd;
	p4d_t *p4d;
	vm_fault_t ret;
        /* 通过内存描述符和虚拟地址，得到该虚拟地址在页全局目录（pgd）中对应的目录项指针 */
	pgd = pgd_offset(mm, address);
        /* 通过内存描述符、pgd 项和虚拟地址，得到 p4d 项指针 */
	p4d = p4d_alloc(mm, pgd, address);
	if (!p4d)
		return VM_FAULT_OOM;

        ...

        /* 通过内存描述符、p4d 项和虚拟地址，得到 pud 项指针 */
	vmf.pud = pud_alloc(mm, p4d, address);
	if (!vmf.pud)
		return VM_FAULT_OOM;

        ...

        /* 通过内存描述符、pud 项和虚拟地址，得到 pmd 项指针 */
	vmf.pmd = pmd_alloc(mm, vmf.pud, address);
	if (!vmf.pmd)
		return VM_FAULT_OOM;

        ...

        /* 将 vmf 的地址传给 handle_pte_fault() 进行页表项的进一步处理 */
	return handle_pte_fault(&vmf);
}

// include/linux/mm.h :2233
static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
		unsigned long address)
{
	return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
		NULL : p4d_offset(pgd, address);
}

static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
		unsigned long address)
{
	return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ?
		NULL : pud_offset(p4d, address);
}
// mm/memory.c:4675
static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
	return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
		NULL: pmd_offset(pud, address);
}

 // mm/memory.c:4874
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
	spinlock_t *ptl;
	pmd_t *new = pmd_alloc_one(mm, address);
	if (!new)
		return -ENOMEM;

	ptl = pud_lock(mm, pud);
	if (!pud_present(*pud)) {
		mm_inc_nr_pmds(mm);
		smp_wmb(); /* See comment in pmd_install() */
		pud_populate(mm, pud, new);
	} else {	/* Another has populated it */
		pmd_free(mm, new);
	}
	spin_unlock(ptl);
	return 0;
}

// include/asm-generic: 119
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	struct page *page;
	gfp_t gfp = GFP_PGTABLE_USER;

	if (mm == &init_mm)
		gfp = GFP_PGTABLE_KERNEL;
	page = alloc_pages(gfp, 0);
	if (!page)
		return NULL;
	if (!pgtable_pmd_page_ctor(page)) {
		__free_pages(page, 0);
		return NULL;
	}
	return (pmd_t *)page_address(page);
}

// arch/riscv/include/asm/pgalloc.h:35
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
	unsigned long pfn = virt_to_pfn(pmd);

	set_pud(pud, __pud((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
}