Page Fault handle on userspace - Userfaultfd

Author : 堇姬 Naup

What is userfaultfd

userfaultfd 是一種 syscall (syscall table rax = 388)
user 可以去自訂 user pagefault handler 來去處理 userspace 的缺頁異常 (https://zh.wikipedia.org/wiki/%E9%A1%B5%E7%BC%BA%E5%A4%B1)

https://chromium.googlesource.com/chromiumos/docs/+/master/constants/syscalls.md

首先需要先去註冊一個 userfaultfd
通過 ioctl 來去監視一塊記憶體
之後去創建一個 uffd monitor 的 thread 來發出 poll() 輪巡，直到出現 page fault

這張圖解釋了如何去處理 userfault
首先是當一個 thread 發生 page fault (faulting thread) ，他會去 kernel space 處理
memory management 會去調用 handle_userfault() 丟去 userfaultfd 來去處理
首先 userfaultfd 先告知 faulting thread 阻塞，之後發送 uffd_msg 到 uffd monitor 並等待處理
隨後 monitor 會處理 page fault 最後處理完發送訊號告知 faulting page 繼續執行

用法

用法可以參考 man page
https://man7.org/linux/man-pages/man2/userfaultfd.2.html

先初始化

#define _GNU_SOURCE
#include <stdio.h>              // printf, perror
#include <stdlib.h>             // malloc, exit
#include <string.h>             // memset
#include <unistd.h>             // read, syscall
#include <sys/syscall.h>        // __NR_userfaultfd
#include <linux/userfaultfd.h>  // struct uffdio_*, UFFD_API 等
#include <sys/mman.h>           // mmap, PROT_*, MAP_*
#include <sys/ioctl.h>          // ioctl
#include <pthread.h>            // pthread_create 等
#include <poll.h>               // poll (可選，用來等待 uffd)
#include <errno.h>              // errno, EAGAIN 等
#include <fcntl.h>              // O_CLOEXEC, O_NONBLOCK

...
           int        s;
           char       c;
           char       *addr;   /* Start of region handled by userfaultfd */
           long       uffd;    /* userfaultfd file descriptor */
           size_t     size, i;  /* Size of region handled by userfaultfd */
           pthread_t  thr;     /* ID of thread that handles page faults */
           struct uffdio_api       uffdio_api;
           struct uffdio_register  uffdio_register;

首先要先註冊 userfaultfd，可以通過 syscall 來註冊，並存到 uffd
第一個參數是 userfaultfd 的 syscall number，第二個是 flags

/* Create and enable userfaultfd object. */

uffd = syscall(SYS_userfaultfd, O_CLOEXEC | O_NONBLOCK);
if (uffd == -1)
    err(EXIT_FAILURE, "userfaultfd");

這邊設置 userfaultfd 的 api
在使用 userfaultfd() 建立 userfaultfd 物件之後，應用程式必須透過 UFFDIO_API 的 ioctl 操作來啟用它。這個操作允許核心與使用者空間之間進行兩階段的握手，以決定核心所支援的 API 版本與功能

/* NOTE: Two-step feature handshake is not needed here, since this
   example doesn't require any specific features.

   Programs that *do* should call UFFDIO_API twice: once with
   `features = 0` to detect features supported by this kernel, and
   again with the subset of features the program actually wants to
   enable. */
uffdio_api.api = UFFD_API;
uffdio_api.features = 0;
if (ioctl(uffd, UFFDIO_API, &uffdio_api) == -1)
    err(EXIT_FAILURE, "ioctl-UFFDIO_API");

接下來創建一個匿名映射區段，並存下他的 address
之後去註冊這個位置，以及要監控的長度
之後通過 ioctl 給 userfaultfd 註冊

UFFDIO_REGISTER_MODE_MISSING（自 Linux 4.10 起提供）
當以 UFFDIO_REGISTER_MODE_MISSING 模式註冊記憶體區段時，
當存取尚未存在的頁面（missing page）時，用戶空間會接收到 page fault 通知。
此時，發生錯誤的執行緒會被暫停，直到用戶空間透過 UFFDIO_COPY 或 UFFDIO_ZEROPAGE 的 ioctl 指令來處理這個 page fault 為止

/* Create a private anonymous mapping. The memory will be
     demand-zero paged--that is, not yet allocated. When we
     actually touch the memory, it will be allocated via
     the userfaultfd. */

  addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
              MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  if (addr == MAP_FAILED)
      err(EXIT_FAILURE, "mmap");

  printf("Address returned by mmap() = %p\n", addr);

  /* Register the memory range of the mapping we just created for
     handling by the userfaultfd object. In mode, we request to track
     missing pages (i.e., pages that have not yet been faulted in). */

  uffdio_register.range.start = (unsigned long) addr;
  uffdio_register.range.len = size;
  uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
  if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
      err(EXIT_FAILURE, "ioctl-UFFDIO_REGISTER");

之後去創建一個 userfaultfd 處理的 process

/* Create a thread that will process the userfaultfd events. */

   s = pthread_create(&thr, NULL, fault_handler_thread, (void *) uffd);
   if (s != 0) {
       errc(EXIT_FAILURE, s, "pthread_create");
   }

poll() 是一個系統呼叫（system call）或函式，常用於事件驅動的 I/O 多工（I/O multiplexing），主要用來監控多個檔案描述符（file descriptors）

創建一個迴圈，用 poll 循環去監視 userfaultfd 狀況

struct pollfd {
    int   fd;         /* file descriptor */
    short events;     /* requested events */
    short revents;    /* returned events */
};

https://man7.org/linux/man-pages/man2/poll.2.html
man 中的實作在我的 VM 上跑不了，所以改成這樣 (準確實作參考最下面的 demo)
總之最後去讀 uffd 有沒有發生事件，有的話去處理
最後返回 UFFDIO_COPY

void * uffdio_handle_thread(void * user_fault_fd){
    static char * page = NULL;
    static struct uffd_msg msg;
    struct uffdio_copy uffdio_copy;
    long uffd = (long)user_fault_fd;
    if (page == NULL) 
    {
        page = mmap(NULL, page_size, PROT_READ | PROT_WRITE,
                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
        printf("thread page addr:%p\n",page);
        if (page == MAP_FAILED){
            printf("error in mmap!\n");
            exit(0);
        }
    }
    memset(page,0x41,page_size);
    printf("thread test\n");

    for(;;){
        struct pollfd pollfd;
        pollfd.fd = uffd;
        pollfd.events=POLLIN;
        poll(&pollfd,1,-1);
        read(uffd,&msg,sizeof(msg));
        if(msg.event != UFFD_EVENT_PAGEFAULT){
            continue;
        }
        uffdio_copy.src = (unsigned long)page;
        uffdio_copy.dst = (unsigned long)msg.arg.pagefault.address & ~(page_size-1);
        uffdio_copy.len = page_size;
        uffdio_copy.mode = 0;
        uffdio_copy.copy = 0;
        ioctl(uffd,UFFDIO_COPY,&uffdio_copy);
    }
}
    int s = pthread_create(&uffdio_handler,NULL,uffdio_handle_thread,(void *)user_fault_fd);

完整 demo

#include <sys/types.h>
#include <stdio.h>
#include <linux/userfaultfd.h>
#include <pthread.h>
#include <errno.h>
#include <unistd.h>
#include <stdlib.h>
#include <fcntl.h>
#include <signal.h>
#include <poll.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <sys/ioctl.h>
#include <poll.h>

static int page_size;

void * uffdio_handle_thread(void * user_fault_fd){
    static char * page = NULL;
    static struct uffd_msg msg;
    struct uffdio_copy uffdio_copy;
    long uffd = (long)user_fault_fd;
    if (page == NULL) 
    {
        page = mmap(NULL, page_size, PROT_READ | PROT_WRITE,
                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
        printf("thread page addr:%p\n",page);
        if (page == MAP_FAILED){
            printf("error in mmap!\n");
            exit(0);
        }
    }
    memset(page,0x41,page_size);
    printf("thread test\n");

    for(;;){
        struct pollfd pollfd;
        pollfd.fd = uffd;
        pollfd.events=POLLIN;
        poll(&pollfd,1,-1);
        read(uffd,&msg,sizeof(msg));
        if(msg.event != UFFD_EVENT_PAGEFAULT){
            continue;
        }
        uffdio_copy.src = (unsigned long)page;
        uffdio_copy.dst = (unsigned long)msg.arg.pagefault.address & ~(page_size-1);
        uffdio_copy.len = page_size;
        uffdio_copy.mode = 0;
        uffdio_copy.copy = 0;
        ioctl(uffd,UFFDIO_COPY,&uffdio_copy);
    }
}

int main(){
    
    long user_fault_fd = syscall(__NR_userfaultfd,O_CLOEXEC|O_NONBLOCK);
    if(user_fault_fd<0){
        printf("create user_fault_fd error!\n");
        return;
    }
    page_size = sysconf(_SC_PAGE_SIZE);
    char* addr = mmap(NULL,page_size,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0);
    if(addr==MAP_FAILED){
        printf("error in mmap!\n");
        return ;
    }
    struct uffdio_register uffdio_register;
    uffdio_register.range.start = (unsigned long) addr;
    uffdio_register.range.len = page_size;
    uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
    struct uffdio_api uffdio_api;
    uffdio_api.api = UFFD_API;
    uffdio_api.features = 0;
    ioctl(user_fault_fd, UFFDIO_API, &uffdio_api);
    ioctl(user_fault_fd, UFFDIO_REGISTER, &uffdio_register);
    printf("test main\n");
    pthread_t uffdio_handler;
    int s = pthread_create(&uffdio_handler,NULL,uffdio_handle_thread,(void *)user_fault_fd);
    if(s!=0){
        printf("error create thread\n");
        return ;
    }
    printf("main page addr:%p\n",addr);
    void * ptr = (void*) *(unsigned long long*) addr;
    printf("data:%p\n",ptr);

}

總之最後就成功了，因為你 mmap 建立映射後，當你第一次去使用，因為還沒有去 mapping 到 physical address
所以理論上他會觸發 page fault 去 kernel 分配
但是你在 userspace 實作了 page fault
所以說就會被 poll 捕捉到事件，進而在 userspace 處理 page fault

這邊將上面的 Demo mmap 方式改成

char* addr = mmap(NULL,page_size,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS| MAP_POPULATE,-1,0);
if(addr==MAP_FAILED){
    printf("error in mmap!\n");
    return ;
}

他就會在 mmap 時候預先分配 physical page
這樣去讀取時候就不會觸發 page fault 了