深入理解Linux内核之HVO(HugeTLB Vmemmap Optimization)

//mm/hugetlb_vmemmap.c
hugetlb_vmemmap_optimize  
->vmemmap_start = (unsigned long)head //hugetlb页面的head vmemmap page所在地址。
->vmemmap_should_optimize //判断当前的hugetlb页面大小是否适合做HVO优化，
没有打开vmemmap_optimize_enabled或者hugetlb页面使用struct page结构占用的
内存小于一个4k页面不做优化。
->  vmemmap_end     = vmemmap_start + hugetlb_vmemmap_size(h);//获得优化的tail vmemmap page地址
 vmemmap_reuse   = vmemmap_start;//重复映射使用head vmemmap page地址
 vmemmap_start   += HUGETLB_VMEMMAP_RESERVE_SIZE;//从第一个tail vmemmap page开始优化

->vmemmap_remap_free(vmemmap_start, vmemmap_end, vmemmap_reuse) //释放掉hugetlb页面使用struct page结构冗余的物理页面。
 ->walk.reuse_page = alloc_pages_node(nid, gfp_mask, 0);
        if (walk.reuse_page) {
                copy_page(page_to_virt(walk.reuse_page),
                        ¦ (void *)walk.reuse_addr);
                list_add(&walk.reuse_page->lru, &vmemmap_pages);
        }
       //优化1：申请一个新的4k页面，即new head vmemmap page，然后将head vmemmap page拷贝到这个页面，然后将new head vmemmap page加入vmemmap_pages链表
    （用于失败释放此页面使用）

 ->mmap_read_lock(&init_mm) //读方式获得init_mm的mmap_lock
 ->vmemmap_remap_range //遍历页表，释放冗余的物理页面（由于优化1，这里会释放掉所有的管理hugetlb页面使用的struct page结构占用的内存）。
 ->mmap_read_unlock(&init_mm) //读方式释放init_mm的mmap_lock
 ->free_vmemmap_page_list(&vmemmap_pages)//释放调用hugetlb页面使用struct page结构冗余的物理页面（例如2M大小的hugetlb页面，释放掉8页（page0-page7））
->SetHPageVmemmapOptimized(head) //为hugetlb页面设置HVO优化标记，定义在include/linux/hugetlb.h （HPAGEFLAG(VmemmapOptimized, vmemmap_optimized)）

vmemmap_remap_pte的核心代码如下：
 static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
                        ¦     struct vmemmap_remap_walk *walk)
{
        /*
        ¦* Remap the tail pages as read-only to catch illegal write operation
        ¦* to the tail pages.
        ¦*/
        pgprot_t pgprot = PAGE_KERNEL_RO; //映射tail vmemmap page为只读
        struct page *page = pte_page(ptep_get(pte)); //通过页表项获得struct page指针
        pte_t entry;

        /* Remapping the head page requires r/w */
        if (unlikely(addr == walk->reuse_addr)) { //如果当前的虚拟地址是reuse_addr
                pgprot = PAGE_KERNEL; ////映射head vmemmap page为可读可写
                list_del(&walk->reuse_page->lru);//之前在hugetlb_vmemmap_optimize中将这个new head vmemmap page加入了vmemmap_pages，现在删除，供页面共享使用。

                /*
                ¦* Makes sure that preceding stores to the page contents from
                ¦* vmemmap_remap_free() become visible before the set_pte_at()
                ¦* write. 
                ¦*/
                smp_wmb();
        }

        entry = mk_pte(walk->reuse_page, pgprot);//重要步骤：页表映射尾页到头页上！
        list_add_tail(&page->lru, walk->vmemmap_pages);//将需要释放的页面加入vmemmap_pages链表
        set_pte_at(&init_mm, addr, pte, entry);//设置页表项
}

mm/hugetlb.c
__setup("hugepages=", hugepages_setup) 
-> hugepages_setup
 -> hugetlb_hstate_alloc_pages
  -> alloc_pool_huge_page
   -> alloc_fresh_hugetlb_folio //分配2M的hugetlb页面
    -> prep_new_hugetlb_folio
     -> __prep_new_hugetlb_folio
      -> hugetlb_vmemmap_optimize //触发HVO优化

mm/hugetlb.c
nr_hugepages_store
->nr_hugepages_store_common
 ->set_max_huge_pages
  ->alloc_pool_huge_page
   -> alloc_fresh_hugetlb_folio //分配2M的hugetlb页面
      -> prep_new_hugetlb_folio
       -> __prep_new_hugetlb_folio
        -> hugetlb_vmemmap_optimize //触发HVO优化

mm/hugetlb.c
hugetlb_table[] 
->hugetlb_sysctl_handler
 ->hugetlb_sysctl_handler_common
  ->__nr_hugepages_store_common
   -> set_max_huge_pages
    ->alloc_pool_huge_page
     -> alloc_fresh_hugetlb_folio //分配2M的hugetlb页面
        -> prep_new_hugetlb_folio
         -> __prep_new_hugetlb_folio
          -> hugetlb_vmemmap_optimize //触发HVO优化

mm/hugetlb_vmemmap.c
hugetlb_vmemmap_restore
->首先通过HPageVmemmapOptimized(head)判断是否hugetlb页面被HVO优化了，没有则直接返回
->  vmemmap_end     = vmemmap_start + hugetlb_vmemmap_size(h);//获得优化的tail vmemmap page地址
 vmemmap_reuse   = vmemmap_start;//重复映射使用的head vmemmap page地址
 vmemmap_start   += HUGETLB_VMEMMAP_RESERVE_SIZE;//从第一个tail vmemmap page开始优化
->vmemmap_remap_alloc(vmemmap_start, vmemmap_end, vmemmap_reuse) //还原HVO之前所作的优化：重新映射vmemmap的虚拟地址范围到vmemmap_pages页面
 ->alloc_vmemmap_page_list(start, end, &vmemmap_pages) //分配所有的tail vmemmap page，如2M大小的hugetlb页面， 分配page1 - page7,共7个页面，page0 已有不需要分配
 ->mmap_read_lock(&init_mm) //读方式获得init_mm的mmap_lock
 ->vmemmap_remap_range(reuse, end, &walk) //遍历页表，重新映射vmemmap的虚拟地址范围到vmemmap_pages中分配的页面
  ->vmemmap_restore_pte //对于每个页表项，调用vmemmap_restore_pte处理
 ->mmap_read_unlock(&init_mm)//读方式释放init_mm的mmap_lock

->ClearHPageVmemmapOptimized(head) //清除hugetlb页面的优化标记

vmemmap_remap_pte的核心代码如下：
/*
 * How many struct page structs need to be reset. When we reuse the head
 * struct page, the special metadata (e.g. page->flags or page->mapping)
 * cannot copy to the tail struct page structs. The invalid value will be
 * checked in the free_tail_page_prepare(). In order to avoid the message
 * of "corrupted mapping in tail page". We need to reset at least 3 (one
 * head struct page struct and two tail struct page structs) struct page
 * structs.
 */
#define NR_RESET_STRUCT_PAGE            3

static inline void reset_struct_pages(struct page *start)
{
        struct page *from = start + NR_RESET_STRUCT_PAGE;

        BUILD_BUG_ON(NR_RESET_STRUCT_PAGE * 2 > PAGE_SIZE / sizeof(struct page));
        memcpy(start, from, sizeof(*from) * NR_RESET_STRUCT_PAGE);
}


static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
                                struct vmemmap_remap_walk *walk)
{
        pgprot_t pgprot = PAGE_KERNEL;//页表属性可读可写
        struct page *page;
        void *to;

        BUG_ON(pte_page(ptep_get(pte)) != walk->reuse_page);
        
        page = list_first_entry(walk->vmemmap_pages, struct page, lru);//vmemmap_pages链表中获得一个物理页面
        list_del(&page->lru);//page从vmemmap_pages链表中删除
        to = page_to_virt(page);
        copy_page(to, (void *)walk->reuse_addr);//将head vmemmap page的页面内容拷贝到这个物理页面
        reset_struct_pages(to); //由于描述hugetlb页面的struct page结构，只有前3个struct page结构用于描述hugetlb页面信息，
  其他的struct page结构都只是有compound_head是有意义的，为了防止free_tail_page_prepare有错误的检查报告，这里将所有tail vmemmap page的
  内容都设置为正常值。

        /*
        ¦* Makes sure that preceding stores to the page contents become visible
        ¦* before the set_pte_at() write.
        ¦*/
        smp_wmb();
        set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot));//重新映射页表到这个物理页面
}

mm/hugetlb.c
nr_hugepages_store
nr_hugepages_store_common
set_max_huge_pages
->flush_free_hpage_work(h);
 ->free_hpage_workfn
  ->__update_and_free_hugetlb_folio
   ->hugetlb_vmemmap_restore
->update_and_free_pages_bulk
 ->__update_and_free_hugetlb_folio
  ->hugetlb_vmemmap_restore

mm/hugetlb.c
hugetlb_table[] 
->hugetlb_sysctl_handler
->hugetlb_sysctl_handler_common
->__nr_hugepages_store_common
-> set_max_huge_pages
 ->flush_free_hpage_work(h);
  ->free_hpage_workfn
   ->__update_and_free_hugetlb_folio
    ->hugetlb_vmemmap_restore
 ->update_and_free_pages_bulk
  ->__update_and_free_hugetlb_folio
   ->hugetlb_vmemmap_restore

mm/hugetlb.c
demote_store
->demote_pool_huge_page
 ->demote_free_hugetlb_folio
  ->hugetlb_vmemmap_restore

include/linux/page-flags.h
page_folio
->_compound_head
 ->static inline unsigned long _compound_head(const struct page *page)
 {
   unsigned long head = READ_ONCE(page->compound_head);//获取页面的compound_head成员

   if (unlikely(head & 1))//是真正的尾页
     return head - 1;//计算出头页地址，返回
   return (unsigned long)page_fixed_fake_head(page);//为真正的头页或者为伪造的头页。
 }
    
   page_fixed_fake_head处理如下：
   /*
 * Return the real head page struct iff the @page is a fake head page, otherwise
 * return the @page itself. See Documentation/mm/vmemmap_dedup.rst.
 */
static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
{
        if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
                return page;

        /*   
        ¦* Only addresses aligned with PAGE_SIZE of struct page may be fake head
        ¦* struct page. The alignment check aims to avoid access the fields (
        ¦* e.g. compound_head) of the @page[1]. It can avoid touch a (possibly)
        ¦* cold cacheline in some cases.
        ¦*/
        if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
        ¦   test_bit(PG_head, &page->flags)) { //struct page结构地址只有对齐PAGE_SIZE才有可能为伪造的头页
                /*   
                ¦* We can safely access the field of the @page[1] with PG_head
                ¦* because the @page is a compound page composed with at least
                ¦* two contiguous pages.
                ¦*/
                unsigned long head = READ_ONCE(page[1].compound_head);//获得下一个 struct page地址的compound_head成员，实际上就是获取第一个尾页的compound_head

                if (likely(head & 1))
                        return (const struct page *)(head - 1);//计算获得真正的头页地址
        }
        return page;
}
  

Documentation/mm/vmemmap_dedup.rst
mm/hugetlb_vmemmap.c
mm/hugetlb_vmemmap.h