Intel FPGA 100G VF(IFCVF) DPDK用户态VDPA设备probe探测流程

intel, vdpa, ifcvf_vdpa.c -> net/ifcvf：添加 ifcvf vDPA 驱动程序，IFCVF vDPA（vhost 数据路径加速）驱动程序为 Intel FPGA 100G VF（IFCVF）提供支持。IFCVF 的数据路径与 virtio 环兼容，它作为 HW vhost 后端工作，可以通过 DMA 直接向/从 virtio 发送/接收数据包。不同的 VF 设备服务于位于不同 VM 中的不同 virtio 前端，因此每个 VF 都需要有自己的 DMA 地址转换服务。在驱动程序探测期间，将创建一个新容器，使用此容器 vDPA 驱动程序可以使用 VM 的内存区域信息对 DMA 重映射表进行编程。实现的关键 vDPA 驱动程序操作：- ifcvf_dev_config：使用 vhost lib 提供的 virtio 信息启用 VF 数据路径，包括 IOMMU 编程以启用 VF DMA 到 VM 的内存，VFIO 中断设置以将 HW 中断路由到 virtio 驱动程序，创建通知中继线程以将 virtio 驱动程序的踢动转换为 MMIO 写入 HW，HW 队列配置。- ifcvf_dev_close：撤销 ifcvf_dev_config 中的所有设置。实时迁移功能由 IFCVF 支持，此驱动程序启用此功能。对于脏页日志记录，VF 有助于记录数据包缓冲区写入，驱动程序有助于在设备停止时将使用的环设为脏的。由于 vDPA 驱动程序需要设置 MSI-X 向量来中断客户机，因此目前仅支持 vfio-pci
RTE_PMD_REGISTER_PCI(net_ifcvf, rte_ifcvf_vdpa)
static struct rte_pci_driver rte_ifcvf_vdpa = {
    .id_table = pci_id_ifcvf_map,
    .drv_flags = 0,
    .probe = ifcvf_pci_probe,
    .remove = ifcvf_pci_remove,
};
// probe扫描硬件
ifcvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_device *pci_dev)
    kvlist = rte_kvargs_parse(pci_dev->device.devargs->args, ifcvf_valid_arguments)
    rte_kvargs_count(kvlist, IFCVF_VDPA_MODE)
    rte_kvargs_process(kvlist, IFCVF_VDPA_MODE, &open_int, &vdpa_mode)
    list = rte_zmalloc("ifcvf", sizeof(*list), 0)
    internal = rte_zmalloc("ifcvf", sizeof(*internal), 0)
    internal->pdev = pci_dev
    ifcvf_vfio_setup(internal)
        rte_vfio_get_group_num(rte_pci_get_sysfs_path(), devname, &iommu_group_num)
        internal->vfio_container_fd = rte_vfio_container_create()
        internal->vfio_group_fd = rte_vfio_container_group_bind(internal->vfio_container_fd, iommu_group_num)
        rte_pci_map_device(dev) -> map hw pcie bar to userspace
            pci_vfio_map_resource
                pci_vfio_map_resource_primary
                    pci_rte_vfio_setup_device
                        pci_vfio_setup_interrupts
                    vfio_enable_msi
        internal->hw.mem_resource[i].addr = internal->pdev->mem_resource[i].addr;
        internal->hw.mem_resource[i].phys_addr = internal->pdev->mem_resource[i].phys_addr;
        internal->hw.mem_resource[i].len = internal->pdev->mem_resource[i].len;
    ifcvf_init_hw(&internal->hw, internal->pdev)
        ret = PCI_READ_CONFIG_BYTE(dev, &pos, PCI_CAPABILITY_LIST)
        PCI_READ_CONFIG_RANGE(dev, (u32 *)&cap, sizeof(cap), pos)
            rte_pci_read_config(dev, val, size, where)
                case RTE_PCI_KDRV_IGB_UIO
                case RTE_PCI_KDRV_UIO_GENERIC
                    pci_uio_read_config(intr_handle, buf, len, offset)
                case RTE_PCI_KDRV_VFIO
                    pci_vfio_read_config(intr_handle, buf, len, offset)
                        int vfio_dev_fd = rte_intr_dev_fd_get(intr_handle)
                        pread64(vfio_dev_fd, buf, len, VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs)
                default
                    rte_pci_device_name(&device->addr, devname, RTE_DEV_NAME_MAX_LEN)
        case IFCVF_PCI_CAP_COMMON_CFG
            hw->common_cfg = get_cap_addr(hw, &cap)
                hw->mem_resource[bar].addr + offset
        case IFCVF_PCI_CAP_NOTIFY_CFG
            hw->notify_base = get_cap_addr(hw, &cap)
        case IFCVF_PCI_CAP_ISR_CFG
            hw->isr = get_cap_addr(hw, &cap) -> 产生int#x中断, 如果设备不支持msi-x capability的话，在设备配置变化或者需要进行队列kick通知时，就需要用到isr capability
        case IFCVF_PCI_CAP_DEVICE_CFG
            hw->dev_cfg = get_cap_addr(hw, &cap)
        hw->lm_cfg = hw->mem_resource[4].addr -> live migration
    features = ifcvf_get_features(&internal->hw)
        features_hi = IFCVF_READ_REG32(&cfg->device_feature) -> read features low and high
    device_id = ifcvf_pci_get_device_type(pci_dev)
    if (device_id == VIRTIO_ID_NET)
        internal->max_queues = MIN(IFCVF_MAX_QUEUES, queue_pairs)
    else if (device_id == VIRTIO_ID_BLOCK) -> when set device_id?
        internal->hw.device_type = IFCVF_BLK
        internal->max_queues = MIN(IFCVF_MAX_QUEUES, internal->hw.blk_cfg->num_queues)
        DRV_LOG(DEBUG, "capacity  : %"PRIu64"G", capacity >> 21)
        DRV_LOG(DEBUG, "size_max  : 0x%08x", internal->hw.blk_cfg->size_max);
        DRV_LOG(DEBUG, "seg_max   : 0x%08x", internal->hw.blk_cfg->seg_max);
        DRV_LOG(DEBUG, "blk_size  : 0x%08x", internal->hw.blk_cfg->blk_size);
        DRV_LOG(DEBUG, "    cylinders: %u", internal->hw.blk_cfg->geometry.cylinders);
        DRV_LOG(DEBUG, "    heads    : %u", internal->hw.blk_cfg->geometry.heads);
        DRV_LOG(DEBUG, "    sectors  : %u", internal->hw.blk_cfg->geometry.sectors);
        DRV_LOG(DEBUG, "num_queues: 0x%08x",internal->hw.blk_cfg->num_queues);
    if (rte_kvargs_count(kvlist, IFCVF_SW_FALLBACK_LM)) -> sw-live-migration
        rte_kvargs_process(kvlist, IFCVF_SW_FALLBACK_LM, &open_int, &sw_fallback_lm)
    TAILQ_INSERT_TAIL(&internal_list, list, next)
    internal->vdev = rte_vdpa_register_device(&pci_dev->device, dev_info[internal->hw.device_type].ops) -> ifcvf_blk_ops | ifcvf_net_ops
        dev = __vdpa_find_device_by_name(rte_dev->name)
        dev = rte_zmalloc(NULL, sizeof(*dev), 0)
        dev->ops = ops
        ops->get_dev_type(dev, &dev->type)
        TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next)
    update_datapath(internal)
        ifcvf_dma_map(internal, true)
            rte_vhost_get_mem_table(internal->vid, &mem)
            rte_vfio_container_dma_map(vfio_container_fd, reg->host_user_addr, reg->guest_phys_addr, reg->size)
        vdpa_enable_vfio_intr(internal, false) -> enable irq -> 启动中断
            irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX
            fd_ptr = (int *)&irq_set->data -> arry -> __u8  data[]
            fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = rte_intr_fd_get(internal->pdev->intr_handle)
                return intr_handle->fd
            fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = vring.callfd -> bind vfio irq to callfd
            fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC)
            internal->intr_fd[i] = fd
            ret = ioctl(internal->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set) -> then kernel trigger irq by ->  eventfd_signal(trigger, 1) -> qemu handle irq by -> event_notifier_test_and_clear
        vdpa_ifcvf_start(internal)
            ifcvf_start_hw(&internal->hw)
                ifcvf_hw_enable(hw)
                    IFCVF_WRITE_REG16(0, &cfg->msix_config)
                    ifcvf_enable_mq(hw)
                    IFCVF_WRITE_REG16(i, &cfg->queue_select);
                    IFCVF_WRITE_REG16(i + 1, &cfg->queue_msix_vector)
                    hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + notify_off * hw->notify_off_multiplier)
                    IFCVF_WRITE_REG16(1, &cfg->queue_enable)
                ifcvf_add_status(hw, IFCVF_CONFIG_STATUS_DRIVER_OK)
                    ifcvf_set_status(hw, status)
                        IFCVF_WRITE_REG8(status, &hw->common_cfg->device_status)
                    ifcvf_get_status(hw)
        setup_notify_relay(internal)
            rte_ctrl_thread_create(&internal->tid, name, NULL, notify_relay, (void *)internal)
                epfd = epoll_create(IFCVF_MAX_QUEUES * 2)
                rte_vhost_get_vhost_vring(internal->vid, qid, &vring)
                epoll_ctl(epfd, EPOLL_CTL_ADD, vring.kickfd, &ev) -> when qemu kick vhost, wakup epoll
                for (;;)
                    nfds = epoll_wait(epfd, events, q_num, -1)
                    ifcvf_notify_queue(hw, qid)
        setup_intr_relay(internal)
            rte_ctrl_thread_create(&internal->intr_tid, name, NULL, intr_relay, (void *)internal) -> 创建具有给定名称和属性的控制线程。新线程的亲和性基于调用 rte_eal_init() 时检索到的 CPU 亲和性，然后排除数据平面和服务 lcore。如果设置线程名称失败，则忽略错误并记录调试消息 -> vdpa/ifc：为配置空间添加中断处理，创建一个线程来轮询和中继配置空间更改中断。使用 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG 通知 QEMU
                csc_epfd = epoll_create(1)
                ev.data.fd = rte_intr_fd_get(internal->pdev->intr_handle)
                for (;;)
                    csc_val = epoll_wait(csc_epfd, &csc_event, 1, -1)
                    nbytes = read(csc_event.data.fd, &buf, 8)
                    virtio_interrupt_handler(internal)
                        rte_vhost_slave_config_change(vid, 1)
                            vhost_user_slave_config_change(dev, need_reply)
                                .request.slave = VHOST_USER_SLAVE_CONFIG_CHANGE_MSG -> change notifications -> kernel virtio_config_changed
                                send_vhost_slave_message(dev, &ctx)
                                process_slave_message_reply(dev, &ctx)
    rte_kvargs_free(kvlist)

static struct rte_vdpa_dev_ops ifcvf_net_ops = {
	.get_queue_num = ifcvf_get_queue_num,
	.get_features = ifcvf_get_vdpa_features,
	.get_protocol_features = ifcvf_get_protocol_features,
	.dev_conf = ifcvf_dev_config,
        update_datapath(internal)
        rte_vhost_host_notifier_ctrl
	.dev_close = ifcvf_dev_close,
	.set_vring_state = ifcvf_set_vring_state,
        hw->vring[vring].enable = enable
        unset_notify_relay(internal)
        vdpa_enable_vfio_intr(internal, false)
            irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX
            fd_ptr = (int *)&irq_set->data -> arry -> __u8	data[]
            fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = rte_intr_fd_get(internal->pdev->intr_handle)
                return intr_handle->fd
            fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = vring.callfd -> bind vfio irq to callfd
            fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC)
            internal->intr_fd[i] = fd
            ret = ioctl(internal->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set) -> then kernel trigger irq by ->  eventfd_signal(trigger, 1) -> qemu handle irq by -> event_notifier_test_and_clear
        ifcvf_config_vring(internal, vring)
            if (hw->vring[vring].enable)
                rte_vhost_get_vhost_vring(vid, vring, &vq)
                gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.desc)
                hw->vring[vring].desc = gpa
                gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.avail)
                hw->vring[vring].avail = gpa
                gpa = hva_to_gpa(vid, (uint64_t)(uintptr_t)vq.used)
                hw->vring[vring].used = gpa
                rte_vhost_get_vring_base(vid, vring, &hw->vring[vring].last_avail_idx, &hw->vring[vring].last_used_idx)
                ifcvf_enable_vring_hw(&internal->hw, vring)
                    ifcvf_enable_mq(hw)
                    IFCVF_WRITE_REG16(i, &cfg->queue_select)
                    io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, &cfg->queue_desc_hi)
                    hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + notify_off * hw->notify_off_multiplier)
                    IFCVF_WRITE_REG16(1, &cfg->queue_enable)
        setup_notify_relay(internal)
        rte_vhost_host_notifier_ctrl(vid, vring, enable)
	.set_features = ifcvf_set_features,
	.migration_done = NULL,
	.get_vfio_group_fd = ifcvf_get_vfio_group_fd,
	.get_vfio_device_fd = ifcvf_get_vfio_device_fd,
	.get_notify_area = ifcvf_get_notify_area,
	.get_dev_type = ifcvf_get_device_type,
};


static struct rte_vdpa_dev_ops ifcvf_blk_ops = {
	.get_queue_num = ifcvf_get_queue_num,
        *queue_num = list->internal->max_queues
	.get_features = ifcvf_get_vdpa_features,
        *features = list->internal->features
	.set_features = ifcvf_set_features,
        if (internal->sw_lm) -> net/ifc：支持 SW 辅助 VDPA 实时迁移, 在 SW 辅助实时迁移模式下，驱动程序将停止设备并设置一个中介 virtio 环来中继 virtio 驱动程序和 VDPA 设备之间的通信。此数据路径干预将允许 SW 帮助客户机记录实时迁移的脏页。此 SW 回退是事件驱动的中继线程，因此当网络吞吐量较低时，此 SW 回退将占用很少的 CPU 资源，但当吞吐量上升时，中继线程的 CPU 使用率将相应上升。用户在选择实时迁移支持模式时需要考虑所有因素，包括 CPU 使用率、客户机性能下降等。
            ifcvf_sw_fallback_switchover(internal)
                stop the direct IO data path
                unset_notify_relay
                vdpa_ifcvf_stop
                unset_intr_relay
                vdpa_disable_vfio_intr
                rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false) -> 内部函数
                    vfio_device_fd = vdpa_dev->ops->get_vfio_device_fd(vid)
                    if (enable)
                        vdpa_dev->ops->get_notify_area(vid, i, &offset, &size) < 0)
                        vhost_user_slave_set_vring_host_notifier(dev, i,vfio_device_fd, offset, size)
                            .request.slave = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
                vdpa_enable_vfio_intr(internal, true)
                config the VF
                m_ifcvf_start
                setup_vring_relay
                    rte_ctrl_thread_create(&internal->tid, name, NULL, vring_relay, (void *)internal)
                        epfd = epoll_create(IFCVF_MAX_QUEUES * 2)
                        epoll_ctl(epfd, EPOLL_CTL_ADD, vring.kickfd, &ev
                        nfds = epoll_wait(epfd, events, q_num * 2, -1)
                        update_used_ring(internal, qid)
                            rte_vhost_vring_call(internal->vid, qid) -> Notify the guest that used descriptors have been added to the vring. This function acts as a memory barrier.
                        for (;;)
                            nfds = epoll_wait(epfd, events, q_num * 2, -1)
                            nbytes = read(fd, &buf, 8)
                            qid = events[i].data.u32 >> 1
                            if (events[i].data.u32 & 1)
                                update_used_ring(internal, qid)
                            else
                                ifcvf_notify_queue(&internal->hw, qid)
                                    IFCVF_WRITE_REG16(qid, hw->notify_addr[qid])
                rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true)
        else
            rte_vhost_get_log_base(vid, &log_base, &log_size)
            rte_vfio_container_dma_map(internal->vfio_container_fd, log_base, IFCVF_LOG_BASE, log_size)
            ifcvf_enable_logging(&internal->hw, IFCVF_LOG_BASE, log_size)
	.get_protocol_features = ifcvf_blk_get_protocol_features,
	.dev_conf = ifcvf_dev_config,
	.dev_close = ifcvf_dev_close,
	.set_vring_state = ifcvf_set_vring_state,
	.migration_done = NULL,
	.get_vfio_group_fd = ifcvf_get_vfio_group_fd,
	.get_vfio_device_fd = ifcvf_get_vfio_device_fd,
	.get_notify_area = ifcvf_get_notify_area,
        ret = ioctl(internal->vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg)
        *offset = ifcvf_get_queue_notify_off(&internal->hw, qid) + reg.offset
            return (u8 *)hw->notify_addr[qid] - (u8 *)hw->mem_resource[hw->notify_region].addr
        *size = 0x1000
	.get_config = ifcvf_blk_get_config,
	.get_dev_type = ifcvf_get_device_type,
};

case VFIO_DEVICE_SET_IRQS
            vfio_pci_ioctl_set_irqs(vdev, uarg)
                struct vfio_irq_set hdr
                max = vfio_pci_get_irq_count(vdev, hdr.index)
                    if (irq_type == VFIO_PCI_INTX_IRQ_INDEX)
                        pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin)
                    else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX)
                        pos = vdev->pdev->msi_cap
                        pci_read_config_word(vdev->pdev, pos + PCI_MSI_FLAGS, &flags)
                    else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX)
                        pos = vdev->pdev->msix_cap
                        pci_read_config_word(vdev->pdev, pos + PCI_MSIX_FLAGS, &flags)
                vfio_set_irqs_validate_and_prepare(&hdr, max, VFIO_PCI_NUM_IRQS, &data_size)
                vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start, hdr.count, data)
                    case VFIO_IRQ_SET_ACTION_TRIGGER
                        vfio_pci_set_msi_trigger
                            vfio_msi_enable
                                pci_alloc_irq_vectors(pdev, 1, nvec, flag)
                            vfio_msi_set_block(vdev, start, count, fds, msix)
                                vfio_msi_set_vector_signal(vdev, j, fd, msix)
                                    irq = pci_irq_vector(pdev, vector)
                                    trigger = eventfd_ctx_fdget(fd)
                                    request_irq(irq, vfio_msihandler, 0, ctx->name, trigger) -> trigger eventfd，进而引起producer状态变化，最终导致consumer状态变化 -> 请求中断
                                        eventfd_signal(trigger)
                                            eventfd_signal_mask(ctx, 0)
                                                wake_up_locked_poll(&ctx->wqh, EPOLLIN | mask)
                                    irq_bypass_register_producer(&ctx->producer)

QEMU提前注册host -> guest通知的eventfd回调:
k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
        bool with_irqfd = msix_enabled(&proxy->pci_dev) && kvm_msi_via_irqfd_enabled()
        msix_unset_vector_notifiers(&proxy->pci_dev)
        virtio_pci_set_guest_notifier(d, n, assign, with_irqfd)
            notifier = virtio_config_get_guest_notifier(vdev)
            or notifier = virtio_queue_get_guest_notifier(vq)
                return &vq->guest_notifier
            virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, true, with_irqfd)
                event_notifier_set_handler(&vq->guest_notifier, virtio_queue_guest_notifier_read)
                    event_notifier_set_handler
                        new_node->io_read = io_read
                    virtio_queue_guest_notifier_read
                    event_notifier_test_and_clear(n)
                        read(e->rfd, buffer, sizeof(buffer)
                    virtio_irq(vq) -> 向GUEST注入中断
                             
QEMU开启主循环:
os_host_main_loop_wait
    glib_pollfds_fill(&timeout)
    ret = qemu_poll_ns((GPollFD *)gpollfds->data, gpollfds->len, timeout)
        ppoll((struct pollfd *)fds, nfds, NULL, NULL)
        or g_poll(fds, nfds, qemu_timeout_ns_to_ms(timeout))
            poll(fds, nfds, timeout)
    glib_pollfds_poll
        g_main_context_dispatch(context) -> aio_ctx_dispatch -> aio_dispatch(ctx) -> aio_dispatch_handlers
            QLIST_FOREACH_SAFE_RCU(node, &ctx->aio_handlers, node, tmp)
                progress = aio_dispatch_handler(ctx, node) || progress
                    node->io_read(node->opaque) -> virtio_queue_guest_notifier_read
    g_main_context_release(context)