ignite

containerd-shim─┬─ignite-spawn─┬─firecracker───2*[{firecracker}]
                │              └─14*[{ignite-spawn}]
                └─11*[{containerd-shim}]

type Interface interface {
	PullImage(image meta.OCIImageRef) error
	InspectImage(image meta.OCIImageRef) (*ImageInspectResult, error)
	ExportImage(image meta.OCIImageRef) (io.ReadCloser, func() error, error)

	InspectContainer(container string) (*ContainerInspectResult, error)
	AttachContainer(container string) error
	RunContainer(image meta.OCIImageRef, config *ContainerConfig, name, id string) (string, error)
	StopContainer(container string, timeout *time.Duration) error
	KillContainer(container, signal string) error
	RemoveContainer(container string) error
	ContainerLogs(container string) (io.ReadCloser, error)

	Name() Name
	RawClient() interface{}

	PreflightChecker() preflight.Checker
}

type Storage interface {
	// New creates a new Object for the specified kind
	New(gvk schema.GroupVersionKind) (runtime.Object, error)
	// Get returns a new Object for the resource at the specified kind/uid path, based on the file content
	Get(gvk schema.GroupVersionKind, uid runtime.UID) (runtime.Object, error)
	// GetMeta returns a new Object's APIType representation for the resource at the specified kind/uid path
	GetMeta(gvk schema.GroupVersionKind, uid runtime.UID) (runtime.Object, error)
	// Set saves the Object to disk. If the Object does not exist, the
	// ObjectMeta.Created field is set automatically
	Set(gvk schema.GroupVersionKind, obj runtime.Object) error
	// Patch performs a strategic merge patch on the Object with the given UID, using the byte-encoded patch given
	Patch(gvk schema.GroupVersionKind, uid runtime.UID, patch []byte) error
	// Delete removes an Object from the storage
	Delete(gvk schema.GroupVersionKind, uid runtime.UID) error
	// List lists Objects for the specific kind
	List(gvk schema.GroupVersionKind) ([]runtime.Object, error)
	// ListMeta lists all Objects' APIType representation. In other words,
	// only metadata about each Object is unmarshalled (uid/name/kind/apiVersion).
	// This allows for faster runs (no need to unmarshal "the world"), and less
	// resource usage, when only metadata is unmarshalled into memory
	ListMeta(gvk schema.GroupVersionKind) ([]runtime.Object, error)
	// Count returns the amount of available Objects of a specific kind
	// This is used by Caches to check if all Objects are cached to perform a List
	Count(gvk schema.GroupVersionKind) (uint64, error)
	// Checksum returns a string representing the state of an Object on disk
	// The checksum should change if any modifications have been made to the
	// Object on disk, it can be e.g. the Object's modification timestamp or
	// calculated checksum
	Checksum(gvk schema.GroupVersionKind, uid runtime.UID) (string, error)
	// RawStorage returns the RawStorage instance backing this Storage
	RawStorage() RawStorage
	// Serializer returns the serializer
	Serializer() serializer.Serializer
	// Close closes all underlying resources (e.g. goroutines) used; before the application exits
	Close() error
}

var (
	// Scheme is the runtime.Scheme to which all types are registered.
	Scheme = runtime.NewScheme()

	// codecs provides access to encoding and decoding for the scheme.
	// codecs is private, as Serializer will be used for all higher-level encoding/decoding
	codecs = k8sserializer.NewCodecFactory(Scheme)

	// Serializer provides high-level encoding/decoding functions
	Serializer = serializer.NewSerializer(Scheme, &codecs)
)

func init() {
	AddToScheme(Scheme)
}

// AddToScheme builds the scheme using all known versions of the api.
func AddToScheme(scheme *runtime.Scheme) {
	utilruntime.Must(ignite.AddToScheme(Scheme))
	utilruntime.Must(v1alpha2.AddToScheme(Scheme))
	utilruntime.Must(v1alpha3.AddToScheme(Scheme))
	utilruntime.Must(v1alpha4.AddToScheme(Scheme))
	utilruntime.Must(scheme.SetVersionPriority(v1alpha4.SchemeGroupVersion))
}

func SetGenericStorage() error {
	log.Trace("Initializing the GenericStorage provider...")
	providers.Storage = cache.NewCache(
		storage.NewGenericStorage(
			storage.NewGenericRawStorage(constants.DATA_DIR), scheme.Serializer))
	return nil
}

func SetManifestStorage() (err error) {
	log.Trace("Initializing the ManifestStorage provider...")
	ManifestStorage, err = manifest.NewTwoWayManifestStorage(constants.MANIFEST_DIR, constants.DATA_DIR, scheme.Serializer)
	if err != nil {
		return
	}

	providers.Storage = cache.NewCache(ManifestStorage)
	return
}

$ ignite image import <OCI image> [flags]
$ ignite kernel import <OCI image> [flags]

$ ctr -n firecracker containers ls 
CONTAINER                  IMAGE                                  RUNTIME                  
ignite-ddf49307b5b27c34    docker.io/weaveworks/ignite:v0.10.0    io.containerd.runc.v2    
$ ctr -n firecracker images ls 
REF           TYPE             DIGEST                                                                  SIZE     PLATFORMS                  LABELS 
docker.io/weaveworks/ignite-kernel:5.10.51 application/vnd.docker.distribution.manifest.list.v2+json sha256:c1d99eafa5b2bcaeab26c0a093d83d709a560e4721f52b6e7c5ef7e9e771189d 15.0 MiB linux/amd64,linux/arm64    -      
docker.io/weaveworks/ignite-ubuntu:latest  application/vnd.docker.distribution.manifest.list.v2+json sha256:11550e0912d24aeaad847f06fdf2133302f2af2fd2ce231723d078ffce9216ba 78.1 MiB linux/amd64,linux/arm64/v8 -      
docker.io/weaveworks/ignite:v0.10.0        application/vnd.docker.distribution.manifest.list.v2+json sha256:b8cc53c5cba81d685b1dc95a0f34ca3fa732ddd450b6f0eba0c829ccc1c67462 16.5 MiB linux/amd64,linux/arm64    -   

func GetContainerdClient() (*ctdClient, error) {
	ctdSocket, err := StatContainerdSocket()
	if err != nil {
		return nil, err
	}

	runtime, err := getNewestAvailableContainerdRuntime()//获取可用的runtime
	if err != nil {
		// proceed with the default runtime -- our PATH can't see a shim binary, but containerd might be able to
		log.Warningf("Proceeding with default runtime %q: %v", runtime, err)
	}

	cli, err := containerd.New(
		ctdSocket,
		containerd.WithDefaultRuntime(runtime),
	)
	if err != nil {
		return nil, err
	}

	return &ctdClient{
		client: cli,
		ctx:    namespaces.WithNamespace(context.Background(), ctdNamespace), //设置ignite的命名空间
	}, nil
}

func getNewestAvailableContainerdRuntime() (string, error) {
	for _, rt := range v2ShimRuntimes {
		binary := v2shim.BinaryName(rt)
		if binary == "" {
			// this shouldn't happen if the matching test is passing, but it's not fatal -- just log and continue
			log.Errorf("shim binary could not be found -- %q is an invalid runtime/v2/shim", rt)
		} else if _, err := exec.LookPath(binary); err == nil {
			return rt, nil
		}
	}
	...
}

v2ShimRuntimes = []string{
  plugin.RuntimeRuncV2,
  plugin.RuntimeRuncV1,
}

const (
	// RuntimeLinuxV1 is the legacy linux runtime
	RuntimeLinuxV1 = "io.containerd.runtime.v1.linux"
	// RuntimeRuncV1 is the runc runtime that supports a single container
	RuntimeRuncV1 = "io.containerd.runc.v1"
	// RuntimeRuncV2 is the runc runtime that supports multiple containers per shim
	RuntimeRuncV2 = "io.containerd.runc.v2"
)

	cli, err := containerd.New(
		ctdSocket,
		containerd.WithDefaultRuntime(runtime),
	)

func GetCNINetworkPlugin(runtime runtime.Interface) (network.Plugin, error) {
	// If the CNI configuration directory doesn't exist, create it
	if !util.DirExists(CNIConfDir) {
		if err := os.MkdirAll(CNIConfDir, constants.DATA_DIR_PERM); err != nil {
			return nil, err
		}
	}

	binDirs := []string{CNIBinDir}
	cniInstance, err := gocni.New(gocni.WithMinNetworkCount(2),
		gocni.WithPluginConfDir(CNIConfDir),
		gocni.WithPluginDir(binDirs))
	if err != nil {
		return nil, err
	}

	return &cniNetworkPlugin{
		runtime: runtime,
		cni:     cniInstance,
		once:    &sync.Once{},
	}, nil
}

	// CNIBinDir describes the directory where the CNI binaries are stored
	CNIBinDir = "/opt/cni/bin"
	// CNIConfDir describes the directory where the CNI plugin's configuration is stored
	CNIConfDir = "/etc/cni/net.d"

func FindOrImportImage(c *client.Client, ociRef meta.OCIImageRef) (*api.Image, error) {
	log.Debugf("Ensuring image %s exists, or importing it...", ociRef)
	image, err := c.Images().Find(filter.NewIDNameFilter(ociRef.String())) //查看元数据中是否存在需要的镜像
	if err == nil {
		// Return the image found
		log.Debugf("Found image with UID %s", image.GetUID())
		return image, nil
	}

	switch err.(type) {
	case *filterer.NonexistentError:
		return importImage(c, ociRef) //从containerd本地或远端加载镜像
	default:
		return nil, err
	}
}

func newImageClient(s storage.Storage, gv schema.GroupVersion) ImageClient {
	return &imageClient{
		storage:  s,
		filterer: filterer.NewFilterer(s),
		gvk:      gv.WithKind(api.KindImage.Title()),
	}
}

func importImage(c *client.Client, ociRef meta.OCIImageRef) (*api.Image, error) {
	log.Debugf("Importing image with ociRef %q", ociRef)
	// Parse the source
	dockerSource := source.NewDockerSource()
	src, err := dockerSource.Parse(ociRef) //从containerd本地加载或远端拉取镜像
	if err != nil {
		return nil, err
	}

	image := c.Images().New() //初始化一个ignite image对象
	// Set the image name
	image.Name = ociRef.String()
	// Set the image's ociRef
	image.Spec.OCI = ociRef
	// Set the image's ociSource
	image.Status.OCISource = *src

	// Generate UID automatically
	if err := metadata.SetNameAndUID(image, c); err != nil { //设置image对象的UID
		return nil, err
	}

	log.Infoln("Starting image import...")

	// Truncate a file for the filesystem, format it with ext4, and copy in the files from the source
	if err := dmlegacy.CreateImageFilesystem(image, dockerSource); err != nil { //创建ext4文件系统，并导入镜像文件
		return nil, err
	}

	if err := c.Images().Set(image); err != nil  //存储新镜像的元数据
		return nil, err
	}

	log.Infof("Imported OCI image %q (%s) to base image with UID %q", ociRef, image.Status.OCISource.Size, image.GetUID())
	return image, nil
}

func (ds *DockerSource) Parse(ociRef meta.OCIImageRef) (*api.OCIImageSource, error) {
	res, err := providers.Runtime.InspectImage(ociRef)
	if err != nil {
		log.Infof("%s image %q not found locally, pulling...", providers.Runtime.Name(), ociRef)
		if err := providers.Runtime.PullImage(ociRef); err != nil {
			return nil, err
		}

		if res, err = providers.Runtime.InspectImage(ociRef); err != nil {
			return nil, err
		}
	}

	if res.Size == 0 || res.ID == nil {
		return nil, fmt.Errorf("parsing %s image %q data failed", providers.Runtime.Name(), ociRef)
	}

	ds.imageRef = ociRef

	return &api.OCIImageSource{
		ID:   res.ID,
		Size: meta.NewSizeFromBytes(uint64(res.Size)),
	}, nil
}

# cd /var/lib/firecracker/image/669a5721d130ef1d

# ll
-rw-r--r--. 1 root root 295698432 Jul 14 10:53 image.ext4
-rw-r--r--. 1 root root       464 Jul 14 10:53 metadata.json

# cat metadata.json 
{
  "kind": "Image",
  "apiVersion": "ignite.weave.works/v1alpha4",
  "metadata": {
    "name": "weaveworks/ignite-ubuntu:latest",
    "uid": "669a5721d130ef1d",
    "created": "2023-07-14T02:53:01Z"
  },
  "spec": {
    "oci": "weaveworks/ignite-ubuntu:latest"
  },
  "status": {
    "ociSource": {
      "id": "oci://docker.io/weaveworks/ignite-ubuntu@sha256:52414720f26c808bc1273845c6d0f0a99472dfa8eaf8df52429261cbac27f1ba",
      "size": "249308KB"
    }
  }
}

type Image struct {
	runtime.TypeMeta `json:",inline"`
	// runtime.ObjectMeta is also embedded into the struct, and defines the human-readable name, and the machine-readable ID
	// Name is available at the .metadata.name JSON path
	// ID is available at the .metadata.uid JSON path (the Go type is k8s.io/apimachinery/pkg/types.UID, which is only a typed string)
	runtime.ObjectMeta `json:"metadata"`

	Spec   ImageSpec   `json:"spec"`
	Status ImageStatus `json:"status"`
}

// ImageSpec declares what the image contains
type ImageSpec struct {
	OCI meta.OCIImageRef `json:"oci"`
}

type OCIImageSource struct {
	// ID defines the source's content ID (e.g. the canonical OCI path or Docker image ID)
	ID *meta.OCIContentID `json:"id"`
	// Size defines the size of the source in bytes
	Size meta.Size `json:"size"`
}

// ImageStatus defines the status of the image
type ImageStatus struct {
	// OCISource contains the information about how this OCI image was imported
	OCISource OCIImageSource `json:"ociSource"`
}

func addFiles(img *api.Image, src source.Source) (err error) {
	log.Debugf("Copying in files to the image file from a source...")
	p := path.Join(img.ObjectPath(), constants.IMAGE_FS) //mkfs创建出来的ext4文件路径
	tempDir, err := ioutil.TempDir("", "") //创建一个临时文件
	if err != nil {
		return
	}
	defer os.RemoveAll(tempDir)

	if _, err := util.ExecuteCommand("mount", "-o", "loop", p, tempDir); err != nil { //挂载ext4文件系统到临时目录
		return fmt.Errorf("failed to mount image %q: %v", p, err)
	}
	defer util.DeferErr(&err, func() error {
		_, execErr := util.ExecuteCommand("umount", tempDir)
		return execErr
	})

	err = source.TarExtract(src, tempDir)//将基础镜像解压到历史目录中
	if err != nil {
		return
	}

	err = setupResolvConf(tempDir)//确保存在/etc/resolv.conf文件

	return
}

$ pwd
/var/lib/firecracker/kernel/1bdd3b2354873157

$ ll
-rw-r--r--. 1 root root 73574400 Jul 14 10:53 kernel.tar
-rw-r--r--. 1 root root      492 Jul 14 10:53 metadata.json
-rwxr-xr-x. 1 root root 43526368 Jul 14 10:53 vmlinux

// importKernel imports a kernel from an OCI image
func importKernel(c *client.Client, ociRef meta.OCIImageRef) (*api.Kernel, error) {
	log.Debugf("Importing kernel with ociRef %q", ociRef)
	// Parse the source
	dockerSource := source.NewDockerSource()
	src, err := dockerSource.Parse(ociRef) //从containerd本地或远端加载镜像
	if err != nil {
		return nil, err
	}

	kernel := c.Kernels().New() //初始化一个image对象
	// Set the kernel name
	kernel.Name = ociRef.String()
	// Set the kernel's ociRef
	kernel.Spec.OCI = ociRef
	// Set the kernel's ociSource
	kernel.Status.OCISource = *src

	// Generate UID automatically
	if err := metadata.SetNameAndUID(kernel, c); err != nil { //设置kernel对象的UID
		return nil, err
	}

	// Cache the kernel contents in the kernel tar file
	kernelTarFile := path.Join(kernel.ObjectPath(), constants.KERNEL_TAR)

	// vmlinuxFile describes the uncompressed kernel file at /var/lib/firecracker/kernel/<id>/vmlinux
	vmlinuxFile := path.Join(kernel.ObjectPath(), constants.KERNEL_FILE)

	// Create both the kernel tar file and the vmlinux file it either doesn't exist
	if !util.FileExists(kernelTarFile) || !util.FileExists(vmlinuxFile) {
		// Create a temporary directory for extracting
		// the necessary files from the OCI image
		tempDir, err := ioutil.TempDir("", "")
		if err != nil {
			return nil, err
		}

		// Extract only the /boot and /lib directories of the tar stream into the tempDir
		err = source.TarExtract(dockerSource, tempDir, "boot", "lib/modules") //抽取所需的内核文件到临时目录
		if err != nil {
			return nil, err
		}

		// Locate the kernel file in the temporary directory
		kernelTmpFile, err := findKernel(tempDir) //查找vmlinux文件
		if err != nil {
			return nil, err
		}

		// Copy the vmlinux file
		if err := util.CopyFile(kernelTmpFile, vmlinuxFile); err != nil {
			return nil, fmt.Errorf("failed to copy kernel file %q to kernel %q: %v", kernelTmpFile, kernel.GetUID(), err)
		}

		// 将抽取出来的内核文件打包到 /var/lib/firecracker/kernel/<UID>/kernel.tar
		if _, err := util.ExecuteCommand("tar", "-cf", kernelTarFile, "-C", tempDir, "."); err != nil {
			return nil, err
		}

		// 移除临时目录
		if err := os.RemoveAll(tempDir); err != nil {
			return nil, err
		}
	}

	// Populate the kernel version field if possible
	if len(kernel.Status.Version) == 0 {
		cmd := fmt.Sprintf("strings %s | grep 'Linux version' | awk '{print $3}'", vmlinuxFile)
		// Use the pipefail option to return an error if any of the pipeline commands is not available
		out, err := util.ExecuteCommand("/bin/bash", "-o", "pipefail", "-c", cmd)
		if err != nil {
			kernel.Status.Version = "<unknown>"
		} else {
			kernel.Status.Version = out
		}
	}

	if err := c.Kernels().Set(kernel); err != nil { //将内核对象保存到Storage中
		return nil, err
	}

	log.Infof("Imported OCI image %q (%s) to kernel image with UID %q", ociRef, kernel.Status.OCISource.Size, kernel.GetUID())
	return kernel, nil
}

# cat metadata.json 
{
  "kind": "Kernel",
  "apiVersion": "ignite.weave.works/v1alpha4",
  "metadata": {
    "name": "weaveworks/ignite-kernel:5.10.51",
    "uid": "1bdd3b2354873157",
    "created": "2023-07-14T02:53:10Z"
  },
  "spec": {
    "oci": "weaveworks/ignite-kernel:5.10.51"
  },
  "status": {
    "version": "5.10.51",
    "ociSource": {
      "id": "oci://docker.io/weaveworks/ignite-kernel@sha256:a992aa9f7b6f5e7945e72610017c3f4f38338ff1452964e30410bb6110a794a7",
      "size": "72588KB"
    }
  }
}

type Kernel struct {
	runtime.TypeMeta `json:",inline"`
	// runtime.ObjectMeta is also embedded into the struct, and defines the human-readable name, and the machine-readable ID
	// Name is available at the .metadata.name JSON path
	// ID is available at the .metadata.uid JSON path (the Go type is k8s.io/apimachinery/pkg/types.UID, which is only a typed string)
	runtime.ObjectMeta `json:"metadata"`

	Spec   KernelSpec   `json:"spec"`
	Status KernelStatus `json:"status"`
}

// KernelSpec describes the properties of a kernel
type KernelSpec struct {
	OCI meta.OCIImageRef `json:"oci"`
	// Optional future feature, support per-kernel specific default command lines
	// DefaultCmdLine string
}

// KernelStatus describes the status of a kernel
type KernelStatus struct {
	Version   string         `json:"version"`
	OCISource OCIImageSource `json:"ociSource"`
}

func (cf *CreateFlags) NewCreateOptions(args []string, fs *flag.FlagSet) (*CreateOptions, error) {
	// Create a new base VM and configure it by combining the component config,
	// VM config file and flags.
	baseVM := providers.Client.VMs().New() //初始化一个vm对象

	// If component config is in use, set the VMDefaults on the base VM.
	if providers.ComponentConfig != nil {
		baseVM.Spec = providers.ComponentConfig.Spec.VMDefaults
	}

	// Resolve registry configuration used for pulling image if required.
	cmdutil.ResolveRegistryConfigDir()

	// Initialize the VM's Prefixer
	baseVM.Status.IDPrefix = providers.IDPrefix //设置vm对象的基本信息
	// Set the runtime and network-plugin on the VM, then override the global config.
	baseVM.Status.Runtime.Name = providers.RuntimeName // 设置runtime 和 CNI实例
	baseVM.Status.Network.Plugin = providers.NetworkPluginName
	// Populate the runtime and network-plugin providers.
	if err := config.SetAndPopulateProviders(providers.RuntimeName, providers.NetworkPluginName); err != nil {
		return nil, err
	}

	// Set the passed image argument on the new VM spec.
	// Image is necessary while serializing the VM spec.
	if len(args) == 1 {
		ociRef, err := meta.NewOCIImageRef(args[0])
		if err != nil {
			return nil, err
		}
		baseVM.Spec.Image.OCI = ociRef
	}

	// Generate a VM name and UID if not set yet.
	if err := metadata.SetNameAndUID(baseVM, providers.Client); err != nil {//设置vm的UID和名称
		return nil, err
	}

	// Apply the VM config on the base VM, if a VM config is given.
	if len(cf.ConfigFile) != 0 {//如果使用文件指定了vm对象的配置信息，则将该配置合并到vm对象中
		if err := applyVMConfigFile(baseVM, cf.ConfigFile); err != nil {
			return nil, err
		}
	}

	// Apply flag overrides.
	if err := applyVMFlagOverrides(baseVM, cf, fs); err != nil {//使用命令行参数覆盖vm对象
		return nil, err
	}

	// If --require-name is true, VM name must be provided.
	if cf.RequireName && len(baseVM.Name) == 0 {
		return nil, fmt.Errorf("must set VM name, flag --require-name set")
	}

	// Assign the new VM to the configFlag.
	cf.VM = baseVM

	// Validate the VM object.
	if err := validation.ValidateVM(cf.VM).ToAggregate(); err != nil { //vm对象有效性校验
		return nil, err
	}

	co := &CreateOptions{CreateFlags: cf}
  //下面用于拉取基础镜像和内核镜像，相当于 ignite image import 和 ignite kernel import
	// Get the image, or import it if it doesn't exist.
	var err error
	co.image, err = operations.FindOrImportImage(providers.Client, cf.VM.Spec.Image.OCI)
	if err != nil {
		return nil, err
	}

	// Populate relevant data from the Image on the VM object.
	cf.VM.SetImage(co.image) //设置vm对象的image信息

	// Get the kernel, or import it if it doesn't exist.
	co.kernel, err = (providers.Client, cf.VM.Spec.Kernel.OCI)
	if err != nil {
		return nil, err
	}

	// Populate relevant data from the Kernel on the VM object.
	cf.VM.SetKernel(co.kernel) //设置vm对象的kernel元数据
	return co, nil
}

$ cat metadata.json 
{
  "kind": "VM",
  "apiVersion": "ignite.weave.works/v1alpha4",
  "metadata": {
    "name": "restless-waterfall",
    "uid": "ddf49307b5b27c34",
    "created": "2023-07-18T08:33:25Z"
  },
  "spec": {
    "image": {
      "oci": "weaveworks/ignite-ubuntu:latest"
    },
    "sandbox": {
      "oci": "weaveworks/ignite:v0.10.0"
    },
    "kernel": {
      "oci": "weaveworks/ignite-kernel:5.10.51",
      "cmdLine": "console=ttyS0 reboot=k panic=1 pci=off ip=dhcp"
    },
    "cpus": 1,
    "memory": "512MB",
    "diskSize": "4GB",
    "network": {
    },
    "storage": {
    },
    "ssh": true
  },
  "status": {
    "running": true,
    "runtime": {
      "id": "ignite-ddf49307b5b27c34",
      "name": "containerd"
    },
    "startTime": "2023-07-18T08:33:25Z",
    "network": {
      "plugin": "cni",
      "ipAddresses": [
        "10.61.0.3"
      ]
    },
    "image": {
      "id": "oci://docker.io/weaveworks/ignite-ubuntu@sha256:52414720f26c808bc1273845c6d0f0a99472dfa8eaf8df52429261cbac27f1ba",
      "size": "249308KB"
    },
    "kernel": {
      "id": "oci://docker.io/weaveworks/ignite-kernel@sha256:a992aa9f7b6f5e7945e72610017c3f4f38338ff1452964e30410bb6110a794a7",
      "size": "72588KB"
    },
    "idPrefix": "ignite"
  }
}

type VM struct {
	runtime.TypeMeta `json:",inline"`
	// runtime.ObjectMeta is also embedded into the struct, and defines the human-readable name, and the machine-readable ID
	// Name is available at the .metadata.name JSON path
	// ID is available at the .metadata.uid JSON path (the Go type is k8s.io/apimachinery/pkg/types.UID, which is only a typed string)
	runtime.ObjectMeta `json:"metadata"`

	Spec   VMSpec   `json:"spec"`
	Status VMStatus `json:"status"`
}

// VMSpec describes the configuration of a VM
type VMSpec struct {
	Image    VMImageSpec   `json:"image"`
	Sandbox  VMSandboxSpec `json:"sandbox"`
	Kernel   VMKernelSpec  `json:"kernel"`
	CPUs     uint64        `json:"cpus"`
	Memory   meta.Size     `json:"memory"`
	DiskSize meta.Size     `json:"diskSize"`
	// TODO: Implement working omitempty without pointers for the following entries
	// Currently both will show in the JSON output as empty arrays. Making them
	// pointers requires plenty of nil checks (as their contents are accessed directly)
	// and is very risky for stability. APIMachinery potentially has a solution.
	Network VMNetworkSpec `json:"network,omitempty"`
	Storage VMStorageSpec `json:"storage,omitempty"`
	// This will be done at either "ignite start" or "ignite create" time
	// TODO: We might revisit this later
	CopyFiles []FileMapping `json:"copyFiles,omitempty"`
	// SSH specifies how the SSH setup should be done
	// nil here means "don't do anything special"
	// If SSH.Generate is set, Ignite will generate a new SSH key and copy it in to authorized_keys in the VM
	// Specifying a path in SSH.Generate means "use this public key"
	// If SSH.PublicKey is set, this struct will marshal as a string using that path
	// If SSH.Generate is set, this struct will marshal as a bool => true
	SSH *SSH `json:"ssh,omitempty"`
}

func Create(co *CreateOptions) (err error) {
	// Generate a random UID and Name
	if err = metadata.SetNameAndUID(co.VM, providers.Client); err != nil {
		return
	}
	// Set VM labels.
	if err = metadata.SetLabels(co.VM, co.Labels); err != nil {
		return
	}
	defer util.DeferErr(&err, func() error { return metadata.Cleanup(co.VM, false) })

	if err = providers.Client.VMs().Set(co.VM); err != nil {// 将vm对象存储到Storage中
		return
	}

	// Allocate and populate the overlay file
	if err = dmlegacy.AllocateAndPopulateOverlay(co.VM); err != nil {//创建vm文件系统
		return
	}

	err = metadata.Success(co.VM)

	return
}

func AllocateAndPopulateOverlay(vm *api.VM) error {
	requestedSize := vm.Spec.DiskSize.Bytes()
	// Truncate only accepts an int64
  if requestedSize > math.MaxInt64 {
		return fmt.Errorf("requested size %d too large, cannot truncate", requestedSize)
	}
	size := int64(requestedSize)

	//获取基础镜像的UID，用于在/var/lib/firecracker/image中查找image.ext4
	imageUID, err := lookup.ImageUIDForVM(vm, providers.Client)
	if err != nil {
		return err
	}

	// Get the size of the image ext4 file
	fi, err := os.Stat(path.Join(constants.IMAGE_DIR, imageUID.String(), constants.IMAGE_FS))//查找image.ext4
	if err != nil {
		return err
	}
	imageSize := fi.Size()

	// The overlay needs to be at least as large as the image
	if size < imageSize { //调整overlay.dm的大小
		log.Warnf("warning: requested overlay size (%s) < image size (%s), using image size for overlay\n",
			vm.Spec.DiskSize.String(), meta.NewSizeFromBytes(uint64(imageSize)).String())
		size = imageSize
	}

	// Make sure the all directories above the snapshot directory exists
	if err := os.MkdirAll(path.Dir(vm.OverlayFile()), constants.DATA_DIR_PERM); err != nil {
		return err
	}

	overlayFile, err := os.Create(vm.OverlayFile())//创建vm的overlay文件
	if err != nil {
		return fmt.Errorf("failed to create overlay file for %q, %v", vm.GetUID(), err)
	}
	defer overlayFile.Close()

	if err := overlayFile.Truncate(size); err != nil {//调整overlay文件大小
		return fmt.Errorf("failed to allocate overlay file for VM %q: %v", vm.GetUID(), err)
	}

	// populate the filesystem
	return copyToOverlay(vm)//创建snapshot类型的devicemapper设备
}

func copyToOverlay(vm *api.VM) (err error) {
	_, err = ActivateSnapshot(vm) //创建devicemapper的snapshot存储，作为vm的启动设备
	if err != nil {
		return
	}
	defer util.DeferErr(&err, func() error { return DeactivateSnapshot(vm) })

	mp, err := util.Mount(vm.SnapshotDev()) //挂载snapshot存储
	if err != nil {
		return
	}
	defer util.DeferErr(&err, mp.Umount)

	// Copy the kernel files to the VM. TODO: Use snapshot overlaying instead.
  //将/var/lib/firecracker/kernel/<UID>/kernel.tar解压到挂载路径下，与基础文件系统进行合并
	if err = copyKernelToOverlay(vm, mp.Path); err != nil { 
		return
	}

	// do not mutate vm.Spec.CopyFiles
	fileMappings := vm.Spec.CopyFiles

	if vm.Spec.SSH != nil { //如果指定了ssh，则需要为vm创建ssh密钥对
		pubKeyPath := vm.Spec.SSH.PublicKey
		if vm.Spec.SSH.Generate {
			// generate a key if PublicKey is empty
			pubKeyPath, err = newSSHKeypair(vm)
			if err != nil {
				return
			}
		}

		if len(pubKeyPath) > 0 {
			fileMappings = append(fileMappings, api.FileMapping{
				HostPath: pubKeyPath,
				VMPath:   vmAuthorizedKeys,
			})
		}
	}

	// TODO: File/directory permissions?
	for _, mapping := range fileMappings { //使用拷贝方式处理vm和host的文件映射
		vmFilePath := path.Join(mp.Path, mapping.VMPath)
		if err = os.MkdirAll(path.Dir(vmFilePath), constants.DATA_DIR_PERM); err != nil {
			return
		}

		if err = util.CopyFile(mapping.HostPath, vmFilePath); err != nil {
			return
		}
	}

	ip := net.IP{127, 0, 0, 1}
	if len(vm.Status.Network.IPAddresses) > 0 {
		ip = vm.Status.Network.IPAddresses[0]
	}

	// Write /etc/hosts for the VM //在/etc/hosts中设置本机主机名地址解析
	if err = writeEtcHosts(mp.Path, vm.GetUID().String(), ip); err != nil {
		return
	}

	// Write the UID to /etc/hostname for the VM // 在/etc/hostname中设置本机主机名
	if err = writeEtcHostname(mp.Path, vm.GetUID().String()); err != nil {
		return
	}

	// Populate /etc/fstab with the VM's volume mounts //在/etc/fstab中配置vm.Spec.Storage中定义的卷挂载
	if err = populateFstab(vm, mp.Path); err != nil {
		return
	}

	// Set overlay root permissions
	err = os.Chmod(mp.Path, constants.DATA_DIR_PERM)

	return
}

$ losetup 
NAME       SIZELIMIT OFFSET AUTOCLEAR RO BACK-FILE                                              DIO LOG-SEC
/dev/loop1         0      0         1  1 /var/lib/firecracker/image/669a5721d130ef1d/image.ext4   0     512
/dev/loop2         0      0         1  0 /var/lib/firecracker/vm/ddf49307b5b27c34/overlay.dm      0     512

$ dmsetup create test-snapshot <<EOF
"0 8388608 linear /dev/loop0 0"
"8388608 12582912 zero"
EOF

$ dmsetup create ignite-ddf49307b5b27c34 --table '0 8388608 snapshot /dev/{loop0,mapper/ignite-<uid>-base} /dev/loop1 P 8'

$ dmsetup status
ignite-ddf49307b5b27c34: 0 8388608 snapshot 274328/8388608 1080 #创建出来的snapshot设备
ignite-ddf49307b5b27c34-base: 0 577536 linear      #扩展image loop所创建的devicemapper，映射到image loop设备
ignite-ddf49307b5b27c34-base: 577536 8388608 zero  #用于扩展ignite-ddf49307b5b27c34-base的设备

$ e2fsck -p -f /dev/mapper/<snapshot>

$ losetup -d /dev/loop0
$ losetup -d /dev/loop1

	if vm.Spec.SSH != nil {
		pubKeyPath := vm.Spec.SSH.PublicKey
		if vm.Spec.SSH.Generate {
			// generate a key if PublicKey is empty
			pubKeyPath, err = newSSHKeypair(vm)
			if err != nil {
				return
			}
		}

		if len(pubKeyPath) > 0 {
			fileMappings = append(fileMappings, api.FileMapping{
				HostPath: pubKeyPath,
				VMPath:   vmAuthorizedKeys,
			})
		}
	}

	for _, mapping := range fileMappings {
		vmFilePath := path.Join(mp.Path, mapping.VMPath)
		if err = os.MkdirAll(path.Dir(vmFilePath), constants.DATA_DIR_PERM); err != nil {
			return
		}

		if err = util.CopyFile(mapping.HostPath, vmFilePath); err != nil {
			return
		}
	}

// Generate a new SSH keypair for the vm
func newSSHKeypair(vm *api.VM) (string, error) {
	privKeyPath := path.Join(vm.ObjectPath(), fmt.Sprintf(constants.VM_SSH_KEY_TEMPLATE, vm.GetUID()))
	// TODO: In future versions, let the user specify what key algorithm to use through the API types
	sshKeyAlgorithm := "ed25519"
	if util.FIPSEnabled() {
		// Use rsa on FIPS machines
		sshKeyAlgorithm = "rsa"
	}
	_, err := util.ExecuteCommand("ssh-keygen", "-q", "-t", sshKeyAlgorithm, "-N", "", "-f", privKeyPath)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s.pub", privKeyPath), nil
}

func Start(so *StartOptions, fs *flag.FlagSet) error {
	// Check if the given VM is already running
	if so.vm.Running() {
		return fmt.Errorf("VM %q is already running", so.vm.GetUID())
	}
  
  //下面主要是配置runtime和networkplugin的名称和client
	// Stopped VMs don't contain the runtime and network information. Set the
	// default runtime and network from the providers if empty.
	if so.vm.Status.Runtime.Name == "" {
		so.vm.Status.Runtime.Name = providers.RuntimeName
	}
	if so.vm.Status.Network.Plugin == "" {
		so.vm.Status.Network.Plugin = providers.NetworkPluginName
	}

	// In case the runtime and network-plugin are specified explicitly at
	// start, set the runtime and network-plugin on the VM. This overrides the
	// global config and config on the VM object, if any.
	if fs.Changed("runtime") {
		so.vm.Status.Runtime.Name = providers.RuntimeName
	}
	if fs.Changed("network-plugin") {
		so.vm.Status.Network.Plugin = providers.NetworkPluginName
	}

	// Set the runtime and network-plugin providers from the VM status.
	if err := config.SetAndPopulateProviders(so.vm.Status.Runtime.Name, so.vm.Status.Network.Plugin); err != nil {
		return err
	}

  //有效性校验，主要校验文件的存在性，如依赖的可执行文件，依赖的CNI文件以及内核/dev/kvm、/dev/net/tun、/dev/mapper/control等
	ignoredPreflightErrors := sets.NewString(util.ToLower(so.StartFlags.IgnoredPreflightErrors)...)
	if err := checkers.StartCmdChecks(so.vm, ignoredPreflightErrors); err != nil {
		return err
	}

  //启动vm
	if err := operations.StartVM(so.vm, so.Debug); err != nil {
		return err
	}

  //等待ssh服务就绪
	// When --ssh is enabled, wait until SSH service started on port 22 at most N seconds
	if ssh := so.vm.Spec.SSH; ssh != nil && ssh.Generate && len(so.vm.Status.Network.IPAddresses) > 0 {
		if err := waitForSSH(so.vm, constants.SSH_DEFAULT_TIMEOUT_SECONDS, constants.IGNITE_SPAWN_TIMEOUT); err != nil {
			return err
		}
	}

	// If starting interactively, attach after starting
	if so.Interactive {
		return Attach(so.AttachOptions)
	}
	return nil
}

func StartVM(vm *api.VM, debug bool) error {

	vmChans, err := StartVMNonBlocking(vm, debug)
	if err != nil {
		return err
	}

	if err := <-vmChans.SpawnFinished; err != nil {
		return err
	}

	return nil
}

func StartVMNonBlocking(vm *api.VM, debug bool) (*VMChannels, error) {
	// Inspect the VM container and remove it if it exists
	inspectResult, _ := providers.Runtime.InspectContainer(vm.PrefixedID())
	RemoveVMContainer(inspectResult)

	// Make sure we always initialize all channels
	vmChans := &VMChannels{
		SpawnFinished: make(chan error),
	}

	// Setup the snapshot overlay filesystem
	snapshotDevPath, err := dmlegacy.ActivateSnapshot(vm)
	if err != nil {
		return vmChans, err
	}

	kernelUID, err := lookup.KernelUIDForVM(vm, providers.Client)
	if err != nil {
		return vmChans, err
	}

  //查找vm路径和kernel路径，用于挂载vm元数据和内核vmlinux文件
	vmDir := filepath.Join(constants.VM_DIR, vm.GetUID().String())
	kernelDir := filepath.Join(constants.KERNEL_DIR, kernelUID.String())

	// Verify that the image containing ignite-spawn is pulled
	// TODO: Integrate automatic pulling into pkg/runtime
  //校验基础镜像和内核镜像是否存在，不存在则重新拉取
	if err := verifyPulled(vm.Spec.Sandbox.OCI); err != nil {
		return vmChans, err
	}

  //设置挂载的卷，主要是/var/lib/firecracker/vm/<UID>/目录和该目录下的vmlinux文件，以及/dev下的一些设备
	config := &runtime.ContainerConfig{
		Cmd: []string{
			fmt.Sprintf("--log-level=%s", logs.Logger.Level.String()),
			vm.GetUID().String(),
		},
		Labels: map[string]string{"ignite.name": vm.GetName()},
		Binds: []*runtime.Bind{
			{
				HostPath:      vmDir,
				ContainerPath: vmDir,
			},
			{
				// Mount the metadata.json file specifically into the container, to a well-known place for ignite-spawn to access
				HostPath:      path.Join(vmDir, constants.METADATA),
				ContainerPath: constants.IGNITE_SPAWN_VM_FILE_PATH,
			},
			{
				// Mount the vmlinux file specifically into the container, to a well-known place for ignite-spawn to access
				HostPath:      path.Join(kernelDir, constants.KERNEL_FILE),
				ContainerPath: constants.IGNITE_SPAWN_VMLINUX_FILE_PATH,
			},
		},
		CapAdds: []string{
			"SYS_ADMIN", // Needed to run "dmsetup remove" inside the container
			"NET_ADMIN", // Needed for removing the IP from the container's interface
		},
		Devices: []*runtime.Bind{
			runtime.BindBoth("/dev/mapper/control"), // This enables containerized Ignite to remove its own dm snapshot
			runtime.BindBoth("/dev/net/tun"),        // Needed for creating TAP adapters
			runtime.BindBoth("/dev/kvm"),            // Pass through virtualization support
			runtime.BindBoth(snapshotDevPath),       // The block device to boot from
		},
		StopTimeout:  constants.STOP_TIMEOUT + constants.IGNITE_TIMEOUT,
		PortBindings: vm.Spec.Network.Ports, // Add the port mappings to Docker
	}

  // 配置环境变量
	var envVars []string
	for k, v := range vm.GetObjectMeta().Annotations {
		if strings.HasPrefix(k, constants.IGNITE_SANDBOX_ENV_VAR) {
			k := strings.TrimPrefix(k, constants.IGNITE_SANDBOX_ENV_VAR)
			envVars = append(envVars, fmt.Sprintf("%s=%s", k, v))
		}
	}
	config.EnvVars = envVars

	// 添加自定义挂载
	for _, volume := range vm.Spec.Storage.Volumes {
		if volume.BlockDevice == nil {
			continue // Skip all non block device volumes for now
		}

		config.Devices = append(config.Devices, &runtime.Bind{
			HostPath:      volume.BlockDevice.Path,
			ContainerPath: path.Join(constants.IGNITE_SPAWN_VOLUME_DIR, volume.Name),
		})
	}

	// Prepare the networking for the container, for the given network plugin
	if err := providers.NetworkPlugin.PrepareContainerSpec(config); err != nil {
		return vmChans, err
	}

	// If we're not debugging, remove the container post-run
	if !debug {
		config.AutoRemove = true
	}

	// Run the VM container in Docker
	containerID, err := providers.Runtime.RunContainer(vm.Spec.Sandbox.OCI, config, vm.PrefixedID(), vm.GetUID().String())
	if err != nil {
		return vmChans, fmt.Errorf("failed to start container for VM %q: %v", vm.GetUID(), err)
	}

	// 配置CNI网络
	result, err := providers.NetworkPlugin.SetupContainerNetwork(containerID, vm.Spec.Network.Ports...)
	if err != nil {
		return vmChans, err
	}

	if !logs.Quiet {
		log.Infof("Networking is handled by %q", providers.NetworkPlugin.Name())
		log.Infof("Started Firecracker VM %q in a container with ID %q", vm.GetUID(), containerID)
	}

	// Set the container ID for the VM
	vm.Status.Runtime.ID = containerID
	vm.Status.Runtime.Name = providers.RuntimeName

	// Append non-loopback runtime IP addresses of the VM to its state
	for _, addr := range result.Addresses {
		if !addr.IP.IsLoopback() {
			vm.Status.Network.IPAddresses = append(vm.Status.Network.IPAddresses, addr.IP)
		}
	}
	vm.Status.Network.Plugin = providers.NetworkPluginName

	// write the API object in a non-running state before we wait for spawn's network logic and firecracker
	if err := providers.Client.VMs().Set(vm); err != nil {
		return vmChans, err
	}

	// TODO: This is temporary until we have proper communication to the container
	// It's best to perform any imperative changes to the VM object pointer before this go-routine starts
	go waitForSpawn(vm, vmChans)

	return vmChans, nil
}

func (cc *ctdClient) RunContainer(image meta.OCIImageRef, config *runtime.ContainerConfig, name, id string) (s string, err error) {
	img, err := cc.client.GetImage(cc.ctx, image.Normalized())
	if err != nil {
		return
	}

	// Remove the container if it exists
	if err = cc.RemoveContainer(name); err != nil {
		return
	}

	// Load the default snapshotter
	snapshotter := cc.client.SnapshotService(containerd.DefaultSnapshotter)

	// Add the /etc/resolv.conf mount, this isn't done automatically by containerd
	// Ensure a resolv.conf exists in the vmDir. Calculate path using the vm id
	resolvConfPath := filepath.Join(constants.VM_DIR, id, resolvConfName)
  //读取主机的/etc/resolv.conf并写入vm目录的runtime.containerd.resolv.conf中
	err = resolvconf.EnsureResolvConf(resolvConfPath, constants.DATA_DIR_FILE_PERM) 
	if err != nil {
		return
	}
	config.Binds = append(
		config.Binds,
		&runtime.Bind{
			HostPath:      resolvConfPath,
			ContainerPath: "/etc/resolv.conf", //将runtime.containerd.resolv.conf挂载到容器中
		},
	)

	// Add the stop timeout as a label, as containerd doesn't natively support it
	config.Labels[stopTimeoutLabel] = strconv.FormatUint(uint64(config.StopTimeout), 10)

	// Build the OCI specification
	opts := []oci.SpecOpts{
		oci.WithDefaultSpec(),
		oci.WithDefaultUnixDevices,
		oci.WithTTY,
		oci.WithImageConfigArgs(img, config.Cmd),
		oci.WithEnv(config.EnvVars),
		withAddedCaps(config.CapAdds),
		withHostname(config.Hostname),
		withMounts(config.Binds), //挂载卷
		withDevices(config.Devices), //挂载设备
	}

	// Known limitations, containerd doesn't support the following config fields:
	// - StopTimeout
	// - AutoRemove
	// - NetworkMode (only CNI supported)
	// - PortBindings

	snapshotOpt := containerd.WithSnapshot(name)
	if _, err = snapshotter.Stat(cc.ctx, name); errdefs.IsNotFound(err) {
		// Even if "read only" is set, we don't use a KindView snapshot here (#1495).
		// We pass the writable snapshot to the OCI runtime, and the runtime remounts
		// it as read-only after creating some mount points on-demand.
		snapshotOpt = containerd.WithNewSnapshot(name, img)
	} else if err != nil {
		return
	}

	cOpts := []containerd.NewContainerOpts{
		containerd.WithImage(img),
		snapshotOpt,
		//containerd.WithImageStopSignal(img, "SIGTERM"),
		containerd.WithNewSpec(opts...),
		containerd.WithContainerLabels(config.Labels),
	}

	cont, err := cc.client.NewContainer(cc.ctx, name, cOpts...)
	if err != nil {
		return
	}

	// This is a dummy PTY to silence output
	// when starting without attach breaking
	con, _, err := console.NewPty()
	if err != nil {
		return
	}
	defer util.DeferErr(&err, con.Close)

	// We need a temporary dummy stdin reader that
	// actually works, can't use nullReader here
	dummyReader, _, err := os.Pipe()
	if err != nil {
		return
	}
	defer util.DeferErr(&err, dummyReader.Close)

	// Spawn the Creator with the dummy streams
	ioCreator := cio.NewCreator(cio.WithTerminal, cio.WithStreams(dummyReader, con, con))

	task, err := cont.NewTask(cc.ctx, ioCreator)
	if err != nil {
		return
	}

	if err = task.Start(cc.ctx); err != nil {
		return
	}

	// TODO: Save task.Pid() somewhere for attaching?
	s = task.ID()
	return
}

ENTRYPOINT ["/usr/local/bin/ignite-spawn"]

func decodeVM(vmID string) (*api.VM, error) {
	filePath := constants.IGNITE_SPAWN_VM_FILE_PATH
	obj, err := scheme.Serializer.DecodeFile(filePath, true)
	if err != nil {
		return nil, err
	}

	vm, ok := obj.(*api.VM)
	if !ok {
		return nil, fmt.Errorf("object couldn't be converted to VM")
	}

	// Explicitly set the GVK on this object
	vm.SetGroupVersionKind(api.SchemeGroupVersion.WithKind(api.KindVM.Title()))
	return vm, nil
}

func StartVM(vm *api.VM) (err error) {

	// Setup networking inside of the container, return the available interfaces
	fcIfaces, dhcpIfaces, err := container.SetupContainerNetworking(vm)//配置容器网络
	if err != nil {
		return fmt.Errorf("network setup failed: %v", err)
	}

	// Serve DHCP requests for those interfaces
	// This function returns the available IP addresses that are being
	// served over DHCP now
	if err = container.StartDHCPServers(vm, dhcpIfaces); err != nil { //配置DHCP
		return
	}

	// Serve metrics over an unix socket in the VM's own directory
	metricsSocket := path.Join(vm.ObjectPath(), constants.PROMETHEUS_SOCKET)
	serveMetrics(metricsSocket)

	// Patches the VM object to set state to stopped, and clear IP addresses
	defer util.DeferErr(&err, func() error { return patchStopped(vm) })

	// Remove the snapshot overlay post-run, which also removes the detached backing loop devices
	defer util.DeferErr(&err, func() error { return dmlegacy.DeactivateSnapshot(vm) })

	// Remove the Prometheus socket post-run
	defer util.DeferErr(&err, func() error { return os.Remove(metricsSocket) })

	// Execute Firecracker
	if err = container.ExecuteFirecracker(vm, fcIfaces); err != nil { //启动vm
		return fmt.Errorf("runtime error for VM %q: %v", vm.GetUID(), err)
	}

	return
}

func SetupContainerNetworking(vm *api.VM) (firecracker.NetworkInterfaces, []DHCPInterface, error) {
   var dhcpIntfs []DHCPInterface
   var fcIntfs firecracker.NetworkInterfaces

   //通过vm的metadata.json的annotation ignite.weave.works/interface可以添加额外的接口
  vmIntfs := parseExtraIntfs(vm) //vmIntfs: map[<interface_name>][interface_mode]

   // 如果没有eth0接口，则添加该接口，并设置为dhcp模式
   if _, ok := vmIntfs[mainInterface]; !ok {
      vmIntfs[mainInterface] = MODE_DHCP
   }

   interval := 1 * time.Second

  //等待接口就绪，就绪则返回true
   err := wait.PollImmediate(interval, constants.IGNITE_SPAWN_TIMEOUT, func() (bool, error) {

      // 检查接口是否存在且配置正确
      retry, err := collectInterfaces(vmIntfs)

      if err == nil {
         // We're done here
         return true, nil
      }
      if retry {
         // We got an error, but let's ignore it and try again
         log.Warnf("Got an error while trying to set up networking, but retrying: %v", err)
         return false, nil
      }
      // The error was fatal, return it
      return false, err
   })

   if err != nil {
      return nil, nil, err
   }

   //为vm准备接口等网络环境
   if err := networkSetup(&fcIntfs, &dhcpIntfs, vmIntfs); err != nil {
      return nil, nil, err
   }

   return fcIntfs, dhcpIntfs, nil
}

func collectInterfaces(vmIntfs map[string]string) (bool, error) {
  //获取所有接口
   allIntfs, err := net.Interfaces()
   if err != nil || allIntfs == nil || len(allIntfs) == 0 {
      return false, fmt.Errorf("cannot get local network interfaces: %v", err)
   }

   // create a map of candidate interfaces
   foundIntfs := make(map[string]net.Interface)
   for _, intf := range allIntfs {
      if _, ok := ignoreInterfaces[intf.Name]; ok {
         continue
      }

      foundIntfs[intf.Name] = intf

      // If the interface is explicitly defined, no changes are needed
      if _, ok := vmIntfs[intf.Name]; ok { //如果已经定义接口，则无需再为接口配置mode
         continue
      }

      // default fallback behaviour to always consider intfs with an address
      addrs, _ := intf.Addrs()
      if len(addrs) > 0 {
         vmIntfs[intf.Name] = MODE_DHCP
      }
   }

   // 校验是否已经创建期望的接口
   for intfName, mode := range vmIntfs {
      if _, ok := foundIntfs[intfName]; !ok {
         return true, fmt.Errorf("interface %q (mode %q) is still not found", intfName, mode)
      }

      // for DHCP interface, we need to make sure IP and route exist
      if mode == MODE_DHCP {
         intf := foundIntfs[intfName]
        _, _, _, noIPs, err := getAddress(&intf) //返回接口的IP/掩码、网关和物理接口(link)，这里判断是否接口配置了IP
         if err != nil {
            return true, err
         }

         if noIPs {
            return true, fmt.Errorf("IP is still not found on %q", intfName)
         }
      }
   }
   return false, nil
}

func networkSetup(fcIntfs *firecracker.NetworkInterfaces, dhcpIntfs *[]DHCPInterface, vmIntfs map[string]string) error {

	// The order in which interfaces are plugged in is intentionally deterministic
	// All interfaces are sorted alphabetically and 'eth0' is always first
	var keys []string
	for k := range vmIntfs {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	sort.Slice(keys, func(i, j int) bool {
		return keys[i] == mainInterface
	})

	for _, intfName := range keys {

		intf, err := net.InterfaceByName(intfName) //根据接口名称获取容器的接口实例
		if err != nil {
			return fmt.Errorf("cannot find interface %q: %s", intfName, err)
		}

		switch vmIntfs[intfName] {
		case MODE_DHCP:
			ipNet, gw, err := takeAddress(intf) //获取容器接口的第一个地址信息，并从容器接口上删除该地址信息，然后返回该信息，后续作为vm的接口地址
			if err != nil {
				return fmt.Errorf("error parsing interface %q: %s", intfName, err)
			}

			dhcpIface, err := bridge(intf) //创建tab和bridge接口，并配置桥接。返回给vm使用的接口dhcpIface
			if err != nil {
				return fmt.Errorf("bridging interface %q failed: %v", intfName, err)
			}

			dhcpIface.VMIPNet = ipNet
			dhcpIface.GatewayIP = gw

			*dhcpIntfs = append(*dhcpIntfs, *dhcpIface) //添加dhcp接口

			*fcIntfs = append(*fcIntfs, firecracker.NetworkInterface{
				StaticConfiguration: &firecracker.StaticNetworkConfiguration{
					MacAddress:  dhcpIface.MACFilter,
					HostDevName: dhcpIface.VMTAP,
				},
			})
		case MODE_TC:
			tcInterface, err := addTcRedirect(intf)
			if err != nil {
				log.Errorf("Failed to setup tc redirect %v", err)
				continue
			}

			*fcIntfs = append(*fcIntfs, *tcInterface)
		}
	}

	return nil
}

$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
3: eth0@if14: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br_eth0 state UP group default 
    link/ether 8e:c9:3a:f0:50:67 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet6 fe80::8cc9:3aff:fef0:5067/64 scope link 
       valid_lft forever preferred_lft foreve
4: vm_eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master br_eth0 state UP group default qlen 1000
    link/ether d6:20:6b:d4:3e:2a brd ff:ff:ff:ff:ff:ff
    inet6 fe80::d420:6bff:fed4:3e2a/64 scope link 
       valid_lft forever preferred_lft forever
5: br_eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 92:79:7d:39:28:b6 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::9079:7dff:fe39:28b6/64 scope link 
       valid_lft forever preferred_lft forever

$ ip link show master br_eth0
3: eth0@if14: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br_eth0 state UP mode DEFAULT group default 
    link/ether 8e:c9:3a:f0:50:67 brd ff:ff:ff:ff:ff:ff link-netnsid 0
4: vm_eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master br_eth0 state UP mode DEFAULT group default qlen 1000
    link/ether d6:20:6b:d4:3e:2a brd ff:ff:ff:ff:ff:ff

$ ip a 
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: bond0: <BROADCAST,MULTICAST,MASTER> mtu 1500 qdisc noqueue state DOWN group default qlen 1000
    link/ether 1e:7d:6c:90:99:18 brd ff:ff:ff:ff:ff:ff
3: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether e6:18:3d:88:a9:ff brd ff:ff:ff:ff:ff:ff
4: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 76:6f:77:5d:6d:1c brd ff:ff:ff:ff:ff:ff
    inet 10.61.0.41/16 brd 10.61.255.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::746f:77ff:fe5d:6d1c/64 scope link 
       valid_lft forever preferred_lft forever
$ ip route
default via 10.61.0.1 dev eth0 

// StartDHCPServers starts multiple DHCP servers for the VM, one per interface
// It returns the IP addresses that the API object may post in .status, and a potential error
func StartDHCPServers(vm *api.VM, dhcpIfaces []DHCPInterface) error {

	// Fetch the DNS servers given to the container
	clientConfig, err := dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		return fmt.Errorf("failed to get DNS configuration: %v", err)
	}

	for i := range dhcpIfaces {
		dhcpIface := &dhcpIfaces[i]
		// Set the VM hostname to the VM ID
		dhcpIface.Hostname = vm.GetUID().String()

		// Add the DNS servers from the container
		dhcpIface.SetDNSServers(clientConfig.Servers)

		go func() {
			log.Infof("Starting DHCP server for interface %q (%s)\n", dhcpIface.Bridge, dhcpIface.VMIPNet.IP)
			if err := dhcpIface.StartBlockingServer(); err != nil {
				log.Errorf("%q DHCP server error: %v\n", dhcpIface.Bridge, err)
			}
		}()
	}

	return nil
}

func ExecuteFirecracker(vm *api.VM, fcIfaces firecracker.NetworkInterfaces) (err error) {
	drivePath := vm.SnapshotDev() //获取vm的devicemapper设备，由于容器中挂载了host的/dev，因此可以直接查看使用

	vCPUCount := int64(vm.Spec.CPUs) //获取CPU和内存资源
	memSizeMib := int64(vm.Spec.Memory.MBytes())

	cmdLine := vm.Spec.Kernel.CmdLine
	if len(cmdLine) == 0 {
		// if for some reason cmdline would be unpopulated, set it to the default
		cmdLine = constants.VM_DEFAULT_KERNEL_ARGS
	}

	// Convert the logrus error level to a Firecracker compatible error level.
	// Firecracker accepts "Error", "Warning", "Info", and "Debug", case-sensitive.
	fcLogLevel := "Debug"
	switch logs.Logger.Level {
	case log.InfoLevel:
		fcLogLevel = "Info"
	case log.WarnLevel:
		fcLogLevel = "Warning"
	case log.ErrorLevel, log.FatalLevel, log.PanicLevel:
		fcLogLevel = "Error"
	}

	firecrackerSocketPath := path.Join(vm.ObjectPath(), constants.FIRECRACKER_API_SOCKET)
	logSocketPath := path.Join(vm.ObjectPath(), constants.LOG_FIFO)
	metricsSocketPath := path.Join(vm.ObjectPath(), constants.METRICS_FIFO)
	cfg := firecracker.Config{
		SocketPath:      firecrackerSocketPath,
		KernelImagePath: constants.IGNITE_SPAWN_VMLINUX_FILE_PATH, //挂载到容器中的vmlinux文件路径
		KernelArgs:      cmdLine,
		Drives: []models.Drive{{
			DriveID:      firecracker.String("1"),
			IsReadOnly:   firecracker.Bool(false),
			IsRootDevice: firecracker.Bool(true),
			PathOnHost:   &drivePath, //设置devicemapper设备
		}},
		NetworkInterfaces: fcIfaces, //设置vm接口
		MachineCfg: models.MachineConfiguration{
			VcpuCount:  &vCPUCount,
			MemSizeMib: &memSizeMib,
			HtEnabled:  firecracker.Bool(true),
		},
		//JailerCfg: firecracker.JailerConfig{
		//	GID:      firecracker.Int(0),
		//	UID:      firecracker.Int(0),
		//	ID:       vm.ID,
		//	NumaNode: firecracker.Int(0),
		//	ExecFile: "firecracker",
		//},

		LogLevel: fcLogLevel,
		// TODO: We could use /dev/null, but firecracker-go-sdk issues Mkfifo which collides with the existing device
		LogFifo:     logSocketPath,
		MetricsFifo: metricsSocketPath,
	}

	// Add the volumes to the VM
	for i, volume := range vm.Spec.Storage.Volumes {
		volumePath := path.Join(constants.IGNITE_SPAWN_VOLUME_DIR, volume.Name)
		if !util.FileExists(volumePath) {
			log.Warnf("Skipping nonexistent volume: %q", volume.Name)
			continue // Skip all nonexistent volumes
		}

		cfg.Drives = append(cfg.Drives, models.Drive{
			DriveID:      firecracker.String(strconv.Itoa(i + 2)),
			IsReadOnly:   firecracker.Bool(false), // TODO: Support read-only volumes
			IsRootDevice: firecracker.Bool(false),
			PathOnHost:   &volumePath, //设置挂载卷，这部分是从host-->container-->vm
		})
	}

	// Remove these FIFOs for now
	defer os.Remove(logSocketPath)
	defer os.Remove(metricsSocketPath)

	ctx, vmmCancel := context.WithCancel(context.Background())
	defer vmmCancel()

	cmd := firecracker.VMCommandBuilder{}.
		WithBin("firecracker").
		WithSocketPath(firecrackerSocketPath).
		WithStdin(os.Stdin).
		WithStdout(os.Stdout).
		WithStderr(os.Stderr).
		Build(ctx)

	m, err := firecracker.NewMachine(ctx, cfg, firecracker.WithProcessRunner(cmd))
	if err != nil {
		return fmt.Errorf("failed to create machine: %s", err)
	}

	//defer os.Remove(cfg.SocketPath)

	//if opts.validMetadata != nil {
	//	m.EnableMetadata(opts.validMetadata)
	//}

	if err = m.Start(ctx); err != nil { //启动vm
		return fmt.Errorf("failed to start machine: %v", err)
	}
	defer util.DeferErr(&err, m.StopVMM)

	installSignalHandlers(ctx, m)

	// wait for the VMM to exit
	if err = m.Wait(ctx); err != nil {
		return fmt.Errorf("wait returned an error %s", err)
	}

	return
}

func Run(ro *RunOptions, fs *flag.FlagSet) error {
   if err := Create(ro.CreateOptions); err != nil {
      return err
   }

   // Copy the pointer over for Start
   // TODO: This is pretty bad, fix this
   ro.vm = ro.VM

   return Start(ro.StartOptions, fs)
}

func StopVM(vm *api.VM, kill, silent bool) error {
	var err error
	container := vm.PrefixedID()
	action := "stop"

	if !vm.Running() && !logs.Quiet {
		log.Warnf("VM %q is not running but trying to cleanup networking for stopped container\n", vm.GetUID())
	}

	// 释放网络资源
	if err = removeNetworking(vm.Status.Runtime.ID, vm.Spec.Network.Ports...); err != nil {
		log.Warnf("Failed to cleanup networking for stopped container %s %q: %v", vm.GetKind(), vm.GetUID(), err)

		return err
	}

	if vm.Running() {
		// Stop or kill the VM container
		if kill {
			action = "kill"
			err = providers.Runtime.KillContainer(container, signalSIGQUIT) // TODO: common constant for SIGQUIT
		} else {
			err = providers.Runtime.StopContainer(container, nil)
		}

		if err != nil {
			return fmt.Errorf("failed to %s container for %s %q: %v", action, vm.GetKind(), vm.GetUID(), err)
		}

		if silent {
			return nil
		}

		if logs.Quiet {
			fmt.Println(vm.GetUID())
		} else {
			log.Infof("Stopped %s with name %q and ID %q", vm.GetKind(), vm.GetName(), vm.GetUID())
		}
	}

	return nil
}

func (cc *ctdClient) KillContainer(container, signal string) (err error) {
	cont, err := cc.client.LoadContainer(cc.ctx, container)
	if err != nil {
		// If the container is not found, return nil, no-op.
		if errdefs.IsNotFound(err) {
			log.Warn(err)
			err = nil
		}
		return
	}

	task, err := cont.Task(cc.ctx, cio.Load)
	if err != nil {
		// If the task is not found, return nil, no-op.
		if errdefs.IsNotFound(err) {
			log.Warn(err)
			err = nil
		}
		return
	}

	// Initiate a wait
	waitC, err := task.Wait(cc.ctx)
	if err != nil {
		return
	}

	// Send a SIGQUIT signal to force stop
	if err = task.Kill(cc.ctx, syscall.SIGQUIT); err != nil {
		return
	}

	// Wait for the container to stop
	<-waitC

	// Delete the task
	_, err = task.Delete(cc.ctx)
	return
}

	waitC, err := task.Wait(cc.ctx)
	if err != nil {
		return
	}

	// Send a SIGTERM signal to request a clean shutdown
	if err = task.Kill(cc.ctx, syscall.SIGTERM); err != nil {
		return
	}

	// After sending the signal, start the timer to force-kill the task
	timeoutC := make(chan error)
	timer := time.AfterFunc(*timeout, func() {
		timeoutC <- task.Kill(cc.ctx, syscall.SIGQUIT)
	})

	// Wait for the task to stop or the timer to fire
	select {
	case exitStatus := <-waitC:
		timer.Stop()             // Cancel the force-kill timer
		err = exitStatus.Error() // TODO: Handle exit code
	case err = <-timeoutC: // The kill timer has fired
	}

	// Delete the task
	if _, e := task.Delete(cc.ctx); e != nil {
		if err != nil {
			err = fmt.Errorf("%v, task deletion failed: %v", err, e) // TODO: Multierror
		} else {
			err = e
		}
	}

func Rm(ro *RmOptions) error {
	for _, vm := range ro.vms {
		// 如果vm是运行状态，则需要指定强制删除才能继续删除，这与docker命令行删除一个运行的容器一样
		if vm.Running() && !ro.Force {
			return fmt.Errorf("%s is running", vm.GetUID())
		}

		// Runtime and network info are present only when the VM is running.
		if vm.Running() {
			// Set the runtime and network-plugin providers from the VM status.
			if err := config.SetAndPopulateProviders(vm.Status.Runtime.Name, vm.Status.Network.Plugin); err != nil {
				return err
			}
		}

		// This will first kill the VM container, and then remove it
		if err := operations.DeleteVM(providers.Client, vm); err != nil {
			return err
		}
	}

	return nil
}

func DeleteVM(c *client.Client, vm *api.VM) error {
	if err := CleanupVM(vm); err != nil {
		return err
	}

  //清除vm对象以及/var/lib/firecracker/vm/<UID>/目录
	return c.VMs().Delete(vm.GetUID())
}

func CleanupVM(vm *api.VM) error {
	// Runtime information is available only when the VM is running.
	if vm.Running() {
		// Inspect the container before trying to stop it and it gets auto-removed
		inspectResult, _ := providers.Runtime.InspectContainer(vm.PrefixedID())

		// If the VM is running, try to kill it first so we don't leave dangling containers. Otherwise, try to cleanup VM networking.
		if err := StopVM(vm, true, true); err != nil {
			if vm.Running() {
				return err
			}
		}

		// Remove the VM container if it exists
		// TODO should this function return a proper error?
		RemoveVMContainer(inspectResult)
	}

	// After removing the VM container, if the Snapshot Device is still there, clean up
	if _, err := os.Stat(vm.SnapshotDev()); err == nil {
		// try remove it again with DeactivateSnapshot
		if err := dmlegacy.DeactivateSnapshot(vm); err != nil {
			return err
		}
	}

	if logs.Quiet {
		fmt.Println(vm.GetUID())
	} else {
		log.Infof("Removed %s with name %q and ID %q", vm.GetKind(), vm.GetName(), vm.GetUID())
	}

	return nil
}

func (cc *ctdClient) RemoveContainer(container string) error {
	// Remove the container if it exists
	cont, contLoadErr := cc.client.LoadContainer(cc.ctx, container)
	if errdefs.IsNotFound(contLoadErr) {
		log.Debug(contLoadErr)
		return nil
	} else if contLoadErr != nil {
		return contLoadErr
	}

	// Load the container's task without attaching
	task, taskLoadErr := cont.Task(cc.ctx, nil)
	if errdefs.IsNotFound(taskLoadErr) {
		log.Debug(taskLoadErr)
	} else if taskLoadErr != nil {
		return taskLoadErr
	} else {
		_, taskDeleteErr := task.Delete(cc.ctx)
		if taskDeleteErr != nil {
			log.Debug(taskDeleteErr)
		}
	}

	// Delete the container
	deleteContErr := cont.Delete(cc.ctx, containerd.WithSnapshotCleanup)
	if errdefs.IsNotFound(contLoadErr) {
		log.Debug(contLoadErr)
	} else if deleteContErr != nil {
		return deleteContErr
	}

	// Remove the log file if it exists
	logFile := fmt.Sprintf(logPathTemplate, container)
	if util.FileExists(logFile) {
		logDeleteErr := os.RemoveAll(logFile)
		if logDeleteErr != nil {
			return logDeleteErr
		}
	}

	return nil
}

func (cc *ctdClient) ContainerLogs(container string) (r io.ReadCloser, err error) {
	var (
		cont containerd.Container
	)

	if cont, err = cc.client.LoadContainer(cc.ctx, container); err != nil {
		return
	}

	var retriever *logRetriever
	if retriever, err = newlogRetriever(fmt.Sprintf(logPathTemplate, container)); err != nil {
		return
	}

	if _, err = cont.Task(cc.ctx, cio.NewAttach(retriever.Opt())); err != nil {
		return
	}

	// Currently we have no way of detecting if the task's attach has filled the stdout and stderr
	// buffers without asynchronous I/O (syscall.Conn and syscall.Splice). If the read reaches
	// the end, the application hangs indefinitely waiting for new output from the container.
	// TODO: Get rid of this, implement asynchronous I/O and read until the streams have been exhausted
	time.Sleep(time.Second)

	// Close the writer to signal EOF
	if err = retriever.CloseWriter(); err != nil {
		return
	}

	return retriever, nil
}

func (cc *ctdClient) AttachContainer(container string) (err error) {
	var (
		cont containerd.Container
		spec *oci.Spec
	)

	if cont, err = cc.client.LoadContainer(cc.ctx, container); err != nil {
		return
	}

	if spec, err = cont.Spec(cc.ctx); err != nil {
		return
	}

	var (
		con console.Console
		tty = spec.Process.Terminal
	)

	if tty {
		con = console.Current() //获取当前的终端
		defer util.DeferErr(&err, con.Reset)
		if err = con.SetRaw(); err != nil {
			return
		}
	}

	var (
		task     containerd.Task
		statusC  <-chan containerd.ExitStatus
		igniteIO *igniteIO
	)

	if igniteIO, err = newIgniteIO(fmt.Sprintf(logPathTemplate, container)); err != nil {
		return
	}
	defer util.DeferErr(&err, igniteIO.Close)

	if task, err = cont.Task(cc.ctx, cio.NewAttach(igniteIO.Opt())); err != nil { //配置日志相关的输出
		return
	}

	if statusC, err = task.Wait(cc.ctx); err != nil {
		return
	}

	if tty {
		if err := HandleConsoleResize(cc.ctx, task, con); err != nil {
			log.Errorf("console resize failed: %v", err)
		}
	} else {
		sigc := ForwardAllSignals(cc.ctx, task)
		defer StopCatch(sigc)
	}

	var code uint32
	select {
	case ec := <-statusC:
		code, _, err = ec.Result()
	case <-igniteIO.Detach():
		fmt.Println() // Use a new line for the log entry
		log.Println("Detached")
	}

	if code != 0 && err == nil {
		err = fmt.Errorf("attach exited with code %d", code)
	}

	return
}

$ ignite ssh my-vm

// runSSH creates and runs ssh session based on the provided arguments.
// If the command list is empty, ssh shell is created, else the ssh command is
// executed.
func runSSH(vm *api.VM, privKeyFile string, command []string, tty bool, timeout uint32) (err error) {
	// Check if the VM is running.
	if !vm.Running() {
		return fmt.Errorf("VM %q is not running", vm.GetUID())
	}

	// Get the IP address.
	ipAddrs := vm.Status.Network.IPAddresses //获取ssh连接的ip地址
	if len(ipAddrs) == 0 {
		return fmt.Errorf("VM %q has no usable IP addresses", vm.GetUID())
	}

	// Get private key file path.
	if len(privKeyFile) == 0 { //获取本地私钥
		privKeyFile = path.Join(vm.ObjectPath(), fmt.Sprintf(constants.VM_SSH_KEY_TEMPLATE, vm.GetUID()))
		if !util.FileExists(privKeyFile) {
			return fmt.Errorf("no private key found for VM %q", vm.GetUID())
		}
	}

	// Create a new ssh signer for the private key.
	signer, err := newSignerForKey(privKeyFile)
	if err != nil {
		return fmt.Errorf("unable to create signer for private key: %v", err)
	}

	// Defer exit here and set the exit code based on any ssh error, so that
	// this ssh command returns the correct ssh exit code. Since this function
	// results in an os.Exit, any error returned by this function won't be
	// received by the caller. Print the error to make the errror message
	// visible and set the error code when an error is found.
	exitCode := 0
	defer func() {
		os.Exit(exitCode)
	}()

	// printErrAndSetExitCode is used to print an error message, set exit code
	// and return nil. This is needed because once the ssh connection is
	// estabilish, to return the error code of the actual ssh session, instead
	// of returning an error, the runSSH function defers os.Exit with the ssh
	// exit code. For showing any error to the user, it needs to be printed.
	printErrAndSetExitCode := func(errMsg error, exitCode *int, code int) error {
		log.Errorf("%v\n", errMsg)
		*exitCode = code
		return nil
	}

	// Create an SSH client, and connect.
	config := newSSHConfig(signer, timeout)
	client, err := ssh.Dial(defaultSSHNetwork, net.JoinHostPort(ipAddrs[0].String(), defaultSSHPort), config)
	if err != nil {
		return printErrAndSetExitCode(fmt.Errorf("failed to dial: %v", err), &exitCode, 1)
	}
	defer util.DeferErr(&err, client.Close)

	// Create a session.
	session, err := client.NewSession()
	if err != nil {
		return printErrAndSetExitCode(fmt.Errorf("failed to create session: %v", err), &exitCode, 1)
	}
	defer util.DeferErr(&err, session.Close)

	// Configure tty if requested.
	if tty {
		// Get stdin file descriptor reference.
		fd := int(os.Stdin.Fd())

		// Store the raw state of the terminal.
		state, err := terminal.MakeRaw(fd)
		if err != nil {
			return printErrAndSetExitCode(fmt.Errorf("failed to make terminal raw: %v", err), &exitCode, 1)
		}
		defer util.DeferErr(&err, func() error { return terminal.Restore(fd, state) })

		// Get the terminal dimensions.
		w, h, err := terminal.GetSize(fd)
		if err != nil {
			return printErrAndSetExitCode(fmt.Errorf("failed to get terminal size: %v", err), &exitCode, 1)
		}

		// Set terminal modes.
		modes := ssh.TerminalModes{
			ssh.ECHO: 1,
		}

		// Read the TERM environment variable and use it to request the PTY.
		term := os.Getenv("TERM")
		if term == "" {
			term = defaultTerm
		}

		if err = session.RequestPty(term, h, w, modes); err != nil {
			return printErrAndSetExitCode(fmt.Errorf("request for pseudo terminal failed: %v", err), &exitCode, 1)
		}
	}

	// Connect input / output.
	// TODO: these should come from the cobra command instead of hardcoding
	// os.Stderr etc.
	session.Stderr = os.Stderr
	session.Stdout = os.Stdout
	session.Stdin = os.Stdin

	if len(command) == 0 {
		if err = session.Shell(); err != nil {
			return printErrAndSetExitCode(fmt.Errorf("failed to start shell: %v", err), &exitCode, 1)
		}

		if err = session.Wait(); err != nil {
			if e, ok := err.(*ssh.ExitError); ok {
				return printErrAndSetExitCode(err, &exitCode, e.ExitStatus())
			}
			return printErrAndSetExitCode(fmt.Errorf("failed waiting for session to exit: %v", err), &exitCode, 1)
		}
	} else {
		if err = session.Run(joinShellCommand(command)); err != nil {
			if e, ok := err.(*ssh.ExitError); ok {
				return printErrAndSetExitCode(err, &exitCode, e.ExitStatus())
			}
			return printErrAndSetExitCode(fmt.Errorf("failed to run shell command: %s", err), &exitCode, 1)
		}
	}
	return
}

func newSSHConfig(publicKey ssh.Signer, timeout uint32) *ssh.ClientConfig {
   return &ssh.ClientConfig{
      User: "root",
      Auth: []ssh.AuthMethod{
         ssh.PublicKeys(publicKey),
      },
      HostKeyCallback: ssh.InsecureIgnoreHostKey(), // TODO: use ssh.FixedPublicKey instead
      Timeout:         time.Second * time.Duration(timeout),
   }
}

func Exec(eo *ExecOptions) error {
	if err := waitForSSH(eo.vm, constants.SSH_DEFAULT_TIMEOUT_SECONDS, time.Duration(eo.Timeout)*time.Second); err != nil {
		return err
	}
	return runSSH(eo.vm, eo.IdentityFile, eo.command, eo.Tty, eo.Timeout)
}

func waitForSSH(vm *ignite.VM, dialSeconds int, sshTimeout time.Duration) error {
	if err := dialSuccess(vm, dialSeconds); err != nil { //验证ssh服务是否可达
		return err
	}

	certCheck := &ssh.CertChecker{
		IsHostAuthority: func(auth ssh.PublicKey, address string) bool {
			return true
		},
		IsRevoked: func(cert *ssh.Certificate) bool {
			return false
		},
		HostKeyFallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
			return nil
		},
	}

	config := &ssh.ClientConfig{ //配置无认证方式登录
		HostKeyCallback: certCheck.CheckHostKey,
		Timeout:         sshTimeout,
	}

	addr := vm.Status.Network.IPAddresses[0].String() + ":22"
	sshConn, err := ssh.Dial("tcp", addr, config) //验证ssh服务是否能够返回无法认证的错误，以此判断ssh服务是否正常
	if err != nil {
		if strings.Contains(err.Error(), "unable to authenticate") {
			// we connected to the ssh server and recieved the expected failure
			return nil
		}
		return err
	}

	defer sshConn.Close()
	return fmt.Errorf("waitForSSH: connected successfully with no authentication -- failure was expected")
}

func SetManifestStorage() (err error) {
	log.Trace("Initializing the ManifestStorage provider...")
	ManifestStorage, err = manifest.NewTwoWayManifestStorage(constants.MANIFEST_DIR, constants.DATA_DIR, scheme.Serializer)
	if err != nil {
		return
	}

	providers.Storage = cache.NewCache(ManifestStorage)
	return
}

func NewTwoWayManifestStorage(manifestDir, dataDir string, ser serializer.Serializer) (*ManifestStorage, error) {
   ws, err := watch.NewGenericWatchStorage(storage.NewGenericStorage(storage.NewGenericMappedRawStorage(manifestDir), ser))
   if err != nil {
      return nil, err
   }

   ss := sync.NewSyncStorage(
      storage.NewGenericStorage(
         storage.NewGenericRawStorage(dataDir), ser),
      ws)

   return &ManifestStorage{
      Storage: ss,
   }, nil
}

func ReconcileManifests(s *manifest.ManifestStorage) {
	startMetricsThread()

	// Wrap the Manifest Storage with a cache for better performance, and create a client
	c = client.NewClient(cache.NewCache(s))

	// 监听syncStorage中传过来的事件
	for upd := range s.GetUpdateStream() {

		// 仅关心vm资源的事件
		if upd.APIType.GetKind() != api.KindVM {
			log.Tracef("GitOps: Ignoring kind %s", upd.APIType.GetKind())
			kindIgnored.Inc()
			continue
		}

		var vm *api.VM
		var err error
    //如果是删除事件，此时 manifeststorage.ManifestStorage 中vm的描述文件已经被删除，无法从ManifestStorage中获取vm
		if upd.Event == update.ObjectEventDelete { 
			// As we know this VM was deleted, it wouldn't show up in a Get() call
			// Construct a temporary VM object for passing to the delete function
			vm = &api.VM{
				TypeMeta:   *upd.APIType.GetTypeMeta(),
				ObjectMeta: *upd.APIType.GetObjectMeta(),
				Status: api.VMStatus{
					Running: true, // TODO: Fix this in StopVM
				},
			}
		} else {
			// Get the real API object
			vm, err = c.VMs().Get(upd.APIType.GetUID())
			if err != nil {
				log.Errorf("Getting %s %q returned an error: %v", upd.APIType.GetKind(), upd.APIType.GetUID(), err)
				continue
			}

			// If the object was existent in the storage; validate it
			// Validate the VM object
			// TODO: Validate name uniqueness
			if err := validation.ValidateVM(vm).ToAggregate(); err != nil {
				log.Warnf("Skipping %s of %s %q, not valid: %v.", upd.Event, upd.APIType.GetKind(), upd.APIType.GetUID(), err)
				continue
			}
		}

		// TODO: Parallelization
		switch upd.Event {
		case update.ObjectEventCreate, update.ObjectEventModify: //处理创建和修改事件
			runHandle(func() error {
				return handleChange(vm)
			})

		case update.ObjectEventDelete: //处理删除事件
			runHandle(func() error {
				// TODO: Temporary VM Object for removal
				return handleDelete(vm)
			})
		default:
			log.Infof("Unrecognized Git update type %s\n", upd.Event)
			continue
		}
	}
}

func handleChange(vm *api.VM) (err error) {
	// Only apply the new state if it
	// differs from the current state
	running := currentState(vm)
	if vm.Status.Running && !running { // 如果vm元数据中状态是 running，而实际非running， 则启动vm
		err = start(vm)
	} else if !vm.Status.Running && running { // 如果vm元数据中状态非running，而实际是running，则停止vm
		err = stop(vm)
	}

	return
}

func handleDelete(vm *api.VM) error {
	return remove(vm)
}

func remove(vm *api.VM) error {
	log.Infof("Removing VM %q with name %q...", vm.GetUID(), vm.GetName())
	vmDeleted.Inc()
	// Object deletion is performed by the SyncStorage, so we just
	// need to clean up any remaining resources of the VM here
	return operations.CleanupVM(vm)
}

{
	"cniVersion": "0.4.0",
	"name": "ignite-cni-bridge",
	"plugins": [
		{
			"type": "bridge",
			"bridge": "ignite0",
			"isGateway": true,
			"isDefaultGateway": true,
			"promiscMode": true,
			"ipMasq": true,
			"ipam": {
				"type": "host-local",
				"subnet": "10.61.0.0/16"
			}
		},
		{
			"type": "portmap",
			"capabilities": {
				"portMappings": true
			}
		},
		{
			"type": "firewall"
		}
	]
}

New(config ...CNIOpt) (CNI, error)  //初始化一个cni对象
// Setup setup the network for the namespace
Setup(ctx context.Context, id string, path string, opts ...NamespaceOpts) (*CNIResult, error)
// Remove tears down the network of the namespace.
Remove(ctx context.Context, id string, path string, opts ...NamespaceOpts) error
// Load loads the cni network config
Load(opts ...CNIOpt) error