Kubernetes多集群管理之路

apiVersion: core.kubefed.io/v1beta1
kind: KubeFedCluster
metadata:
  creationTimestamp: "2022-11-14T11:34:44Z"
  generation: 1
  labels:
    ... ...
  name: bbf93fd3-921b-488a-a06b-ceff037cb923
  namespace: kube-federation-system
  resourceVersion: "115683914"
  uid: 39950081-c5d1-4eeb-a69c-6892149142b2
spec:
  apiEndpoint: https://127.0.0.1:6443
  caBundle: xxx
  secretRef:
    name: bbf93fd3-921b-488a-a06b-ceff037cb923-secret
status:
  conditions:
  - lastProbeTime: "2022-11-14T21:21:38Z"
    lastTransitionTime: "2022-11-14T11:34:53Z"
    message: /healthz responded with ok
    reason: ClusterReady
    status: "True"
    type: Ready

type KubeFedClusterSpec struct {
    APIEndpoint string `json:"apiEndpoint"`
    CABundle []byte `json:"caBundle,omitempty"`
    SecretRef LocalSecretReference `json:"secretRef"`
    DisabledTLSValidations []TLSValidation `json:"disabledTLSValidations,omitempty"`
}
 
type KubeFedClusterStatus struct {
    Conditions []ClusterCondition `json:"conditions"`
    Zones []string `json:"zones,omitempty"`
    Region *string `json:"region,omitempty"`
}
 
type ClusterCondition struct {
    Type common.ClusterConditionType `json:"type"`
    Status apiv1.ConditionStatus `json:"status"`
    LastProbeTime metav1.Time `json:"lastProbeTime"`
    LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"`
    Reason *string `json:"reason,omitempty"`
Message *string `json:"message,omitempty"`
}

type ClusterClient struct {
    kubeClient  *kubeclientset.Clientset
    clusterName string
}

func NewClusterClientSet (c *fedv1b1.KubeFedCluster, client generic.Client, fedNamespace string, timeout time.Duration) (*ClusterClient, error) {
    clusterConfig, err := util.BuildClusterConfig(c, client, fedNamespace)
    if err != nil {
        return nil, err
    }
    clusterConfig.Timeout = timeout
    var clusterClientSet = ClusterClient{clusterName: c.Name}
    if clusterConfig != nil {
        clusterClientSet.kubeClient = kubeclientset.NewForConfigOrDie((restclient.AddUserAgent(clusterConfig, UserAgentName)))
        if clusterClientSet.kubeClient == nil {
            return nil, nil
        }
    }
    return &clusterClientSet, nil
}

func (self *ClusterClient) GetClusterHealthStatus() (*fedv1b1.KubeFedClusterStatus, error) {
    body, err := self.kubeClient.DiscoveryClient.RESTClient().Get().AbsPath("/healthz").Do().Raw()
    if err != nil {
        runtime.HandleError(errors.Wrapf(err, "Failed to do cluster health check for cluster %q", self.clusterName))
        clusterStatus.Conditions = append(clusterStatus.Conditions, newClusterOfflineCondition)
        metrics.RegisterKubefedClusterTotal(metrics.ClusterOffline, self.clusterName)
    } else {
        if !strings.EqualFold(string(body), "ok") {
            metrics.RegisterKubefedClusterTotal(metrics.ClusterNotReady, self.clusterName)
            clusterStatus.Conditions = append(clusterStatus.Conditions, newClusterNotReadyCondition, newClusterNotOfflineCondition)
        } else {
            metrics.RegisterKubefedClusterTotal(metrics.ClusterReady, self.clusterName)
            clusterStatus.Conditions = append(clusterStatus.Conditions, newClusterReadyCondition)
        }
    }
    return &clusterStatus, err
}

type ClusterData struct {
    clusterKubeClient *ClusterClient
    clusterStatus *fedv1b1.KubeFedClusterStatus
    resultRun int64
cachedObj *fedv1b1.KubeFedCluster
}
 
type ClusterController struct {
    client genericclient.Client
    clusterHealthCheckConfig *util.ClusterHealthCheckConfig
    mu sync.RWMutex
    clusterDataMap map[string]*ClusterData
    clusterController cache.Controller
    fedNamespace string
    eventRecorder record.EventRecorder
}

func newClusterController(config *util.ControllerConfig, clusterHealthCheckConfig *util.ClusterHealthCheckConfig) (*ClusterController, error) {
   ... ...
   kubeConfig := restclient.CopyConfig(config.KubeConfig)
   cc := &ClusterController{
      client:                   client,
      clusterHealthCheckConfig: clusterHealthCheckConfig,
      clusterDataMap:           make(map[string]*ClusterData),
      fedNamespace:             config.KubeFedNamespace,
   }
   ... ...
   kubeClient := kubeclient.NewForConfigOrDie(kubeConfig)
 
   var err error
   _, cc.clusterController, err = util.NewGenericInformerWithEventHandler(
      config.KubeConfig,
      config.KubeFedNamespace,
      &fedv1b1.KubeFedCluster{},
      util.NoResyncPeriod,
      &cache.ResourceEventHandlerFuncs{
         DeleteFunc: func(obj interface{}) {
            castObj, ok := obj.(*fedv1b1.KubeFedCluster)
            if !ok {
               tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
               if !ok {
                  klog.Errorf("Couldn't get object from tombstone %#v", obj)
                  return
               }
               castObj, ok = tombstone.Obj.(*fedv1b1.KubeFedCluster)
               if !ok {
                  klog.Errorf("Tombstone contained object that is not expected %#v", obj)
                  return
               }
            }
            cc.delFromClusterSet(castObj)
         },
         AddFunc: func(obj interface{}) {
            castObj := obj.(*fedv1b1.KubeFedCluster)
            cc.addToClusterSet(castObj)
         },
         UpdateFunc: func(oldObj, newObj interface{}) {
            var clusterChanged bool
            cluster := newObj.(*fedv1b1.KubeFedCluster)
            cc.mu.Lock()
            clusterData, ok := cc.clusterDataMap[cluster.Name]
            if !ok || !equality.Semantic.DeepEqual(clusterData.cachedObj.Spec, cluster.Spec) ||
               !equality.Semantic.DeepEqual(clusterData.cachedObj.ObjectMeta.Annotations, cluster.ObjectMeta.Annotations) ||
               !equality.Semantic.DeepEqual(clusterData.cachedObj.ObjectMeta.Labels, cluster.ObjectMeta.Labels) {
               clusterChanged = true
            }
            cc.mu.Unlock()
            if !clusterChanged {
               return
            }
            cc.delFromClusterSet(cluster)
            cc.addToClusterSet(cluster)
         },
      },
   )
   return cc, err
}

apiVersion: core.kubefed.k8s.io/v1beta1
kind: FederatedTypeConfig
metadata:
  name: configmaps
  namespace: kube-federation-system
spec:
  federatedType:
    group: types.kubefed.k8s.io
    kind: FederatedConfigMap
    pluralName: federatedconfigmaps
    scope: Namespaced
    version: v1beta1
  propagation: Enabled
  targetType:
    kind: ConfigMap
    pluralName: configmaps
    scope: Namespaced
    version: v1

// 启用CRD，执行后会会创建一个CRD：federatedcustomresourcedefinitions.types.kubefed.io
kubefedctl enable customresourcedefinitions
// 检索Kubefed控制平面所在命名空间是否生成与上述一样的CRD
kubectl -n kube-federation-system get federatedtypeconfigs.core.kubefed.io
// 执行创建名为envoyfilters.networking.istio.io的联邦资源
kubefedctl federate crd envoyfilters.networking.istio.io

apiVersion: types.kubefed.k8s.io/v1beta1
kind: FederatedDeployment
metadata:
  name: fed-deployment
  namespace: fed-namespace
spec:
  template:
    metadata:
      labels:
        app: nginx
    spec:
      ... ...
  placement:
    clusters:
    - name: cluster1
    - name: cluster2
  overrides:
  - clusterName: cluster2
    clusterOverrides:
    - path: spec.replicas
      value: 6

apiVersion: scheduling.kubefed.k8s.io/v1alpha1
kind: ReplicaSchedulingPreference
metadata:
  name: fed-deployment
  namespace: fed-namespace
spec:
  targetKind: FederatedDeployment
  totalReplicas: 10
  clusters:
    cluster1:
      minReplicas: 3
      maxReplicas: 6
      weight: 1
    cluster2:
      minReplicas: 5
      maxReplicas: 12
      weight: 2

karmadactl join member1 --cluster-kubeconfig=$HOME/.kube/karmada.config
karmadactl unjoin member1 --cluster-kubeconfig=$HOME/.kube/karmada.config

kind: Cluster
metadata:
  finalizers:
  - karmada.io/cluster-controller
  name: member1
spec:
  apiEndpoint: https://127.0.0.1:6443
  secretRef:
    name: member1
    namespace: karmada-cluster
  syncMode: Push

apiVersion: cluster.karmada.io/v1alpha1
kind: Cluster
metadata:
  finalizers:
  - karmada.io/cluster-controller
  name: member1
spec:
  apiEndpoint: https://127.0.0.1:6443
  secretRef:
    name: member1
    namespace: karmada-cluster
  syncMode: Push
status:
  apiEnablements:
  - groupVersion: v1
    resources:
    - kind: Endpoints
      name: endpoints
    - kind: Node
      name: nodes
    - kind: Pod
      name: pods
- kind: Service
      name: services
    ... ...
  conditions:
  - lastTransitionTime: "2022-11-13T10:18:36Z"
    message: cluster is reachable and health endpoint responded with ok
    reason: ClusterReady
    status: "True"
    type: Ready
  kubernetesVersion: v1.21.5
  nodeSummary:
    readyNum: 3
    totalNum: 3
  resourceSummary:
    allocatable:
      cpu: "32"
      pods: "256"
    ... ...

apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: example-policy
spec:
  resourceSelectors:
  - apiVersion: apps/v1
    kind: Deployment
    name: deployment-1
    labelSelector: 
  propagateDependensies: false
  placement:
    clusterAffinity:
      clusterNames:
        - cluster1
        - cluster3
     clusterTolerations:
     spreadConstraints:
       - spreadByLabel: failuredomain.kubernetes.io/zone
         maxGroups: 3
         minGroups: 3
   schedulerName: default

apiVersion: policy.karmada.io/v1alpha1
kind: OverridePolicy
metadata:
  name: example-override
  namespace: default
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
  targetCluster:
    labelSelector:
      matchLabels:
      failuredomain.kubernetes.io/region: region1
  overriders:
    imageOverrider:
    - component: prefix
      operator: replace
      value: "region-1.registry.io

apiVersion: apps/v1
kind: Deployment
metadata:
  name: busybox
  namespace: default
  labels:
    app: busybox
spec:
  replicas: 1
  selector:
    matchLabels:
      app: busybox
  template:
    metadata:
      labels:
        app: busybox
    spec:
      containers:
      - image: busybox:latest
        name: busybox

apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-propagation
  namespace: default
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
      name: busybox
  placement:
    clusterAffinity:
      clusterNames:
        - member1
        - member2
  dependentOverrides:
    - nginx-override

apiVersion: policy.karmada.io/v1alpha1
kind: OverridePolicy
metadata:
  name: busybox-override
  namespace: default
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
      name: busybox
  targetCluster:
    clusterNames:
      - member2
  overriders:
    plaintext:
    - path: "/spec/replicas"
      operator: replace
      value: 2