Skip to content

Commit

Permalink
fix: update kosmosctl to support new version
Browse files Browse the repository at this point in the history 
Signed-off-by: ONE7live <[email protected]>
  • Loading branch information
@ONE7live
ONE7live committed Nov 25, 2024
1 parent ff2019f commit 86b8122
Show file tree
Hide file tree
Showing 11 changed files with 725 additions and 209 deletions.
9 changes: 6 additions & 3 deletions pkg/kosmosctl/floater/analysis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,8 @@ type PrintAnalysisData struct {
}

func NewCmdAnalysis(f ctlutil.Factory) *cobra.Command {
o := &CommandAnalysisOptions{
Version: version.GetReleaseVersion().PatchRelease(),
}
o := &CommandAnalysisOptions{}

cmd := &cobra.Command{
Use: "analysis",
Short: i18n.T("Analysis network connectivity between Kosmos clusters"),
Expand Down Expand Up @@ -98,6 +97,10 @@ func (o *CommandAnalysisOptions) Complete(f ctlutil.Factory) error {
return fmt.Errorf("kosmosctl analysis complete error, generate dynamic client failed: %s", err)
}

if len(o.Version) == 0 {
o.Version = version.GetReleaseVersion().PatchRelease()
}

af := NewAnalysisFloater(o)
if err = af.completeFromKubeConfigPath(o.KubeConfig, o.Context); err != nil {
return err
Expand Down
78 changes: 74 additions & 4 deletions pkg/kosmosctl/floater/check.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
package floater

import (
"context"
"fmt"
"os"
"strconv"

"github.com/olekukonko/tablewriter"
"github.com/spf13/cobra"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
ctlutil "k8s.io/kubectl/pkg/cmd/util"
"k8s.io/kubectl/pkg/util/i18n"
"k8s.io/kubectl/pkg/util/templates"
Expand Down Expand Up @@ -59,9 +62,8 @@ type PrintCheckData struct {
}

func NewCmdCheck() *cobra.Command {
o := &CommandCheckOptions{
Version: version.GetReleaseVersion().PatchRelease(),
}
o := &CommandCheckOptions{}

cmd := &cobra.Command{
Use: "check",
Short: i18n.T("Check network connectivity between Kosmos clusters"),
Expand Down Expand Up @@ -89,6 +91,7 @@ func NewCmdCheck() *cobra.Command {
flags.StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.")
flags.StringVarP(&o.ImageRepository, "image-repository", "r", utils.DefaultImageRepository, "Image repository.")
flags.StringVarP(&o.DstImageRepository, "dst-image-repository", "", "", "Destination cluster image repository.")
flags.StringVar(&o.Version, "version", "", "image version for pull images")
flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the host kubeconfig file.")
flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.")
flags.StringVar(&o.SrcKubeConfig, "src-kubeconfig", "", "Absolute path to the source cluster kubeconfig file.")
Expand All @@ -104,6 +107,10 @@ func NewCmdCheck() *cobra.Command {
}

func (o *CommandCheckOptions) Complete() error {
if len(o.Version) == 0 {
o.Version = version.GetReleaseVersion().PatchRelease()
}

if len(o.DstImageRepository) == 0 {
o.DstImageRepository = o.ImageRepository
}
Expand All @@ -127,7 +134,70 @@ func (o *CommandCheckOptions) Complete() error {

func (o *CommandCheckOptions) Validate() error {
if len(o.Namespace) == 0 {
return fmt.Errorf("namespace must be specified")
return fmt.Errorf("kosmosctl check validate error, namespace must be specified")
}

nodeRoleLabel := utils.NodeRoleLabel + "=" + utils.NodeRoleValue
srcConfig, err := utils.RestConfig(o.SrcKubeConfig, o.Context)
if err != nil {
return fmt.Errorf("kosmosctl check validate error, exception in kubeconfig verification of source cluster")
}
srcK8sClient, err := kubernetes.NewForConfig(srcConfig)
if err != nil {
return fmt.Errorf("kosmosctl check validate error, generate source K8s basic client failed: %v", err)
}
srcNodes, err := srcK8sClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
LabelSelector: nodeRoleLabel,
})
if err != nil {
return fmt.Errorf("kosmosctl check validate error, src cluster has no available nodes: %v", err)
}
for _, node := range srcNodes.Items {
labels := node.Labels
if value, exists := labels[utils.KosmosExcludeNodeLabel]; !exists || value != utils.KosmosExcludeNodeValue {
nodeCopy := node.DeepCopy()
if nodeCopy.Labels == nil {
nodeCopy.Labels = make(map[string]string)
}
nodeCopy.Labels[utils.KosmosExcludeNodeLabel] = utils.KosmosExcludeNodeValue

_, err = srcK8sClient.CoreV1().Nodes().Update(context.TODO(), nodeCopy, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("kosmosctl check validate error, source cluster agent node label update exception: %v", err)
}
}
}

if o.DstFloater != nil {
dstConfig, err := utils.RestConfig(o.DstKubeConfig, o.Context)
if err != nil {
return fmt.Errorf("kosmosctl check verify error, kubeconfig verification of the target cluster failed")
}
dstK8sClient, err := kubernetes.NewForConfig(dstConfig)
if err != nil {
return fmt.Errorf("kosmosctl check validate error, generate target K8s basic client failed: %v", err)
}
dstNodes, err := dstK8sClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
LabelSelector: nodeRoleLabel,
})
if err != nil {
return fmt.Errorf("kosmosctl check validate error, dst cluster has no available nodes: %v", err)
}
for _, node := range dstNodes.Items {
labels := node.Labels
if value, exists := labels[utils.KosmosExcludeNodeLabel]; !exists || value != utils.KosmosExcludeNodeValue {
nodeCopy := node.DeepCopy()
if nodeCopy.Labels == nil {
nodeCopy.Labels = make(map[string]string)
}
nodeCopy.Labels[utils.KosmosExcludeNodeLabel] = utils.KosmosExcludeNodeValue

_, err = dstK8sClient.CoreV1().Nodes().Update(context.TODO(), nodeCopy, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("kosmosctl check validate error, target cluster agent node label update exception: %v", err)
}
}
}
}

return nil
Expand Down
60 changes: 32 additions & 28 deletions pkg/kosmosctl/install/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,11 +292,11 @@ func (o *CommandInstallOptions) runClusterlink() error {
if err != nil {
return err
}
clusterlinkClusterNode, err := util.GenerateCustomResourceDefinition(manifest.ClusterNode, nil)
clusterlinkClusterNode, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkClusterNode, nil)
if err != nil {
return err
}
clusterlinkNodeConfig, err := util.GenerateCustomResourceDefinition(manifest.NodeConfig, nil)
clusterlinkNodeConfig, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkNodeConfig, nil)
if err != nil {
return err
}
Expand Down Expand Up @@ -334,7 +334,7 @@ func (o *CommandInstallOptions) runClusterlink() error {
}
klog.Info("Deployment " + networkManagerDeploy.Name + " has been created.")

operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
operatorDeploy, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.DeploymentReplace{
Namespace: o.Namespace,
Version: o.Version,
UseProxy: o.UseProxy,
Expand Down Expand Up @@ -489,27 +489,6 @@ func (o *CommandInstallOptions) runClustertree() error {
}
klog.Info("Deployment clustertree-cluster-manager has been created.")

operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
Namespace: o.Namespace,
Version: o.Version,
UseProxy: o.UseProxy,
ImageRepository: o.ImageRegistry,
})
if err != nil {
return fmt.Errorf("kosmosctl install operator run error, operator generate deployment failed: %s", err)
}
_, err = o.K8sClient.AppsV1().Deployments(operatorDeploy.Namespace).Get(context.TODO(), operatorDeploy.Name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
err = o.createOperator()
if err != nil {
return err
}
} else {
return fmt.Errorf("kosmosctl install operator run error, operator get deployment failed: %s", err)
}
}

return nil
}

Expand Down Expand Up @@ -583,7 +562,7 @@ func (o *CommandInstallOptions) runScheduler() error {
}
klog.Infof("ConfigMap %s has been created.", scheduleConfigFile.Name)

hostkubeConfigMap := &corev1.ConfigMap{
hostKubeConfigMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: utils.HostKubeConfigName,
Namespace: o.Namespace,
Expand All @@ -597,14 +576,39 @@ func (o *CommandInstallOptions) runScheduler() error {
}()),
},
}
_, err = o.K8sClient.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), hostkubeConfigMap, metav1.CreateOptions{})
_, err = o.K8sClient.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), hostKubeConfigMap, metav1.CreateOptions{})
if err != nil {
if !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("kosmosctl install scheduler run error, configmap options failed: %v", err)
}
}
klog.Info("ConfigMap host-kubeconfig has been created.")

klog.Info("Attempting to create Kosmos-Scheduler CRDs...")
crds := apiextensionsv1.CustomResourceDefinitionList{}
schedulerCDP, err := util.GenerateCustomResourceDefinition(manifest.SchedulerClusterDistributionPolicies, manifest.CRDReplace{
Namespace: o.Namespace,
})
if err != nil {
return err
}
schedulerDP, err := util.GenerateCustomResourceDefinition(manifest.SchedulerDistributionPolicies, nil)
if err != nil {
return err
}
crds.Items = append(crds.Items, *schedulerCDP, *schedulerDP)
for i := range crds.Items {
_, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), &crds.Items[i], metav1.CreateOptions{})
if err != nil {
if apierrors.IsAlreadyExists(err) {
klog.Warningf("CRD %v is existed, creation process will skip", &crds.Items[i].Name)
continue
}
return fmt.Errorf("kosmosctl install scheduler run error, crd options failed: %v", err)
}
klog.Info("Create CRD " + crds.Items[i].Name + " successful.")
}

klog.Info("Start creating kosmos-scheduler Deployment...")
schedulerDeploy, err := util.GenerateDeployment(manifest.SchedulerDeployment, manifest.DeploymentReplace{
Namespace: o.Namespace,
Expand All @@ -631,7 +635,7 @@ func (o *CommandInstallOptions) runScheduler() error {

func (o *CommandInstallOptions) createOperator() error {
klog.Info("Start creating ClusterLink-Operator...")
operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
operatorDeploy, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.DeploymentReplace{
Namespace: o.Namespace,
Version: o.Version,
UseProxy: o.UseProxy,
Expand Down Expand Up @@ -685,7 +689,7 @@ func (o *CommandInstallOptions) createOperator() error {
return fmt.Errorf("kosmosctl install operator run error, operator options clusterrolebinding failed: %s", err)
}

operatorSA, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, manifest.ServiceAccountReplace{
operatorSA, err := util.GenerateServiceAccount(manifest.ClusterlinkOperatorServiceAccount, manifest.ServiceAccountReplace{
Namespace: o.Namespace,
})
if err != nil {
Expand Down
65 changes: 20 additions & 45 deletions pkg/kosmosctl/join/join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,13 @@ func (o *CommandJoinOptions) CreateTreeRelatedCRDs() error {

func (o *CommandJoinOptions) runCluster() error {
klog.Info("Start registering cluster to kosmos control plane...")
// create ns if it does not exist
kosmosNS := &corev1.Namespace{}
kosmosNS.Name = o.Namespace
_, err := o.K8sClient.CoreV1().Namespaces().Create(context.TODO(), kosmosNS, metav1.CreateOptions{})
if err != nil && !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("kosmosctl join run error, create namespace failed: %s", err)
}

// create cluster in control panel
cluster := v1alpha1.Cluster{
Expand Down Expand Up @@ -354,38 +361,26 @@ func (o *CommandJoinOptions) runCluster() error {

cluster.Spec.ClusterLinkOptions.DefaultNICName = o.DefaultNICName
cluster.Spec.ClusterLinkOptions.CNI = o.CNI
}

if o.EnableTree {
// create ClusterTree related crds
err := o.CreateTreeRelatedCRDs()
if err != nil {
return err
}

clusterPodConvert, err := util.GenerateCustomResourceDefinition(manifest.ClusterPodConvert, nil)
clusterlinkOperatorSA, err := util.GenerateServiceAccount(manifest.ClusterlinkOperatorServiceAccount, manifest.ServiceAccountReplace{
Namespace: o.Namespace,
})
if err != nil {
return err
return fmt.Errorf("kosmosctl join run error, generate serviceaccount failed: %s", err)
}
_, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), clusterPodConvert, metav1.CreateOptions{})
if err != nil {
if !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("kosmosctl join run error, crd options failed: %v", err)
}
_, err = o.K8sClient.CoreV1().ServiceAccounts(clusterlinkOperatorSA.Namespace).Create(context.TODO(), clusterlinkOperatorSA, metav1.CreateOptions{})
if err != nil && !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("kosmosctl join run error, create serviceaccount failed: %s", err)
}
klog.Info("Create CRD " + clusterPodConvert.Name + " successful.")
klog.Info("ServiceAccount " + clusterlinkOperatorSA.Name + " has been created.")
}

podConvert, err := util.GenerateCustomResourceDefinition(manifest.PodConvert, nil)
if o.EnableTree {
// create ClusterTree related crds
err := o.CreateTreeRelatedCRDs()
if err != nil {
return err
}
_, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), podConvert, metav1.CreateOptions{})
if err != nil {
if !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("kosmosctl join run error, crd options failed: %v", err)
}
}
klog.Info("Create CRD " + podConvert.Name + " successful.")

if len(o.LeafModel) > 0 {
switch o.LeafModel {
Expand Down Expand Up @@ -425,20 +420,12 @@ func (o *CommandJoinOptions) runCluster() error {
}
}

_, err := o.KosmosClient.KosmosV1alpha1().Clusters().Create(context.TODO(), &cluster, metav1.CreateOptions{})
_, err = o.KosmosClient.KosmosV1alpha1().Clusters().Create(context.TODO(), &cluster, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("kosmosctl join run error, create cluster failed: %s", err)
}
klog.Info("Cluster " + o.Name + " has been created.")

// create ns if it does not exist
kosmosNS := &corev1.Namespace{}
kosmosNS.Name = o.Namespace
_, err = o.K8sClient.CoreV1().Namespaces().Create(context.TODO(), kosmosNS, metav1.CreateOptions{})
if err != nil && !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("kosmosctl join run error, create namespace failed: %s", err)
}

// create rbac
kosmosControlSA, err := util.GenerateServiceAccount(manifest.KosmosControlServiceAccount, manifest.ServiceAccountReplace{
Namespace: o.Namespace,
Expand Down Expand Up @@ -490,18 +477,6 @@ func (o *CommandJoinOptions) runCluster() error {
}
klog.Info("ClusterRoleBinding " + kosmosCRB.Name + " has been created.")

kosmosOperatorSA, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, manifest.ServiceAccountReplace{
Namespace: o.Namespace,
})
if err != nil {
return fmt.Errorf("kosmosctl join run error, generate serviceaccount failed: %s", err)
}
_, err = o.K8sClient.CoreV1().ServiceAccounts(kosmosOperatorSA.Namespace).Create(context.TODO(), kosmosOperatorSA, metav1.CreateOptions{})
if err != nil && !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("kosmosctl join run error, create serviceaccount failed: %s", err)
}
klog.Info("ServiceAccount " + kosmosOperatorSA.Name + " has been created.")

// ToDo Wait for all services to be running

klog.Info("Cluster [" + o.Name + "] registration successful.")
Expand Down
2 changes: 1 addition & 1 deletion pkg/kosmosctl/manifest/manifest_configmaps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,10 @@ data:
disabled:
- name: "VolumeBinding"
- name: "TaintToleration"
- name: "LeafNodeDistribution"
enabled:
- name: "LeafNodeTaintToleration"
- name: "LeafNodeVolumeBinding"
- name: "LeafNodeDistribution"
score:
disabled:
- name: "VolumeBinding"
Expand Down
Loading

0 comments on commit 86b8122

Please sign in to comment.