Skip to content

Commit

Permalink
Merge pull request #85 from canonical/KU-344/cloudinit-run
Browse files Browse the repository at this point in the history 
Add support for boot, preRun and postRun commands on cloud-init
  • Loading branch information
@neoaggelos
neoaggelos authored Feb 14, 2024
2 parents b095c3f + cbdbda5 commit c93e66b
Show file tree
Hide file tree
Showing 11 changed files with 208 additions and 4 deletions.
14 changes: 14 additions & 0 deletions apis/v1beta1/microk8sconfig_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,20 @@ type InitConfiguration struct {
// ExtraKubeletArgs is a list of extra arguments to add to the kubelet.
// +optional
ExtraKubeletArgs []string `json:"extraKubeletArgs,omitempty"`

// BootCommands is a list of commands to run during boot.
// These will be injected into the `bootcmd` section of cloud-init.
BootCommands []string `json:"bootCommands,omitempty"`

// PreRunCommands is a list of commands to run before installing MicroK8s.
// These will be injected into the `runcmd` section of cloud-init.
// +optional
PreRunCommands []string `json:"preRunCommands,omitempty"`

// PostRunCommands is a list of commands to run after installing MicroK8s.
// These will be injected into the `runcmd` section of cloud-init.
// +optional
PostRunCommands []string `json:"postRunCommands,omitempty"`
}

// CloudInitWriteFile is a file that will be injected by cloud-init
Expand Down
15 changes: 15 additions & 0 deletions apis/v1beta1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

41 changes: 41 additions & 0 deletions bootstrap-components.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,12 @@ spec:
schemas to the latest internal value, and may reject unrecognized
values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
bootCommands:
description: BootCommands is a list of commands to run during
boot. These will be injected into the `bootcmd` section of cloud-init.
items:
type: string
type: array
confinement:
description: The confinement (strict or classic) configuration
enum:
Expand Down Expand Up @@ -145,6 +151,20 @@ spec:
noProxy:
description: The optional no proxy configuration
type: string
postRunCommands:
description: PostRunCommands is a list of commands to run after
installing MicroK8s. These will be injected into the `runcmd`
section of cloud-init.
items:
type: string
type: array
preRunCommands:
description: PreRunCommands is a list of commands to run before
installing MicroK8s. These will be injected into the `runcmd`
section of cloud-init.
items:
type: string
type: array
riskLevel:
default: stable
description: The risk-level (stable, candidate, beta, or edge)
Expand Down Expand Up @@ -336,6 +356,13 @@ spec:
convert recognized schemas to the latest internal value,
and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
bootCommands:
description: BootCommands is a list of commands to run
during boot. These will be injected into the `bootcmd`
section of cloud-init.
items:
type: string
type: array
confinement:
description: The confinement (strict or classic) configuration
enum:
Expand Down Expand Up @@ -397,6 +424,20 @@ spec:
noProxy:
description: The optional no proxy configuration
type: string
postRunCommands:
description: PostRunCommands is a list of commands to
run after installing MicroK8s. These will be injected
into the `runcmd` section of cloud-init.
items:
type: string
type: array
preRunCommands:
description: PreRunCommands is a list of commands to run
before installing MicroK8s. These will be injected into
the `runcmd` section of cloud-init.
items:
type: string
type: array
riskLevel:
default: stable
description: The risk-level (stable, candidate, beta,
Expand Down
20 changes: 20 additions & 0 deletions config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ spec:
schemas to the latest internal value, and may reject unrecognized
values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
bootCommands:
description: BootCommands is a list of commands to run during
boot. These will be injected into the `bootcmd` section of cloud-init.
items:
type: string
type: array
confinement:
description: The confinement (strict or classic) configuration
enum:
Expand Down Expand Up @@ -135,6 +141,20 @@ spec:
noProxy:
description: The optional no proxy configuration
type: string
postRunCommands:
description: PostRunCommands is a list of commands to run after
installing MicroK8s. These will be injected into the `runcmd`
section of cloud-init.
items:
type: string
type: array
preRunCommands:
description: PreRunCommands is a list of commands to run before
installing MicroK8s. These will be injected into the `runcmd`
section of cloud-init.
items:
type: string
type: array
riskLevel:
default: stable
description: The risk-level (stable, candidate, beta, or edge)
Expand Down
21 changes: 21 additions & 0 deletions config/crd/bases/bootstrap.cluster.x-k8s.io_microk8sconfigtemplates.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,13 @@ spec:
convert recognized schemas to the latest internal value,
and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
bootCommands:
description: BootCommands is a list of commands to run
during boot. These will be injected into the `bootcmd`
section of cloud-init.
items:
type: string
type: array
confinement:
description: The confinement (strict or classic) configuration
enum:
Expand Down Expand Up @@ -143,6 +150,20 @@ spec:
noProxy:
description: The optional no proxy configuration
type: string
postRunCommands:
description: PostRunCommands is a list of commands to
run after installing MicroK8s. These will be injected
into the `runcmd` section of cloud-init.
items:
type: string
type: array
preRunCommands:
description: PreRunCommands is a list of commands to run
before installing MicroK8s. These will be injected into
the `runcmd` section of cloud-init.
items:
type: string
type: array
riskLevel:
default: stable
description: The risk-level (stable, candidate, beta,
Expand Down
3 changes: 2 additions & 1 deletion controllers/cloudinit/cloudinit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func NewBaseCloudConfig() *CloudConfig {
})
}
return &CloudConfig{
WriteFiles: writeFiles,
WriteFiles: writeFiles,
RunCommands: []string{"set -x"},
}
}
56 changes: 56 additions & 0 deletions controllers/cloudinit/cloudinit_common_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,4 +326,60 @@ func TestCloudConfigInput(t *testing.T) {
})
}
})

t.Run("CustomCommands", func(t *testing.T) {
for _, tc := range []struct {
name string
makeCloudConfig func() (*cloudinit.CloudConfig, error)
}{
{
name: "ControlPlaneInit",
makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
return cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{
BootCommands: []string{"cmd1"},
PreRunCommands: []string{"cmd2"},
PostRunCommands: []string{"cmd3"},
KubernetesVersion: "v1.25.0",
Token: strings.Repeat("a", 32),
TokenTTL: 100,
})
},
},
{
name: "ControlPlaneJoin",
makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
return cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{
BootCommands: []string{"cmd1"},
PreRunCommands: []string{"cmd2"},
PostRunCommands: []string{"cmd3"},
KubernetesVersion: "v1.25.0",
Token: strings.Repeat("a", 32),
TokenTTL: 100,
})
},
},
{
name: "Worker",
makeCloudConfig: func() (*cloudinit.CloudConfig, error) {
return cloudinit.NewJoinWorker(&cloudinit.WorkerInput{
BootCommands: []string{"cmd1"},
PreRunCommands: []string{"cmd2"},
PostRunCommands: []string{"cmd3"},
KubernetesVersion: "v1.25.0",
Token: strings.Repeat("a", 32),
})
},
},
} {
t.Run(tc.name, func(t *testing.T) {
g := NewWithT(t)
c, err := tc.makeCloudConfig()
g.Expect(err).NotTo(HaveOccurred())

g.Expect(c.BootCommands).To(ConsistOf(`cmd1`))
g.Expect(c.RunCommands).To(ContainElement(`cmd2`))
g.Expect(c.RunCommands[len(c.RunCommands)-1]).To(Equal(`cmd3`))
})
}
})
}
11 changes: 10 additions & 1 deletion controllers/cloudinit/controlplane_init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ type ControlPlaneInitInput struct {
SnapstoreHTTPProxy string
// SnapstoreHTTPSProxy is https_proxy configuration for snap store.
SnapstoreHTTPSProxy string
// BootCommands is a list of commands to add to the "bootcmd" section of cloud-init.
BootCommands []string
// PreRunCommands is a list of commands to add to the "runcmd" section of cloud-init before installing MicroK8s.
PreRunCommands []string
// PostRunCommands is a list of commands to add to the "runcmd" section of cloud-init after installing MicroK8s.
PostRunCommands []string
}

func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
Expand Down Expand Up @@ -128,8 +134,10 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
Owner: "root:root",
})
}
cloudConfig.BootCommands = append(cloudConfig.BootCommands, input.BootCommands...)

cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PreRunCommands...)
cloudConfig.RunCommands = append(cloudConfig.RunCommands,
"set -x",
fmt.Sprintf("%s %q %q", scriptPath(snapstoreHTTPProxyScript), input.SnapstoreHTTPProxy, input.SnapstoreHTTPSProxy),
fmt.Sprintf("%s %q %q", scriptPath(snapstoreProxyScript), input.SnapstoreProxyDomain, input.SnapstoreProxyId),
scriptPath(disableHostServicesScript),
Expand All @@ -146,6 +154,7 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
fmt.Sprintf("%s %s", scriptPath(microk8sEnableScript), strings.Join(addons, " ")),
fmt.Sprintf("microk8s add-node --token-ttl %v --token %q", input.TokenTTL, input.Token),
)
cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PostRunCommands...)

return cloudConfig, nil
}
11 changes: 10 additions & 1 deletion controllers/cloudinit/controlplane_join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ type ControlPlaneJoinInput struct {
SnapstoreHTTPProxy string
// SnapstoreHTTPSProxy is https_proxy configuration for snap store.
SnapstoreHTTPSProxy string
// BootCommands is a list of commands to add to the "bootcmd" section of cloud-init.
BootCommands []string
// PreRunCommands is a list of commands to add to the "runcmd" section of cloud-init before installing MicroK8s.
PreRunCommands []string
// PostRunCommands is a list of commands to add to the "runcmd" section of cloud-init after installing MicroK8s.
PostRunCommands []string
}

func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
Expand Down Expand Up @@ -108,8 +114,10 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
joinStr := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[0], input.ClusterAgentPort, input.Token)
joinStrAlt := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[1], input.ClusterAgentPort, input.Token)

cloudConfig.BootCommands = append(cloudConfig.BootCommands, input.BootCommands...)

cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PreRunCommands...)
cloudConfig.RunCommands = append(cloudConfig.RunCommands,
"set -x",
fmt.Sprintf("%s %q %q", scriptPath(snapstoreHTTPProxyScript), input.SnapstoreHTTPProxy, input.SnapstoreHTTPSProxy),
fmt.Sprintf("%s %q %q", scriptPath(snapstoreProxyScript), input.SnapstoreProxyDomain, input.SnapstoreProxyId),
scriptPath(disableHostServicesScript),
Expand All @@ -126,6 +134,7 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
scriptPath(configureAPIServerScript),
fmt.Sprintf("microk8s add-node --token-ttl %v --token %q", input.TokenTTL, input.Token),
)
cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PostRunCommands...)

return cloudConfig, nil
}
11 changes: 10 additions & 1 deletion controllers/cloudinit/worker_join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ type WorkerInput struct {
SnapstoreHTTPProxy string
// SnapstoreHTTPSProxy is https_proxy configuration for snap store.
SnapstoreHTTPSProxy string
// BootCommands is a list of commands to add to the "bootcmd" section of cloud-init.
BootCommands []string
// PreRunCommands is a list of commands to add to the "runcmd" section of cloud-init before installing MicroK8s.
PreRunCommands []string
// PostRunCommands is a list of commands to add to the "runcmd" section of cloud-init after installing MicroK8s.
PostRunCommands []string
}

func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) {
Expand Down Expand Up @@ -97,8 +103,10 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) {
joinStr := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[0], input.ClusterAgentPort, input.Token)
joinStrAlt := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[1], input.ClusterAgentPort, input.Token)

cloudConfig.BootCommands = append(cloudConfig.BootCommands, input.BootCommands...)

cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PreRunCommands...)
cloudConfig.RunCommands = append(cloudConfig.RunCommands,
"set -x",
fmt.Sprintf("%s %q %q", scriptPath(snapstoreHTTPProxyScript), input.SnapstoreHTTPProxy, input.SnapstoreHTTPSProxy),
fmt.Sprintf("%s %q %q", scriptPath(snapstoreProxyScript), input.SnapstoreProxyDomain, input.SnapstoreProxyId),
scriptPath(disableHostServicesScript),
Expand All @@ -110,6 +118,7 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) {
fmt.Sprintf("%s yes %q %q", scriptPath(microk8sJoinScript), joinStr, joinStrAlt),
fmt.Sprintf("%s %s 6443 %s", scriptPath(configureTraefikScript), input.ControlPlaneEndpoint, stopApiServerProxyRefreshes),
)
cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PostRunCommands...)

return cloudConfig, nil
}
9 changes: 9 additions & 0 deletions controllers/microk8sconfig_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,9 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte
ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs,
SnapstoreHTTPProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPProxy,
SnapstoreHTTPSProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPSProxy,
BootCommands: microk8sConfig.Spec.InitConfiguration.BootCommands,
PreRunCommands: microk8sConfig.Spec.InitConfiguration.PreRunCommands,
PostRunCommands: microk8sConfig.Spec.InitConfiguration.PostRunCommands,
}
if controlPlaneInput.TokenTTL == 0 {
controlPlaneInput.TokenTTL = 315569260
Expand Down Expand Up @@ -410,6 +413,9 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con
ExtraKubeletArgs: microk8sConfig.Spec.InitConfiguration.ExtraKubeletArgs,
SnapstoreHTTPProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPProxy,
SnapstoreHTTPSProxy: microk8sConfig.Spec.InitConfiguration.SnapstoreHTTPSProxy,
BootCommands: microk8sConfig.Spec.InitConfiguration.BootCommands,
PreRunCommands: microk8sConfig.Spec.InitConfiguration.PreRunCommands,
PostRunCommands: microk8sConfig.Spec.InitConfiguration.PostRunCommands,
}
if controlPlaneInput.TokenTTL == 0 {
controlPlaneInput.TokenTTL = 315569260
Expand Down Expand Up @@ -509,6 +515,9 @@ func (r *MicroK8sConfigReconciler) handleJoiningWorkerNode(ctx context.Context,

workerInput.ExtraKubeletArgs = c.ExtraKubeletArgs
workerInput.ExtraWriteFiles = cloudinit.WriteFilesFromAPI(c.ExtraWriteFiles)
workerInput.BootCommands = c.BootCommands
workerInput.PreRunCommands = c.PreRunCommands
workerInput.PostRunCommands = c.PostRunCommands
}
bootstrapInitData, err := cloudinit.NewJoinWorker(workerInput)
if err != nil {
Expand Down

0 comments on commit c93e66b

Please sign in to comment.