From 78a9caf3baf37b22cfec39a65d3763e756c61425 Mon Sep 17 00:00:00 2001 From: Ian Furst Date: Thu, 11 Jan 2024 18:36:55 -0800 Subject: [PATCH 1/3] feat: added script and docs for worker volume add Short script to add volumes to worker pool nodes --- kubernetes-cluster-kubeone/docs/setup-demo.md | 124 ++++++++++++------ kubernetes-cluster-kubeone/terraform/run.sh | 6 + .../terraform/terraform.tfvars.example | 1 + .../terraform/volumizer.py | 65 +++++++++ 4 files changed, 154 insertions(+), 42 deletions(-) create mode 100755 kubernetes-cluster-kubeone/terraform/volumizer.py diff --git a/kubernetes-cluster-kubeone/docs/setup-demo.md b/kubernetes-cluster-kubeone/docs/setup-demo.md index 6bac5ef..b45ce7f 100644 --- a/kubernetes-cluster-kubeone/docs/setup-demo.md +++ b/kubernetes-cluster-kubeone/docs/setup-demo.md @@ -1,51 +1,60 @@ ## Setup a Kubernetes Cluster for KSD using Hetzner Cloud -This guide is to be ready and test KSD easily. +This document will walk you through the steps required to set up a Kubernetes cluster in the Hetzner Cloud for use with KSD. ### IMPORTANT: -In order to be able to follow all the steps, you would need a Hetzner Api token, to get this you will need: -- Sign in into the Hetzner Cloud Console -- choose a Project -- go to Security → API Tokens -- generate a new token. +To follow this guide, you will need a Hetzner API token. To generate this you will need to: +- Sign in into the [Hetzner Cloud Console](https://console.hetzner.cloud/) +- Select your project +- Go to Security → API Tokens +- Generate a new token -Once you have your token, you must export your token in order to use it during the process +Once you have your token, export it to your shell's environment: ```console $ export HCLOUD_TOKEN=GlPz..... ``` -> If you have troubles, please visit the Hetzner Cloud Documents [https://docs.hetzner.cloud/](https://docs.hetzner.cloud/) +> If you have trouble, please visit the [Hetzner Cloud documentation](https://docs.hetzner.cloud/) site. ## Requirements -- Kubeone - - You can install it by using `curl -sfL https://get.kubeone.io | sh` -- Terraform v1.5.2 -- A new ssh key only for this purpose (how to generate a new ssh-key) -- Kubectl (how to install kubectl) -- Api Token Hetzner Cloud +- Kubeone v1.6.2 + - AMD64 and ARM64 binaries can be found on [Kubermatic's GitHub page](https://github.com/kubermatic/kubeone/releases). +- Terraform v1.5.2 or greater + - Various installation methods can be found on [Hashicorph's installation page](https://developer.hashicorp.com/terraform/install). +- A new ssh key generated for this project + - This can be created by running `ssh-keygen` on your console. When prompted, enter a name and a location for the new key pair. +- Kubectl + - Various installation methods can be found in the [Kubernetes installation guide](https://kubernetes.io/docs/tasks/tools/). +- A Hetzner Cloud API token +- `ssh-agent` running in your shell + - On your console, run the following: + ```console + $ eval `ssh-agent -s` + $ ssh-add /path/to/private/key + ``` ## Architecture -![architecture.png](architecture.png) +![architecture.png](../architecture.png) ## Hands On It's time to prepare your Kubernetes cluster for KSD usage. -#### 1. Clone this repository +#### 1. Clone the Koor demonstration repository ```console git clone git@github.com:koor-tech/demo-gitops.git ``` -#### 2. Navigate to kubernetes-cluster-kubeone +#### 2. Navigate to the terraform directory beneath kubernetes-cluster-kubeone ```console -$ cd kubernetes-cluster-kubeone/terraform/ +$ cd demo-gitops/kubernetes-cluster-kubeone/terraform ``` #### 3. Initialize the terraform configuration @@ -82,25 +91,34 @@ commands will detect it and remind you to do so if necessary. #### 4. Setup your cluster -Inside the terraform folder you could find a file called `terraform.tfvars.example` use that file to set up your cluster as you need +In the terraform directory, copy the `terraform.tfvars.example` file to `terraform.tfvars`, and modify the values to describe your cluster: ```console $ cp terraform.tfvars.example terraform.tfvars ``` +```source +cluster_name = "koor-demo" +ssh_public_key_file = "~/.ssh/id_rsa.pub" +control_plane_vm_count=3 +initial_machinedeployment_replicas=3 +worker_type="cpx41" +control_plane_type="cpx31" +os="ubuntu" +worker_os="ubuntu" +``` -KSD is versatile and can run on various clusters, yet in a production environment, -the following are the essential minimum requirements: +KSD is versatile, and can run on many different cluster configurations. For a production environment, these are the minimum requirements: - - 3 Nodes in control plane + - 3 control plane nodes - 4 CPU - 8 GB RAM - - 3 Nodes on data/worker nodes + - 3 data/worker nodes - 8 CPU - 16 GB RAM - - Calico as CNI (Other CNI plugins work pretty well) + - Calico as the CNI. Other CNI plugins work as well, but haven't been as extensively tested. #### 4. Validate your changes -Run `terraform plan` to examine what changes will be applied in your infrastructure +Run `terraform plan` to see what changes will be applied to your infrastructure: ```console $ terraform plan hcloud_placement_group.control_plane: Refreshing state... [id=185187] @@ -122,7 +140,7 @@ hcloud_server_network.control_plane[1]: Refreshing state... [id=35048830-3137203 #### 4. Apply your changes -These changes only will create your infrastructure and Kubernetes will be installed later +Once you're happy with the proposed changes, apply them to create your infrastructure. Kubernetes will be installed later. ```console $ terraform apply @@ -155,35 +173,57 @@ Terraform planned the following actions, but then encountered a problem: #### 5. Save your infrastructure -You need to save your terraform state into a tf.json file that will be used later for setup your Kubernetes Clusters +Save the generated terraform state to a file. This will be used in the next step to stand up the Kubernetes cluster: ```console -$ terraform output -json > tf.json +$ terraform output -json -no-color > tf.json ``` #### 6. Deploy your Cluster -You already have a `kubeone.yaml` file with the required configuration, but you can update it as you need, and just you need to run: +The `kubeone.yaml` file in the terraform directory already has all necessary configuration details, but you can modify it to meet your requirements. + +Once you're ready, run: ```console $ kubeone apply -m kubeone.yaml -t tf.json +INFO[17:31:59 UTC] Determine hostname... +INFO[17:32:03 UTC] Determine operating system... +INFO[17:32:04 UTC] Running host probes... +The following actions will be taken: +Run with --verbose flag for more information. + + initialize control plane node "koor-demo-test-ian-control-plane-1" (192.168.0.3) using 1.25.6 + + join control plane node "koor-demo-test-ian-control-plane-2" (192.168.0.4) using 1.25.6 + + ensure machinedeployment "koor-demo-test-ian-pool1" with 4 replica(s) exists + + apply embedded addons +Do you want to proceed (yes/no): yes + +INFO[17:32:31 UTC] Determine hostname... +INFO[17:32:31 UTC] Determine operating system... +INFO[17:32:31 UTC] Running host probes... +INFO[17:32:32 UTC] Installing prerequisites... +INFO[17:32:32 UTC] Creating environment file... node=1.2.3.4 os=ubuntu +INFO[17:32:32 UTC] Creating environment file... node=5.6.7.8 os=ubuntu +INFO[17:32:33 UTC] Configuring proxy... node=1.2.3.4 os=ubuntu +INFO[17:32:33 UTC] Installing kubeadm... node=1.2.3.4 os=ubuntu +INFO[17:32:33 UTC] Configuring proxy... node=5.6.7.8 os=ubuntu +INFO[17:32:33 UTC] Installing kubeadm... node=5.6.7.8 os=ubuntu ``` #### 7. Add your volumes -For this step, you will need to access to your hetzner cloud account [https://accounts.hetzner.com/login](https://accounts.hetzner.com/login) +For this step, you will need to access to your [Hetzner Cloud account](https://accounts.hetzner.com/login). -1. Access to your hetzner cloud account +>To utilize all of KSD's features, we recommend associating at least one volume with each data plane node. +1. Navigate to the [Hetzner Cloud console](https://console.hetzner.cloud/) 2. Open your project -3. Go to volumes and add a new volume of your desire size -4. Set the volume name -5. Choose the server - - **important: Choose one server that contains in its name "pool" to use nodes from the data plane** - - Caution: Avoid selecting control plane nodes for KSD, as it relies on deploying pods tied to the volumes. Control plane nodes are unable to host such pods due to [taints and tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). -6. Select in *Choose mount options*, **manually** to be able to manage completely by KSD +3. Go to the volumes tab in the sidebar +4. Set the volume size and name +5. Choose a data plane node to tie the volume to + - **Important:** Data plane nodes have the word "pool" in their names. + - **Caution:** Do not associate volumes to KSD control plane nodes. KSD works by deploying pods on the data plane nodes tied to your volumes. Control plane nodes are unable to host those pods themselves due to [taints and tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). +6. Set the *Choose mount options* slider to **manually** so that your cluster has raw block devices to consume. You'll receive a warning, which can be dismissed. 7. Finally, click on create and buy -8. We recommended setting at least one volume per node to be able to use all the KSD features - -See the image to check how to do that +![how to create a volume](../how-to-create-volume.gif) -![how to create a volume](how-to-create-volume.gif) +Congratulations! You have created a minimal production Kubernetes Cluster that can now be used to deploy KSD. -With the steps above, you will have readied your minimum production Kubernetes Cluster to be used to deploy KSD \ No newline at end of file +>Note: To _destroy_ your cluster, detach all volumes from your worker nodes, delete them, thens navigate to the `demo-gitops/kubernetes-cluster-kubeone/terraform` directory, and run `terraform apply -destroy`. \ No newline at end of file diff --git a/kubernetes-cluster-kubeone/terraform/run.sh b/kubernetes-cluster-kubeone/terraform/run.sh index 2ed9f02..fab9328 100755 --- a/kubernetes-cluster-kubeone/terraform/run.sh +++ b/kubernetes-cluster-kubeone/terraform/run.sh @@ -41,6 +41,8 @@ ssh_key_file=$(grep -E "^ssh_public_key_file" terraform.tfvars | awk -F= '{gsub( ssh_key_file="${ssh_key_file/#\~/$HOME}" +# Extract worker volume size +worker_volume_size=$(grep -E "^worker_volume_size" terraform.tfvars.example | awk -F= '{gsub(/[ \047"]/, "", $2); print $2}' | sed 's/^[ \t]*//;s/[ \t]*$//') if ! [ -f "$ssh_key_file" ]; then echo "Error: SSH public key file '$ssh_key_file' not found." @@ -115,6 +117,10 @@ run() { kubeconfig run ;; + 6) # assign volumes + echo "Assigning volumes to worker nodes." + ./volumizer -c ${cluster_name} -s ${worker_volume_size} + ;; *) echo "Invalid exiting..." echo "The script has finished." diff --git a/kubernetes-cluster-kubeone/terraform/terraform.tfvars.example b/kubernetes-cluster-kubeone/terraform/terraform.tfvars.example index 5cf0bc5..758f763 100644 --- a/kubernetes-cluster-kubeone/terraform/terraform.tfvars.example +++ b/kubernetes-cluster-kubeone/terraform/terraform.tfvars.example @@ -6,3 +6,4 @@ worker_type="cpx41" control_plane_type="cpx31" os="ubuntu" worker_os="ubuntu" +worker_volume_size=10 diff --git a/kubernetes-cluster-kubeone/terraform/volumizer.py b/kubernetes-cluster-kubeone/terraform/volumizer.py new file mode 100755 index 0000000..b1e114e --- /dev/null +++ b/kubernetes-cluster-kubeone/terraform/volumizer.py @@ -0,0 +1,65 @@ +import os +import sys +import getopt +import re + +import hcloud + +expression = re.compile('pool', re.IGNORECASE) + + +def create_and_associate_volume(client, worker, size): + try: + response = client.volumes.create(size=size, name=f"${worker.name}-vol-1") + except hcloud.APIException: + raise + volume = response.volume + volume.attach(worker) + + + +def parseOpts(): + args = sys.argv[1:] + size = 10 + name = "koor-generic" + try: + opts, args = getopt.getopt(args, 's:h', ["size=", "help"]) + except getopt.GetoptError: + print('Usage: volumizer.py --size ') + sys.exit(2) + + for opt, arg in opts: + if opt in ("-h", "--help"): + print('Usage: volumizer.py --size ') + sys.exit() + if opt in ("-s", "--size"): + size = int(arg) + + return size + + +if __name__ == "__main__": + assert ( + "HCLOUD_TOKEN" in os.environ + ), "Please export your API token in the HCLOUD_TOKEN environment variable" + token = os.environ["HCLOUD_TOKEN"] + client = hcloud.Client(token=token) + + size = parseOpts() + + try: + servers = client.servers.get_all() + except hcloud.APIException: + raise + + for i in range(len(servers)): + if expression.search(servers[i].name): + if not servers[i].volumes: + print(f"No volumes attached to ${servers[i].name}. Creating and associating.") + create_and_associate_volume(client=client, worker=servers[i], size=size) + + + + + + From 250a39253823a05bdefd384d7370db24c1b3e46b Mon Sep 17 00:00:00 2001 From: Zuhair AlSader Date: Mon, 15 Jan 2024 17:11:02 -0500 Subject: [PATCH 2/3] chore: replace KSD with Rook Signed-off-by: Zuhair AlSader --- README.md | 8 ++++---- kubernetes-cluster-kubeone/docs/setup-demo.md | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2f63438..d813423 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,15 @@ For managing the infrastructure to run our demo system A new best practice for managing K8s infrastructure is to store configuration in Git and use PRs to make sure changes get reviewed. Seems like a great idea. -## Setting up KSD +## Setting up Rook -KSD can be effortlessly deployed in various environments, including bare metal or the cloud. -In this example, we will demonstrate an automated environment configuration to showcase how easily you can prepare for testing KSD in your preferred setup. +Rook can be effortlessly deployed in various environments, including bare metal or the cloud. +In this example, we will demonstrate an automated environment configuration to showcase how easily you can prepare for testing Rook in your preferred setup. ### 1. Deploy your Kubernetes Cluster in Hetzner Cloud -Read how you can [install your Kubernetes Cluster to use KSD](kubernetes-cluster-demo/docs/setup-demo.md) +Read how you can [install your Kubernetes Cluster to use Rook](kubernetes-cluster-demo/docs/setup-demo.md) diff --git a/kubernetes-cluster-kubeone/docs/setup-demo.md b/kubernetes-cluster-kubeone/docs/setup-demo.md index b45ce7f..2a78e14 100644 --- a/kubernetes-cluster-kubeone/docs/setup-demo.md +++ b/kubernetes-cluster-kubeone/docs/setup-demo.md @@ -1,6 +1,6 @@ -## Setup a Kubernetes Cluster for KSD using Hetzner Cloud +## Setup a Kubernetes Cluster for Rook using Hetzner Cloud -This document will walk you through the steps required to set up a Kubernetes cluster in the Hetzner Cloud for use with KSD. +This document will walk you through the steps required to set up a Kubernetes cluster in the Hetzner Cloud for use with Rook. ### IMPORTANT: @@ -43,7 +43,7 @@ $ export HCLOUD_TOKEN=GlPz..... ## Hands On -It's time to prepare your Kubernetes cluster for KSD usage. +It's time to prepare your Kubernetes cluster for Rook usage. #### 1. Clone the Koor demonstration repository @@ -106,7 +106,7 @@ os="ubuntu" worker_os="ubuntu" ``` -KSD is versatile, and can run on many different cluster configurations. For a production environment, these are the minimum requirements: +Rook is versatile, and can run on many different cluster configurations. For a production environment, these are the minimum requirements: - 3 control plane nodes - 4 CPU @@ -212,18 +212,18 @@ INFO[17:32:33 UTC] Installing kubeadm... node=5.6.7.8 os For this step, you will need to access to your [Hetzner Cloud account](https://accounts.hetzner.com/login). ->To utilize all of KSD's features, we recommend associating at least one volume with each data plane node. +>To utilize all of Rook's features, we recommend associating at least one volume with each data plane node. 1. Navigate to the [Hetzner Cloud console](https://console.hetzner.cloud/) 2. Open your project 3. Go to the volumes tab in the sidebar 4. Set the volume size and name 5. Choose a data plane node to tie the volume to - **Important:** Data plane nodes have the word "pool" in their names. - - **Caution:** Do not associate volumes to KSD control plane nodes. KSD works by deploying pods on the data plane nodes tied to your volumes. Control plane nodes are unable to host those pods themselves due to [taints and tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). + - **Caution:** Do not associate volumes to Rook control plane nodes. Rook works by deploying pods on the data plane nodes tied to your volumes. Control plane nodes are unable to host those pods themselves due to [taints and tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). 6. Set the *Choose mount options* slider to **manually** so that your cluster has raw block devices to consume. You'll receive a warning, which can be dismissed. 7. Finally, click on create and buy ![how to create a volume](../how-to-create-volume.gif) -Congratulations! You have created a minimal production Kubernetes Cluster that can now be used to deploy KSD. +Congratulations! You have created a minimal production Kubernetes Cluster that can now be used to deploy Rook. >Note: To _destroy_ your cluster, detach all volumes from your worker nodes, delete them, thens navigate to the `demo-gitops/kubernetes-cluster-kubeone/terraform` directory, and run `terraform apply -destroy`. \ No newline at end of file From 4c226bcca16ddade82c1c3b05e96522c0fcca51a Mon Sep 17 00:00:00 2001 From: Zuhair AlSader Date: Mon, 15 Jan 2024 18:50:18 -0500 Subject: [PATCH 3/3] feat: add changes from #18 Signed-off-by: Zuhair AlSader --- kubernetes-cluster-kubeone/docs/setup-demo.md | 11 +++- kubernetes-cluster-kubeone/terraform/run.sh | 5 +- .../terraform/terraform.tfvars.example | 2 +- .../terraform/volumizer.py | 60 +++++++++---------- 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/kubernetes-cluster-kubeone/docs/setup-demo.md b/kubernetes-cluster-kubeone/docs/setup-demo.md index 2a78e14..26c10ab 100644 --- a/kubernetes-cluster-kubeone/docs/setup-demo.md +++ b/kubernetes-cluster-kubeone/docs/setup-demo.md @@ -210,7 +210,16 @@ INFO[17:32:33 UTC] Installing kubeadm... node=5.6.7.8 os #### 7. Add your volumes -For this step, you will need to access to your [Hetzner Cloud account](https://accounts.hetzner.com/login). +You can do this by using `volumizer.py` as follows: + +```console +$ pip3 install hcloud +$ ./volumizer.py -s 10 +There are no volumes attached to . Creating and associating. +Creating volume to Hetzner Cloud for . +``` + +Alternatively, you can do this manually. For this step, you will need to access to your [Hetzner Cloud account](https://accounts.hetzner.com/login). >To utilize all of Rook's features, we recommend associating at least one volume with each data plane node. 1. Navigate to the [Hetzner Cloud console](https://console.hetzner.cloud/) diff --git a/kubernetes-cluster-kubeone/terraform/run.sh b/kubernetes-cluster-kubeone/terraform/run.sh index fab9328..fe508d4 100755 --- a/kubernetes-cluster-kubeone/terraform/run.sh +++ b/kubernetes-cluster-kubeone/terraform/run.sh @@ -77,7 +77,7 @@ kubeconfig(){ run() { local choice - read -p "What do you want to do? (1: plan, 2: apply, 3: save infra, 4:deploy cluster, 5: export kubeconfig, other: exit): " choice + read -p "What do you want to do? (1: plan, 2: apply, 3: save infra, 4:deploy cluster, 5: export kubeconfig, 6: add volumes to workers, other: exit): " choice case $choice in 1) # Run terraform plan @@ -119,7 +119,8 @@ run() { ;; 6) # assign volumes echo "Assigning volumes to worker nodes." - ./volumizer -c ${cluster_name} -s ${worker_volume_size} + pip3 install hcloud + ./volumizer.py -c ${cluster_name} -s ${worker_volume_size} ;; *) echo "Invalid exiting..." diff --git a/kubernetes-cluster-kubeone/terraform/terraform.tfvars.example b/kubernetes-cluster-kubeone/terraform/terraform.tfvars.example index 758f763..5a82323 100644 --- a/kubernetes-cluster-kubeone/terraform/terraform.tfvars.example +++ b/kubernetes-cluster-kubeone/terraform/terraform.tfvars.example @@ -6,4 +6,4 @@ worker_type="cpx41" control_plane_type="cpx31" os="ubuntu" worker_os="ubuntu" -worker_volume_size=10 +worker_volume_size=30 diff --git a/kubernetes-cluster-kubeone/terraform/volumizer.py b/kubernetes-cluster-kubeone/terraform/volumizer.py index b1e114e..1d31081 100755 --- a/kubernetes-cluster-kubeone/terraform/volumizer.py +++ b/kubernetes-cluster-kubeone/terraform/volumizer.py @@ -1,42 +1,39 @@ +#!/usr/bin/python3 + +######## +# +# volumizer.py, the simple and effective way of adding volumes to your cluster! +# +# usage: ./volumizer.py -c -s +# +# Relies on the hcloud python library: +# $ pip3 install hcloud +# + import os -import sys -import getopt +import argparse import re import hcloud -expression = re.compile('pool', re.IGNORECASE) - - def create_and_associate_volume(client, worker, size): + volumeName = f"{worker.name}-vol-1" try: - response = client.volumes.create(size=size, name=f"${worker.name}-vol-1") + print(f"Creating volume to Hetzner Cloud for {worker.name}.") + response = client.volumes.create(size=size, name=volumeName, location=worker.datacenter) except hcloud.APIException: raise volume = response.volume volume.attach(worker) - - def parseOpts(): - args = sys.argv[1:] - size = 10 - name = "koor-generic" - try: - opts, args = getopt.getopt(args, 's:h', ["size=", "help"]) - except getopt.GetoptError: - print('Usage: volumizer.py --size ') - sys.exit(2) + parser = argparse.ArgumentParser() - for opt, arg in opts: - if opt in ("-h", "--help"): - print('Usage: volumizer.py --size ') - sys.exit() - if opt in ("-s", "--size"): - size = int(arg) - - return size + parser.add_argument("-s", "--size", type=int, default=10, help="the size of each worker's volume, in gigabyes") + parser.add_argument("-c", "--cluster-name", type=str, default="pool", help="the name of your cluster") + args = parser.parse_args() + return args.cluster_name, args.size if __name__ == "__main__": assert ( @@ -45,7 +42,10 @@ def parseOpts(): token = os.environ["HCLOUD_TOKEN"] client = hcloud.Client(token=token) - size = parseOpts() + name, size = parseOpts() + + inCluster = re.compile(f'{name}', re.IGNORECASE) + inPool = re.compile('pool', re.IGNORECASE) try: servers = client.servers.get_all() @@ -53,13 +53,7 @@ def parseOpts(): raise for i in range(len(servers)): - if expression.search(servers[i].name): + if inCluster.search(servers[i].name) and inPool.search(servers[i].name): if not servers[i].volumes: - print(f"No volumes attached to ${servers[i].name}. Creating and associating.") + print(f"There are no volumes attached to {servers[i].name}. Creating and associating.") create_and_associate_volume(client=client, worker=servers[i], size=size) - - - - - -