Add cluster tests (#52)

* Add cluster tests * Add cloud command to tests * Add destory steps * update wkflow names * Update k8s clusters * add tfc support and fix gcp and aws wkf * Fix gke workflow * Update readme with test status' * Add dp tests
TykTechnologies · May 27, 2024 · 6d39003 · 6d39003
1 parent 4b82866
commit 6d39003
Show file tree

Hide file tree

Showing 10 changed files with 307 additions and 26 deletions.
diff --git a/.github/tfc/aws.tf b/.github/tfc/aws.tf
@@ -0,0 +1,9 @@
+terraform {
+	cloud {
+		organization = "tyk-k8s-demo"
+
+		workspaces {
+			name = "eks"
+		}
+	}
+}
diff --git a/.github/tfc/azure.tf b/.github/tfc/azure.tf
@@ -0,0 +1,9 @@
+terraform {
+	cloud {
+		organization = "tyk-k8s-demo"
+
+		workspaces {
+			name = "aks"
+		}
+	}
+}
diff --git a/.github/tfc/gcp.tf b/.github/tfc/gcp.tf
@@ -0,0 +1,9 @@
+terraform {
+	cloud {
+		organization = "tyk-k8s-demo"
+
+		workspaces {
+			name = "gke"
+		}
+	}
+}
diff --git a/.github/workflows/aks.yml b/.github/workflows/aks.yml
@@ -0,0 +1,81 @@
+name: AKS Tyk Tests
+
+# Controls when the workflow will run
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: azure
+
+jobs:
+  aks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Configure AKS credentials
+        uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Install Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+          terraform_version: "1.8.2"
+
+      - name: Install helm
+        uses: Azure/setup-helm@v4
+
+      - name: Install newman
+        run: |
+          sudo apt update && \
+          sudo apt install nodejs npm -y && \
+          npm install -g newman
+
+      - name: Copy .env file
+        run: cp .env.example .env
+
+      - name: Create Terraform Cloud descriptor
+        run: cp .github/tfc/azure.tf src/clouds/azure/terraform.cloud.tf
+
+      - name: Deploy CP tyk-cp
+        run: |
+          LICENSE="${{ secrets.DASH_LICENSE }}" \
+          MDCB_LICENSE="${{ secrets.MDCB_LICENSE }}" \
+          CLUSTER_LOCATION="${{ vars.AZURE_CLUSTER_LOCATION }}" \
+          CLUSTER_MACHINE_TYPE="${{ vars.AZURE_CLUSTER_MACHINE_TYPE }}" \
+          CLUSTER_NODE_COUNT="${{ vars.CLUSTER_NODE_COUNT }}" \
+          ./up.sh -v -c azure tyk-cp
+
+      - name: Sleep 1 minute
+        run: sleep 60
+
+      - name: Test Tyk deployment
+        run: newman run --folder gateway --env-var "deployment=tyk-cp" tyk-k8s-demo.postman_collection.json
+
+      - name: Deploy tyk-dp
+        run: |
+          TYK_WORKER_CONNECTIONSTRING=mdcb-svc-tyk-cp-tyk-mdcb.tyk.svc:9091 \
+          TYK_WORKER_ORGID=$(kubectl get secrets -n tyk tyk-operator-conf -o=jsonpath="{.data.TYK_ORG}" | base64 -d) \
+          TYK_WORKER_AUTHTOKEN=$(kubectl get secrets -n tyk tyk-operator-conf -o=jsonpath="{.data.TYK_AUTH}" | base64 -d) \
+          TYK_WORKER_USESSL=false \
+          ./up.sh --namespace tyk-dp tyk-dp
+
+      - name: Sleep 1 minute
+        run: sleep 60
+
+      - name: Test Tyk DP deployment
+        run: newman run --folder gateway --env-var "gateway.port=8081" --env-var "deployment=data-plane" tyk-k8s-demo.postman_collection.json
+
+
+      - name: Destroy tyk-cp
+        run: |
+          CLUSTER_LOCATION="${{ vars.AZURE_CLUSTER_LOCATION }}" \
+          CLUSTER_MACHINE_TYPE="${{ vars.AZURE_CLUSTER_MACHINE_TYPE }}" \
+          CLUSTER_NODE_COUNT="${{ vars.CLUSTER_NODE_COUNT }}" \
+          ./down.sh -v -c azure tyk-cp
diff --git a/.github/workflows/eks.yml b/.github/workflows/eks.yml
@@ -0,0 +1,84 @@
+name: EKS Tyk Tests
+
+# Controls when the workflow will run
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: aws
+
+jobs:
+  eks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_CLUSTER_LOCATION }}
+
+      - name: Install Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+          terraform_version: "1.8.2"
+
+      - name: Install helm
+        uses: Azure/setup-helm@v4
+
+      - name: Install newman
+        run: |
+          sudo apt update && \
+          sudo apt install nodejs npm -y && \
+          npm install -g newman
+
+      - name: Copy .env file
+        run: cp .env.example .env
+
+      - name: Create Terraform Cloud descriptor
+        run: cp .github/tfc/aws.tf src/clouds/aws/terraform.cloud.tf
+
+      - name: Deploy CP tyk-cp
+        run: |
+          LICENSE="${{ secrets.DASH_LICENSE }}" \
+          MDCB_LICENSE="${{ secrets.MDCB_LICENSE }}" \
+          CLUSTER_LOCATION="${{ vars.AWS_CLUSTER_LOCATION }}" \
+          CLUSTER_MACHINE_TYPE="${{ vars.AWS_CLUSTER_MACHINE_TYPE }}" \
+          CLUSTER_NODE_COUNT="${{ vars.CLUSTER_NODE_COUNT }}" \
+          ./up.sh -v -c aws tyk-cp
+
+      - name: Sleep 1 minute
+        run: sleep 60
+
+      - name: Test Tyk deployment
+        run: newman run --folder gateway --env-var "deployment=tyk-cp" tyk-k8s-demo.postman_collection.json
+
+      - name: Deploy tyk-dp
+        run: |
+          TYK_WORKER_CONNECTIONSTRING=mdcb-svc-tyk-cp-tyk-mdcb.tyk.svc:9091 \
+          TYK_WORKER_ORGID=$(kubectl get secrets -n tyk tyk-operator-conf -o=jsonpath="{.data.TYK_ORG}" | base64 -d) \
+          TYK_WORKER_AUTHTOKEN=$(kubectl get secrets -n tyk tyk-operator-conf -o=jsonpath="{.data.TYK_AUTH}" | base64 -d) \
+          TYK_WORKER_USESSL=false \
+          ./up.sh --namespace tyk-dp tyk-dp
+
+      - name: Sleep 1 minute
+        run: sleep 60
+
+      - name: Test Tyk DP deployment
+        run: newman run --folder gateway --env-var "gateway.port=8081" --env-var "deployment=data-plane" tyk-k8s-demo.postman_collection.json
+
+
+      - name: Destroy tyk-cp
+        run: |
+          CLUSTER_LOCATION="${{ vars.AWS_CLUSTER_LOCATION }}" \
+          CLUSTER_MACHINE_TYPE="${{ vars.AWS_CLUSTER_MACHINE_TYPE }}" \
+          CLUSTER_NODE_COUNT="${{ vars.CLUSTER_NODE_COUNT }}" \
+          ./down.sh -v -c aws tyk-cp
+
diff --git a/.github/workflows/gke.yml b/.github/workflows/gke.yml
@@ -0,0 +1,88 @@
+name: GKE Tyk Tests
+
+# Controls when the workflow will run
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: gcp
+
+jobs:
+  gke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Authenticate into gcloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_CREDENTIALS }}
+
+      - name: Install gcloud CLI
+        uses: google-github-actions/[email protected]
+
+      - name: Install gcloud k8s auth component
+        run: gcloud components install gke-gcloud-auth-plugin
+
+      - name: Install Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+          terraform_version: "1.8.2"
+
+      - name: Install helm
+        uses: Azure/setup-helm@v4
+
+      - name: Install newman
+        run: |
+          sudo apt update && \
+          sudo apt install nodejs npm -y && \
+          npm install -g newman
+
+      - name: Copy .env file
+        run: cp .env.example .env
+
+      - name: Create Terraform Cloud descriptor
+        run: cp .github/tfc/gcp.tf src/clouds/gcp/terraform.cloud.tf
+
+      - name: Deploy CP tyk-cp
+        run: |
+          LICENSE="${{ secrets.DASH_LICENSE }}" \
+          MDCB_LICENSE="${{ secrets.MDCB_LICENSE }}" \
+          GCP_PROJECT="${{ secrets.GCP_PROJECT }}" \
+          CLUSTER_LOCATION="${{ vars.GCP_CLUSTER_LOCATION }}" \
+          CLUSTER_MACHINE_TYPE="${{ vars.GCP_CLUSTER_MACHINE_TYPE }}" \
+          CLUSTER_NODE_COUNT="${{ vars.CLUSTER_NODE_COUNT }}" \
+          ./up.sh -v -c gcp tyk-cp
+
+      - name: Sleep 1 minute
+        run: sleep 60
+
+      - name: Test Tyk deployment
+        run: newman run --folder gateway --env-var "deployment=tyk-cp" tyk-k8s-demo.postman_collection.json
+
+      - name: Deploy tyk-dp
+        run: |
+          TYK_WORKER_CONNECTIONSTRING=mdcb-svc-tyk-cp-tyk-mdcb.tyk.svc:9091 \
+          TYK_WORKER_ORGID=$(kubectl get secrets -n tyk tyk-operator-conf -o=jsonpath="{.data.TYK_ORG}" | base64 -d) \
+          TYK_WORKER_AUTHTOKEN=$(kubectl get secrets -n tyk tyk-operator-conf -o=jsonpath="{.data.TYK_AUTH}" | base64 -d) \
+          TYK_WORKER_USESSL=false \
+          ./up.sh --namespace tyk-dp tyk-dp
+
+      - name: Sleep 1 minute
+        run: sleep 60
+
+      - name: Test Tyk DP deployment
+        run: newman run --folder gateway --env-var "gateway.port=8081" --env-var "deployment=data-plane" tyk-k8s-demo.postman_collection.json
+
+      - name: Destroy tyk-cp
+        run: |
+          GCP_PROJECT="${{ secrets.GCP_PROJECT }}" \
+          CLUSTER_LOCATION="${{ vars.GCP_CLUSTER_LOCATION }}" \
+          CLUSTER_MACHINE_TYPE="${{ vars.GCP_CLUSTER_MACHINE_TYPE }}" \
+          CLUSTER_NODE_COUNT="${{ vars.CLUSTER_NODE_COUNT }}" \
+          ./down.sh -v -c gcp tyk-cp
diff --git a/README.md b/README.md
@@ -6,6 +6,9 @@
 [![Clusters](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/clusters.yml/badge.svg?query=branch%3Amain)](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/clusters.yml?query=branch%3Amain)
 [![Redis](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/redis.yml/badge.svg?query=branch%3Amain)](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/redis.yml?query=branch%3Amain)
 [![Pumps](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/tyk-pumps.yml/badge.svg?query=branch%3Amain)](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/tyk-pumps.yml?query=branch%3Amain)
+[![AKS](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/aks.yml/badge.svg?query=branch%3Amain)](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/aks.yml?query=branch%3Amain)
+[![EKS](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/eks.yml/badge.svg?query=branch%3Amain)](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/eks.yml?query=branch%3Amain)
+[![GKE](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/gke.yml/badge.svg?query=branch%3Amain)](https://github.com/TykTechnologies/tyk-k8s-demo/actions/workflows/gke.yml?query=branch%3Amain)
 
 ## About
 The [tyk-k8s-demo](https://github.com/TykTechnologies/tyk-k8s-demo) library allows you to stand up an entire Tyk Stack

diff --git a/src/clouds/aws/main.tf b/src/clouds/aws/main.tf
@@ -9,7 +9,7 @@ data "aws_caller_identity" "this" {}
 module "vpc" {
 	source = "terraform-aws-modules/vpc/aws"
 
-	name                 = "pt-${var.cluster_location}-vpc"
+	name                 = "tyk-demo-${var.cluster_location}-vpc"
 	cidr                 = "10.0.0.0/16"
 	azs                  = data.aws_availability_zones.this.names
 	private_subnets      = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
@@ -19,29 +19,23 @@ module "vpc" {
 	enable_dns_hostnames = true
 }
 
-module "ebs_csi_controller_role" {
-	source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
-	create_role                   = true
-	role_name                     = "${module.eks.cluster_name}-ebs-csi-controller"
-	provider_url                  = module.eks.cluster_oidc_issuer_url
-	role_policy_arns              = ["arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"]
-	oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:ebs-csi-controller-sa"]
-}
-
 module "eks" {
-	source = "terraform-aws-modules/eks/aws"
+	source  = "terraform-aws-modules/eks/aws"
+	version = "20.8.2"
 
 	cluster_name    = "tyk-demo-${var.cluster_location}"
-	cluster_version = "1.24"
+	cluster_version = "1.29"
 
 	vpc_id     = module.vpc.vpc_id
 	subnet_ids = module.vpc.private_subnets
 
 	cluster_endpoint_public_access = true
+	enable_cluster_creator_admin_permissions = true
 }
 
 module "eks_node_groups" {
-	source = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group"
+	source   = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group"
+	version  = "20.8.2"
 
 	name            = "${module.eks.cluster_name}-np"
 	cluster_name    = module.eks.cluster_name
@@ -51,16 +45,19 @@ module "eks_node_groups" {
 	instance_types  = [var.cluster_machine_type]
 }
 
-resource "aws_eks_addon" "this" {
-	cluster_name             = module.eks.cluster_name
-	addon_name               = "aws-ebs-csi-driver"
-	resolve_conflicts   		 = "OVERWRITE"
-	service_account_role_arn = module.ebs_csi_controller_role.iam_role_arn
-	depends_on               = [module.eks, module.eks_node_groups]
+module "ebs_csi_controller_role" {
+	source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+	create_role                   = true
+	role_name                     = "${module.eks.cluster_name}-ebs-csi-controller"
+	provider_url                  = module.eks.cluster_oidc_issuer_url
+	role_policy_arns              = ["arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"]
+	oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:ebs-csi-controller-sa"]
 }
 
-data "aws_eks_cluster_auth" "this" {
-	name = module.eks.cluster_name
-
-	depends_on = [module.eks, module.eks_node_groups]
+resource "aws_eks_addon" "this" {
+	cluster_name                = module.eks.cluster_name
+	addon_name                  = "aws-ebs-csi-driver"
+	resolve_conflicts_on_create = "OVERWRITE"
+	service_account_role_arn    = module.ebs_csi_controller_role.iam_role_arn
+	depends_on                  = [module.eks, module.eks_node_groups]
 }
diff --git a/src/clouds/azure/main.tf b/src/clouds/azure/main.tf
@@ -10,7 +10,7 @@ resource "azurerm_resource_group" "this" {
 resource "azurerm_kubernetes_cluster" "this" {
 	name                = azurerm_resource_group.this.name
 	location            = azurerm_resource_group.this.location
-	kubernetes_version  = "1.22"
+	kubernetes_version  = "1.28"
 	resource_group_name = azurerm_resource_group.this.name
 	dns_prefix          = replace(azurerm_resource_group.this.name, "_", "-")
 

diff --git a/src/clouds/gcp/main.tf b/src/clouds/gcp/main.tf
@@ -5,8 +5,9 @@ provider "google" {
 data "google_client_config" "this" {}
 
 resource "google_container_cluster" "this" {
-	name     = "tyk-demo-${var.cluster_location}"
-	location = var.cluster_location
+	name                = "tyk-demo-${var.cluster_location}"
+	location            = var.cluster_location
+	deletion_protection = false
 
 	initial_node_count = var.cluster_node_count
 	node_config {