contribfest exercise na 2024

Signed-off-by: Andre Aguas <[email protected]>

Makefile

@@ -481,6 +481,19 @@ terratest: # Run terratest suite
 	fi
 	cd terratest/test/ && go mod download && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) go test -v -timeout 25m -parallel=12 --tags=$(TEST_TAGS)
 
+.PHONY: chainsaw
+chainsaw:
+	mkdir -p chainsaw/kubeconfig
+	k3d kubeconfig get test-gslb1 > chainsaw/kubeconfig/eu.config
+	k3d kubeconfig get test-gslb2 > chainsaw/kubeconfig/us.config
+	@$(eval RUNNING_CLUSTERS := $(shell k3d cluster list --no-headers | grep $(CLUSTER_NAME) -c))
+	@if [ "$(RUNNING_CLUSTERS)" -lt 2 ] ; then \
+		echo -e "$(RED)Make sure you run the tests against at least two running clusters$(NC)" ;\
+		exit 1;\
+	fi
+	cd chainsaw && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) chainsaw test --config ./config.yaml --values ./values.yaml
+	rm -r chainsaw/kubeconfig
+
 .PHONY: website
 website:
 	@if [ "$(CI)" = "true" ]; then\

chainsaw/config.yaml

@@ -0,0 +1,12 @@
+apiVersion: chainsaw.kyverno.io/v1alpha2
+kind: Configuration
+metadata:
+  name: custom-config
+spec:
+  clusters:
+    eu:
+      kubeconfig: kubeconfig/eu.config
+      context: k3d-test-gslb1
+    us:
+      kubeconfig: kubeconfig/us.config
+      context: k3d-test-gslb2

chainsaw/step-templates/apply-podinfo.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: apply-podinfo
+spec:
+  try:
+  - apply:
+      file: ../../testdata/podinfo.yaml
+  - wait:
+      apiVersion: v1
+      kind: Pod
+      name: frontend-podinfo
+      for:
+        condition:
+          name: Ready
+          value: 'true'

chainsaw/step-templates/assert-dns.yaml

@@ -0,0 +1,29 @@
+---
+# asserts that both clusters resolve the domain to the correct cluster
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: assert-dns
+spec:
+  try:
+  - sleep: 
+      duration: 30s # reconcile requeue seconds + coreDNS
+  - script:
+      env:
+      - name: NAME
+        value: ($test.metadata.name)
+      - name: DNS_PORT_EU
+        value: ($values.dns_port_eu)
+      - name: DNS_PORT_US
+        value: ($values.dns_port_us)
+      content: |
+        EXPECTED=$(kubectl get dnsendpoint $NAME -n $NAMESPACE -o json | jq -r '.spec.endpoints[] | select(.dnsName | contains("localtargets")).targets | join(",")')
+        ACTUAL_EU=$(dig @localhost -p $DNS_PORT_EU "$NAME.cloud.example.com" +short | paste -sd, -)
+
+        EXPECTED_SORTED=$(echo "$EXPECTED" | tr ',' '\n' | sort | tr '\n' ',')
+        ACTUAL_EU_SORTED=$(echo "$ACTUAL_EU" | tr ',' '\n' | sort | tr '\n' ',')
+        [ "$EXPECTED_SORTED" = "$ACTUAL_EU_SORTED" ] || { echo "EU expected targets '$EXPECTED_SORTED' but found '$ACTUAL_EU_SORTED'"; exit 1; }
+
+        ACTUAL_US=$(dig @localhost -p $DNS_PORT_US "$NAME.cloud.example.com" +short | paste -sd, -)
+        ACTUAL_US_SORTED=$(echo "$ACTUAL_US" | tr ',' '\n' | sort | tr '\n' ',')
+        [ "$EXPECTED_SORTED" = "$ACTUAL_US_SORTED" ] || { echo "US expected targets '$EXPECTED_SORTED' but found '$ACTUAL_US_SORTED'"; exit 1; }

chainsaw/step-templates/assert-number-dns-records.yaml

@@ -0,0 +1,25 @@
+---
+# asserts that both clusters resolve the domain to the expected number of DNS records
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: assert-number-dns-records
+spec:
+  try:
+  - sleep: 
+      duration: 30s # reconcile requeue seconds + coreDNS
+  - script:
+      env:
+      - name: NAME
+        value: ($test.metadata.name)
+      - name: DNS_PORT_EU
+        value: ($values.dns_port_eu)
+      - name: DNS_PORT_US
+        value: ($values.dns_port_us)
+      - name: EXPECTED
+        value: ($expected_records)
+      content: |
+        ACTUAL_EU=$(dig @localhost -p $DNS_PORT_EU "$NAME.cloud.example.com" +short | wc -l)
+        [ "$EXPECTED" -eq "$ACTUAL_EU" ] || { echo "EU expected $EXPECTED targets but found $ACTUAL_EU"; exit 1; }
+        ACTUAL_US=$(dig @localhost -p $DNS_PORT_US "$NAME.cloud.example.com" +short | wc -l)
+        [ "$EXPECTED" -eq "$ACTUAL_US" ] || { echo "US expected $EXPECTED targets but found $ACTUAL_US"; exit 1; }

chainsaw/step-templates/init-istio.yaml

@@ -0,0 +1,28 @@
+---
+# Creates a namespace, a service, a pod and a gslb
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: init-ingress
+spec:
+  try:
+  - apply:
+      file: ../../testdata/namespace-istio.yaml
+  - apply:
+      file: ../../testdata/podinfo.yaml
+  - apply:
+      file: testdata/gslb.yaml
+  - assert:
+      resource:
+        apiVersion: externaldns.k8s.io/v1alpha1
+        kind: DNSEndpoint
+        metadata:
+          name: ($test.metadata.name)
+          namespace: ($namespace)
+        spec: 
+          endpoints:
+            - targets:
+                (length(@)): 2
+            - targets:
+                (length(@)): 2
+      timeout: 15s

chainsaw/step-templates/init-lbservice.yaml

@@ -0,0 +1,40 @@
+---
+# Creates a namespace, a service, a pod and a gslb
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: init-ingress
+spec:
+  try:
+  - apply:
+      file: ../../testdata/namespace.yaml
+  - apply:
+      file: ../../testdata/podinfo.yaml
+  - apply:
+      file: testdata/gslb.yaml
+  - assert:
+      resource:
+        apiVersion: v1
+        kind: Service
+        metadata:
+          name: ($test.metadata.name)
+          namespace: ($namespace)
+        status: 
+          loadBalancer:
+            ingress:
+              (length(@)): 2
+      timeout: 60s
+  - assert:
+      resource:
+        apiVersion: externaldns.k8s.io/v1alpha1
+        kind: DNSEndpoint
+        metadata:
+          name: ($test.metadata.name)
+          namespace: ($namespace)
+        spec: 
+          endpoints:
+            - targets:
+                (length(@)): 2
+            - targets:
+                (length(@)): 2
+      timeout: 15s

chainsaw/step-templates/init.yaml

@@ -0,0 +1,40 @@
+---
+# Creates a namespace, a service, a pod and a gslb
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: init-ingress
+spec:
+  try:
+  - apply:
+      file: ../../testdata/namespace.yaml
+  - apply:
+      file: ../../testdata/podinfo.yaml
+  - apply:
+      file: ./testdata/gslb.yaml
+  - assert:
+      resource:
+        apiVersion: networking.k8s.io/v1
+        kind: Ingress
+        metadata:
+          name: ($test.metadata.name)
+          namespace: ($namespace)
+        status: 
+          loadBalancer:
+            ingress:
+              (length(@)): 2
+      timeout: 60s
+  - assert:
+      resource:
+        apiVersion: externaldns.k8s.io/v1alpha1
+        kind: DNSEndpoint
+        metadata:
+          name: ($test.metadata.name)
+          namespace: ($namespace)
+        spec: 
+          endpoints:
+          - dnsName: (join('', ['localtargets-', $test.metadata.name, '.cloud.example.com']))
+            targets:
+              (length(@)): 2
+          - dnsName: (join('', [$test.metadata.name, '.cloud.example.com']))
+      timeout: 15s

chainsaw/step-templates/namespace.yaml

@@ -0,0 +1,10 @@
+---
+# Creates a namespace, a service, a pod and a gslb
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: namespace
+spec:
+  try:
+  - apply:
+      file: ../../testdata/namespace.yaml

chainsaw/step-templates/stop-podinfo.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: stop-podinfo
+spec:
+  try:
+  - delete:
+      ref:
+        apiVersion: v1
+        kind: Pod
+        name: frontend-podinfo
+  - wait:
+      apiVersion: v1
+      kind: Pod
+      name: frontend-podinfo
+      for:
+        deletion: {}

chainsaw/testdata/namespace-istio.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ($namespace)
+  labels:
+    istio-inject: enabled

chainsaw/testdata/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ($namespace)

chainsaw/testdata/podinfo.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: frontend-podinfo
+  labels:
+    app: frontend-podinfo
+spec:
+  type: ClusterIP
+  ports:
+    - port: 9898
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: frontend-podinfo
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: frontend-podinfo
+  labels:
+    app: frontend-podinfo
+spec:
+  containers:
+    - name: podinfo
+      image: "ghcr.io/stefanprodan/podinfo:5.1.1"
+      command:
+        - ./podinfo
+        - --port=9898
+      env:
+        - name: PODINFO_UI_MESSAGE
+          value: "eu"
+      ports:
+        - name: http
+          containerPort: 9898
+          protocol: TCP
+      resources:
+        requests:
+          memory: 64Mi
+          cpu: 250m
+        limits:
+          memory: 128Mi
+          cpu: 500m

chainsaw/tests/failover-playground-lbservice/chainsaw-test.yaml

@@ -0,0 +1,60 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: failover-playground-lbservice
+spec:
+  namespace: failover-playground-lbservice
+  steps:
+  - name: namespace EU
+    description: create namespace on EU cluster
+    cluster: eu
+    use:
+      template: ../../step-templates/namespace.yaml
+  - name: namespace US
+    description: create namespace on US cluster
+    cluster: us
+    use:
+      template: ../../step-templates/namespace.yaml
+  - name: init EU
+    description: setup podinfo application and GSLB on EU cluster
+    cluster: eu
+    use:
+      with:
+        bindings:
+        - name: port
+          value: 9898
+      template: ../../step-templates/init-lbservice.yaml
+  - name: init US
+    description: setup podinfo application and GSLB on US cluster
+    cluster: us
+    use:
+      with:
+        bindings:
+        - name: port
+          value: 9898
+      template: ../../step-templates/init-lbservice.yaml
+  - name: DNS resolves to EU pre failover
+    description: verify that coredns on both clusters resolves to EU
+    cluster: eu
+    use:
+      template: ../../step-templates/assert-dns.yaml
+  - name: failover
+    description: stop app on EU cluster
+    cluster: eu
+    use:
+      template: ../../step-templates/stop-podinfo.yaml
+  - name: DNS resolves to US post failover
+    description: assert that coredns on both clusters resolves to US
+    cluster: us
+    use:
+      template: ../../step-templates/assert-dns.yaml
+  - name: failback
+    description: restore application on EU cluster
+    cluster: eu
+    use:
+      template: ../../step-templates/apply-podinfo.yaml
+  - name: DNS resolves to EU post failback
+    description: assert that coredns on both clusters resolves to EU
+    cluster: eu
+    use:
+      template: ../../step-templates/assert-dns.yaml

chainsaw/tests/failover-playground-lbservice/testdata/gslb.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: k8gb.absa.oss/v1beta1
+kind: Gslb
+metadata:
+  name: failover-playground-lbservice
+spec:
+  resourceRef:
+    apiVersion: v1
+    kind: Service
+    matchLabels:
+      app: failover-playground-lbservice
+  strategy:
+    type: failover
+    dnsTtlSeconds: 5
+    primaryGeoTag: "eu"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: failover-playground-lbservice
+  labels:
+    app: failover-playground-lbservice
+spec:
+  ports:
+    - port: 9898
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: frontend-podinfo
+  type: LoadBalancer