GR null route-map: run many times test to catch the bug

- debug in watchfrr - capture coredump on bgp crashes - increase prefix scale on ipv4 Signed-off-by: karampok <[email protected]>
metallb · Nov 21, 2024 · ea7d3ce · ea7d3ce
1 parent 3ac4d9f
commit ea7d3ce
Show file tree

Hide file tree

Showing 11 changed files with 403 additions and 35 deletions.
diff --git a/Makefile b/Makefile
@@ -238,7 +238,7 @@ $(APIDOCSGEN): $(LOCALBIN)
 
 .PHONY: e2etests
 e2etests: ginkgo kubectl
-	$(GINKGO) -v $(GINKGO_ARGS) --timeout=3h ./e2etests -- --kubectl=$(KUBECTL) $(TEST_ARGS)
+	$(GINKGO) -v $(GINKGO_ARGS) --repeat=2 --timeout=3h ./e2etests -- --kubectl=$(KUBECTL) $(TEST_ARGS)
 
 
 .PHONY: kind-cluster
@@ -272,6 +272,9 @@ KIND_EXPORT_LOGS ?=/tmp/kind_logs
 .PHONY: kind-export-logs
 kind-export-logs:
 	$(LOCALBIN)/kind export logs --name ${KIND_CLUSTER_NAME} ${KIND_EXPORT_LOGS}
+	for node in $(shell docker ps -q); do \
+	  docker cp "$$node:/home" "$(KIND_EXPORT_LOGS)/home-$$node"; \
+	done
 
 .PHONY: generate-all-in-one
 generate-all-in-one: manifests kustomize ## Create manifests

diff --git a/config/all-in-one/frr-k8s-prometheus.yaml b/config/all-in-one/frr-k8s-prometheus.yaml
@@ -1166,10 +1166,25 @@ spec:
         - mountPath: /etc/frr_reloader
           name: reloader
       - command:
-        - /bin/sh
-        - -c
+        - /bin/bash
+        - -cx
         - |
-          /sbin/tini -- /usr/lib/frr/docker-start &
+          if [ -r "/lib/lsb/init-functions" ]; then
+                  . /lib/lsb/init-functions
+          else
+                  log_success_msg() {
+                          echo "$@"
+                  }
+                  log_warning_msg() {
+                          echo "$@" >&2
+                  }
+                  log_failure_msg() {
+                          echo "$@" >&2
+                  }
+          fi
+
+          source /usr/lib/frr/frrcommon.sh
+          /usr/lib/frr/watchfrr -l 7 $(daemon_list) &
           attempts=0
           until [[ -f /etc/frr/frr.log || $attempts -eq 60 ]]; do
             sleep 1
@@ -1203,6 +1218,8 @@ spec:
             port: 7573
           periodSeconds: 5
         volumeMounts:
+        - mountPath: /tmp/cores
+          name: core-path
         - mountPath: /var/run/frr
           name: frr-sockets
         - mountPath: /etc/frr
@@ -1280,6 +1297,9 @@ spec:
         key: node-role.kubernetes.io/control-plane
         operator: Exists
       volumes:
+      - hostPath:
+          path: /home/core-dump
+        name: core-path
       - emptyDir: {}
         name: frr-sockets
       - configMap:

diff --git a/config/all-in-one/frr-k8s.yaml b/config/all-in-one/frr-k8s.yaml
@@ -1135,10 +1135,25 @@ spec:
         - mountPath: /etc/frr_reloader
           name: reloader
       - command:
-        - /bin/sh
-        - -c
+        - /bin/bash
+        - -cx
         - |
-          /sbin/tini -- /usr/lib/frr/docker-start &
+          if [ -r "/lib/lsb/init-functions" ]; then
+                  . /lib/lsb/init-functions
+          else
+                  log_success_msg() {
+                          echo "$@"
+                  }
+                  log_warning_msg() {
+                          echo "$@" >&2
+                  }
+                  log_failure_msg() {
+                          echo "$@" >&2
+                  }
+          fi
+
+          source /usr/lib/frr/frrcommon.sh
+          /usr/lib/frr/watchfrr -l 7 $(daemon_list) &
           attempts=0
           until [[ -f /etc/frr/frr.log || $attempts -eq 60 ]]; do
             sleep 1
@@ -1172,6 +1187,8 @@ spec:
             port: 7573
           periodSeconds: 5
         volumeMounts:
+        - mountPath: /tmp/cores
+          name: core-path
         - mountPath: /var/run/frr
           name: frr-sockets
         - mountPath: /etc/frr
@@ -1249,6 +1266,9 @@ spec:
         key: node-role.kubernetes.io/control-plane
         operator: Exists
       volumes:
+      - hostPath:
+          path: /home/core-dump
+        name: core-path
       - emptyDir: {}
         name: frr-sockets
       - configMap:

diff --git a/config/frr-k8s/frr-k8s.yaml b/config/frr-k8s/frr-k8s.yaml
@@ -103,6 +103,8 @@ spec:
           - name: TINI_SUBREAPER
             value: "true"
         volumeMounts:
+          - mountPath: /tmp/cores
+            name: core-path
           - name: frr-sockets
             mountPath: /var/run/frr
           - name: frr-conf
@@ -111,10 +113,25 @@ spec:
         # If the log file isn't created in 60 seconds the tail fails and the container is restarted.
         # This workaround is needed to have the frr logs as part of kubectl logs -c frr < k8s-frr-podname >.
         command:
-          - /bin/sh
-          - -c
+          - /bin/bash
+          - -cx
           - |
-            /sbin/tini -- /usr/lib/frr/docker-start &
+            if [ -r "/lib/lsb/init-functions" ]; then
+                    . /lib/lsb/init-functions
+            else
+                    log_success_msg() {
+                            echo "$@"
+                    }
+                    log_warning_msg() {
+                            echo "$@" >&2
+                    }
+                    log_failure_msg() {
+                            echo "$@" >&2
+                    }
+            fi
+
+            source /usr/lib/frr/frrcommon.sh
+            /usr/lib/frr/watchfrr -l 7 $(daemon_list) &
             attempts=0
             until [[ -f /etc/frr/frr.log || $attempts -eq 60 ]]; do
               sleep 1
@@ -169,6 +186,9 @@ spec:
         key: node-role.kubernetes.io/control-plane
         operator: Exists
       volumes:
+        - name: core-path
+          hostPath:
+            path: /home/core-dump
         - name: frr-sockets
           emptyDir: {}
         - name: frr-startup

diff --git a/e2etests/go.work.sum b/e2etests/go.work.sum
@@ -818,6 +818,7 @@ github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0Gq
 github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
 github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI=
 github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -1517,8 +1518,6 @@ k8s.io/code-generator v0.29.3/go.mod h1:x47ofBhN4gxYFcxeKA1PYXeaPreAGaDN85Y/lNUs
 k8s.io/code-generator v0.30.2/go.mod h1:RQP5L67QxqgkVquk704CyvWFIq0e6RCMmLTXxjE8dVA=
 k8s.io/code-generator v0.31.0/go.mod h1:84y4w3es8rOJOUUP1rLsIiGlO1JuEaPFXQPA9e/K6U0=
 k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M=
-k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs=
-k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo=
 k8s.io/component-helpers v0.28.3 h1:te9ieTGzcztVktUs92X53P6BamAoP73MK0qQP0WmDqc=
 k8s.io/component-helpers v0.28.3/go.mod h1:oJR7I9ist5UAQ3y/CTdbw6CXxdMZ1Lw2Ua/EZEwnVLs=
 k8s.io/controller-manager v0.26.0/go.mod h1:GxUYtQDBE/RHh7AnZSZqwi2xBPIXlOaWsnDLflKGYrE=

diff --git a/e2etests/pkg/config/from_containers.go b/e2etests/pkg/config/from_containers.go
@@ -3,6 +3,7 @@
 package config
 
 import (
+	"fmt"
 	"net"
 
 	frrk8sv1beta1 "github.com/metallb/frr-k8s/api/v1beta1"
@@ -136,3 +137,12 @@ func ContainersForVRF(frrs []*frrcontainer.FRR, vrf string) []*frrcontainer.FRR
 	}
 	return res
 }
+
+func ContainerByName(frrs []*frrcontainer.FRR, name string) (*frrcontainer.FRR, error) {
+	for _, f := range frrs {
+		if f.Name == name {
+			return f, nil
+		}
+	}
+	return nil, fmt.Errorf("container with name %s not found", name)
+}
diff --git a/e2etests/pkg/k8s/pods.go b/e2etests/pkg/k8s/pods.go
@@ -41,6 +41,46 @@ func FRRK8sPods(cs clientset.Interface) ([]*corev1.Pod, error) {
 	}
 	return res, nil
 }
+func RestartFRRK8sPodForNode(cs clientset.Interface, nodeName string) error {
+	oldPod, err := podForNode(cs, nodeName)
+	if err != nil {
+		return err
+	}
+	if err := cs.CoreV1().Pods(FRRK8sNamespace).Delete(context.Background(), oldPod.Name, metav1.DeleteOptions{}); err != nil {
+		return err
+	}
+
+	return wait.PollUntilContextTimeout(context.Background(), 10*time.Second,
+		120*time.Second, false, func(context.Context) (bool, error) {
+			newPod, err := podForNode(cs, nodeName)
+			if err != nil {
+				return false, err
+			}
+
+			if newPod.Name == oldPod.Name {
+				return false, fmt.Errorf("pod was not deleted")
+			}
+
+			if !podIsRunningAndReady(newPod) {
+				return false, nil
+			}
+
+			return true, nil
+		})
+}
+
+func podForNode(cs clientset.Interface, nodeName string) (*corev1.Pod, error) {
+	pods, err := FRRK8sPods(cs)
+	if err != nil {
+		return nil, err
+	}
+	for _, p := range pods {
+		if p.Spec.NodeName == nodeName {
+			return p, nil
+		}
+	}
+	return nil, fmt.Errorf("no frr-k8s pod found on node %s", nodeName)
+}
 
 func RestartFRRK8sPods(cs clientset.Interface) error {
 	pods, err := FRRK8sPods(cs)