Tail journald logs into report on suite failure

Signed-off-by: Brad Davidson <[email protected]>

tests/e2e/dualstack/dualstack_test.go

@@ -195,7 +195,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/embeddedmirror/embeddedmirror_test.go

@@ -146,7 +146,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/externalip/externalip_test.go

@@ -165,7 +165,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/privateregistry/privateregistry_test.go

@@ -145,8 +145,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/rootless/rootless_test.go

@@ -167,7 +167,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, serverNodeNames))
+	} else {
 		Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/rotateca/rotateca_test.go

@@ -138,7 +138,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/s3/s3_test.go

@@ -175,8 +175,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -221,7 +221,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, serverNodeNames))
+	} else {
 		Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/snapshotrestore/snapshotrestore_test.go

@@ -317,7 +317,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/splitserver/splitserver_test.go

@@ -283,9 +283,11 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
-		allNodes := append(cpNodeNames, etcdNodeNames...)
-		allNodes = append(allNodes, agentNodeNames...)
+	allNodes := append(cpNodeNames, etcdNodeNames...)
+	allNodes = append(allNodes, agentNodeNames...)
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, allNodes))
+	} else {
 		Expect(e2e.GetCoverageReport(allNodes)).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/startup/startup_test.go

@@ -310,7 +310,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/svcpoliciesandfirewall/svcpoliciesandfirewall_test.go

@@ -128,7 +128,7 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
 		Eventually(func(g Gomega) {
 			externalIPs, _ := e2e.FetchExternalIPs(kubeConfigFile, lbSvcExt)
 			g.Expect(externalIPs).To(HaveLen(1), "more than 1 exernalIP found")
-			g.Expect(externalIPs[0]).To(Equal(serverNodeIP),"external IP does not match servernodeIP")
+			g.Expect(externalIPs[0]).To(Equal(serverNodeIP), "external IP does not match servernodeIP")
 		}, "25s", "5s").Should(Succeed())
 	})
 
@@ -154,7 +154,6 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
 			return e2e.RunCommand(cmd)
 		}, "25s", "5s").ShouldNot(ContainSubstring("10.42"))
 
-
 		// Verify connectivity to the other nodeIP does not work because of external traffic policy=local
 		for _, externalIP := range lbSvcExternalIPs {
 			if externalIP == lbSvcExtExternalIPs[0] {
@@ -250,7 +249,7 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
 		))
 
 		// Check the non working command fails because of internal traffic policy=local
-		Eventually(func() (bool) {
+		Eventually(func() bool {
 			_, err := e2e.RunCommand(nonWorkingCmd)
 			if err != nil && strings.Contains(err.Error(), "exit status") {
 				// Treat exit status as a successful condition
@@ -348,7 +347,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/tailscale/tailscale_test.go

@@ -118,7 +118,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/testutils.go

@@ -48,7 +48,7 @@ type NodeError struct {
 
 type SvcExternalIP struct {
 	IP     string `json:"ip"`
-	ipMode string `json:"ipMode"`
+	IPMode string `json:"ipMode"`
 }
 
 type ObjIP struct {
@@ -364,6 +364,19 @@ func GetJournalLogs(node string) (string, error) {
 	return RunCmdOnNode(cmd, node)
 }
 
+func TailJournalLogs(lines int, nodes []string) string {
+	logs := &strings.Builder{}
+	for _, node := range nodes {
+		cmd := fmt.Sprintf("journalctl -u k3s* --no-pager --lines=%d", lines)
+		if l, err := RunCmdOnNode(cmd, node); err != nil {
+			fmt.Fprintf(logs, "** failed to read journald log for node %s ***\n%v\n", node, err)
+		} else {
+			fmt.Fprintf(logs, "** journald log for node %s ***\n%s\n", node, l)
+		}
+	}
+	return logs.String()
+}
+
 // GetVagrantLog returns the logs of on vagrant commands that initialize the nodes and provision K3s on each node.
 // It also attempts to fetch the systemctl logs of K3s on nodes where the k3s.service failed.
 func GetVagrantLog(cErr error) string {

tests/e2e/token/token_test.go

@@ -202,7 +202,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -215,14 +215,13 @@ var _ = Describe("Verify Upgrade", Ordered, func() {
 			}, "420s", "2s").Should(Succeed())
 
 			cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'"
-			_, err = e2e.RunCommand(cmd)
-			Expect(err).NotTo(HaveOccurred())
+			res, err := e2e.RunCommand(cmd)
+			Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
 			fmt.Println("Data stored in pvc: local-path-test")
 
 			cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile
-			res, err := e2e.RunCommand(cmd)
-			Expect(err).NotTo(HaveOccurred())
-			fmt.Println(res)
+			res, err = e2e.RunCommand(cmd)
+			Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
 
 			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
@@ -245,7 +244,7 @@ var _ = Describe("Verify Upgrade", Ordered, func() {
 			Eventually(func() (string, error) {
 				cmd := "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test"
 				return e2e.RunCommand(cmd)
-			}, "180s", "2s").Should(ContainSubstring("local-path-test"))
+			}, "180s", "2s").Should(ContainSubstring("local-path-test"), "Failed to retrieve data from pvc")
 		})
 
 		It("Upgrades with no issues", func() {
@@ -385,7 +384,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/validatecluster/validatecluster_test.go

@@ -381,7 +381,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if !failed {
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}
 	if !failed || *ci {

tests/e2e/wasm/wasm_test.go

@@ -135,10 +135,12 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-	if failed && !*ci {
-		fmt.Println("FAILED!")
+	if failed {
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
+	}
+	if !failed || *ci {
 		Expect(e2e.DestroyCluster()).To(Succeed())
 		Expect(os.Remove(kubeConfigFile)).To(Succeed())
 	}