Add a perf compare tool

Signed-off-by: Sergen Yalçın <[email protected]>

Makefile

@@ -22,7 +22,7 @@ GO_REQUIRED_VERSION = 1.21
 # Uncomment below if you need to override the version.
 # GOLANGCILINT_VERSION ?= 1.54.0
 
-GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/crddiff $(GO_PROJECT)/cmd/buildtagger $(GO_PROJECT)/cmd/updoc $(GO_PROJECT)/cmd/ttr $(GO_PROJECT)/cmd/perf $(GO_PROJECT)/cmd/linter/lint-provider-family
+GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/crddiff $(GO_PROJECT)/cmd/buildtagger $(GO_PROJECT)/cmd/updoc $(GO_PROJECT)/cmd/ttr $(GO_PROJECT)/cmd/perf $(GO_PROJECT)/cmd/perfcompare $(GO_PROJECT)/cmd/linter/lint-provider-family
 GO_LDFLAGS += -X $(GO_PROJECT)/internal/version.Version=$(VERSION)
 GO_SUBDIRS += cmd internal
 GO111MODULE = on

cmd/perf/internal/common/common.go

@@ -17,7 +17,7 @@ package common
 
 import (
 	"fmt"
-	"strings"
+	"io"
 	"time"
 
 	"github.com/pkg/errors"
@@ -112,10 +112,18 @@ func (r Result) Print() {
 	log.Info(fmt.Sprintf("Peak %s: %f %s \n", r.Metric, r.Peak, r.MetricUnit))
 }
 
-func (r Result) PrintYaml(b *strings.Builder) {
+// PrintYaml reports the results in a yaml format
+func (r Result) PrintYaml(b io.Writer, yamlTag string) error {
 	if r.PodName != "" {
-		b.WriteString(fmt.Sprintf("Pod: %s\n", r.PodName))
+		if _, err := fmt.Fprintf(b, "Pod: %s\n", r.PodName); err != nil {
+			return errors.Wrap(err, "error printing pod data")
+		}
+	}
+	if _, err := fmt.Fprintf(b, "average_%s: %f\n", yamlTag, r.Average); err != nil {
+		return errors.Wrap(err, "error printing average data")
+	}
+	if _, err := fmt.Fprintf(b, "peak_%s: %f\n", yamlTag, r.Peak); err != nil {
+		return errors.Wrap(err, "error printing peak data")
 	}
-	b.WriteString(fmt.Sprintf("Average%s: %f\n", r.Metric, r.Average))
-	b.WriteString(fmt.Sprintf("Peak%s: %f\n", r.Metric, r.Peak))
+	return nil
 }

cmd/perf/internal/quantify.go

@@ -16,6 +16,7 @@
 package internal
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os"
@@ -111,7 +112,7 @@ func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
 }
 
 // processPods calculated metrics for provider pods
-func (o *QuantifyOptions) processPods(timeToReadinessResults []common.Result) error {
+func (o *QuantifyOptions) processPods(timeToReadinessResults []common.Result) error { //nolint:gocyclo // sequential flow easier to follow
 	// Initialize aggregated results
 	var aggregatedMemoryResult = &common.Result{Metric: "Memory", MetricUnit: "Bytes"}
 	var aggregatedCPURateResult = &common.Result{Metric: "CPU", MetricUnit: "Rate"}
@@ -162,10 +163,16 @@ func (o *QuantifyOptions) processPods(timeToReadinessResults []common.Result) er
 
 	if o.yamlOutput {
 		for _, timeToReadinessResult := range timeToReadinessResults {
-			b := strings.Builder{}
-			timeToReadinessResult.PrintYaml(&b)
-			aggregatedMemoryResult.PrintYaml(&b)
-			aggregatedCPURateResult.PrintYaml(&b)
+			b := &bytes.Buffer{}
+			if err := timeToReadinessResult.PrintYaml(b, "time_to_readiness"); err != nil {
+				return errors.Wrap(err, "cannot print ttr data")
+			}
+			if err := aggregatedMemoryResult.PrintYaml(b, "memory"); err != nil {
+				return errors.Wrap(err, "cannot print memory data")
+			}
+			if err := aggregatedCPURateResult.PrintYaml(b, "cpu"); err != nil {
+				return errors.Wrap(err, "cannot print cpu data")
+			}
 
 			f, err := os.Create("results.yaml")
 			if err != nil {

cmd/perfcompare/main.go

@@ -0,0 +1,103 @@
+// Copyright 2024 The Crossplane Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package main
+package main
+
+import (
+	"math"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/alecthomas/kingpin.v2"
+	"gopkg.in/yaml.v2"
+)
+
+type applicationPerformance struct {
+	AverageTimeToReadiness float64 `yaml:"average_time_to_readiness"`
+	PeakTimeToReadiness    float64 `yaml:"peak_time_to_readiness"`
+	AverageMemory          float64 `yaml:"average_memory"`
+	PeakMemory             float64 `yaml:"peak_memory"`
+	AverageCPU             float64 `yaml:"average_cpu"`
+	PeakCPU                float64 `yaml:"peak_cpu"`
+}
+
+// Compare two applicationPerformance structs and print the differences with a threshold check
+func (ap applicationPerformance) Compare(newData applicationPerformance, threshold float64) {
+	log.Infoln("Comparing Performance:")
+	compareField("Average Time to Readiness", ap.AverageTimeToReadiness, newData.AverageTimeToReadiness, threshold)
+	compareField("Peak Time to Readiness", ap.PeakTimeToReadiness, newData.PeakTimeToReadiness, threshold)
+	compareField("Average Memory", ap.AverageMemory, newData.AverageMemory, threshold)
+	compareField("Peak Memory", ap.PeakMemory, newData.PeakMemory, threshold)
+	compareField("Average CPU", ap.AverageCPU, newData.AverageCPU, threshold)
+	compareField("Peak CPU", ap.PeakCPU, newData.PeakCPU, threshold)
+}
+
+func compareField(fieldName string, oldData, newData float64, threshold float64) {
+	diff := newData - oldData
+	percentChange := diff / oldData * 100
+	diff = math.Round(diff*100) / 100
+	percentChange = math.Round(percentChange*100) / 100
+
+	// Print the comparison result with 2 decimal places
+	log.Infof("%s: Old Data = %.2f, New Data = %.2f, Difference = %.2f (%.2f%% change)\n", fieldName, oldData, newData, diff, percentChange)
+
+	// Check if the percent change exceeds the threshold
+	if percentChange > threshold*100 {
+		log.Warnf("Attention: %s increased by more than %.0f%% -> %f\n", fieldName, threshold*100, percentChange)
+	}
+}
+
+// Read YAML file and unmarshal into applicationPerformance struct
+func readYAMLFile(filename string) (applicationPerformance, error) {
+	data, err := os.ReadFile(filepath.Clean(filename))
+	if err != nil {
+		return applicationPerformance{}, err
+	}
+
+	var performance applicationPerformance
+	err = yaml.Unmarshal(data, &performance)
+	if err != nil {
+		return applicationPerformance{}, err
+	}
+
+	return performance, nil
+}
+
+func main() {
+	// Command-line argument parsing
+	app := kingpin.New("perf-compare", "A tool to compare application performance data from YAML files.")
+	old := app.Flag("old", "Path to the old results as YAML file.").Short('o').Required().String()
+	current := app.Flag("new", "Path to the new results as YAML file.").Short('n').Required().String()
+	threshold := app.Flag("threshold", "The threshold for the performance comparison like 0.10 for %10.").Short('t').Required().Float64()
+
+	if _, err := app.Parse(os.Args[1:]); err != nil {
+		log.Fatal(err)
+	}
+
+	// Read data from the YAML files
+	oldData, err := readYAMLFile(*old)
+	if err != nil {
+		log.Fatalf("Failed to read old data: %v", err)
+	}
+
+	newData, err := readYAMLFile(*current)
+	if err != nil {
+		log.Fatalf("Failed to read new data: %v", err)
+	}
+
+	// Compare the two datasets
+	oldData.Compare(newData, *threshold)
+}

cmd/perfcompare/testdata/newData.yaml

@@ -0,0 +1,6 @@
+average_time_to_readiness: 5.800000
+peak_time_to_readiness: 11.000000
+average_memory: 33800000.00000
+peak_memory: 44000000.000000
+average_cpu: 65.000000
+peak_cpu: 85.000000

cmd/perfcompare/testdata/oldData.yaml

@@ -0,0 +1,6 @@
+average_time_to_readiness: 5.000000
+peak_time_to_readiness: 10.000000
+average_memory: 33000000.00000
+peak_memory: 40000000.000000
+average_cpu: 60.000000
+peak_cpu: 70.000000