Skip to content

Commit

Permalink
feat(slo): some more refactoring
Browse files Browse the repository at this point in the history 
Signed-off-by: Hy3n4 <[email protected]>
  • Loading branch information
@Hy3n4
Hy3n4 committed Nov 1, 2023
1 parent b082e32 commit d8a055f
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 48 deletions.
50 changes: 32 additions & 18 deletions internal/controller/openslo/slo_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,77 +206,88 @@ func (r *SLOReconciler) createPrometheusRule(slo *openslov1.SLO, sli *openslov1.
defaultRateWindow := "1m"
//burnRateTimeWindows := []string{"1h", "6h", "3d"}
sloTimeWindowDuration := string(slo.Spec.TimeWindow[0].Duration)
l := utils.LabelGeneratorParams{Slo: slo, Sli: sli}
m := utils.MetricLabelParams{Slo: slo, Sli: sli}

targetVector.Record = fmt.Sprintf("osko_slo_target")
targetVector.Expr = intstr.Parse(fmt.Sprintf("vector(%s)", slo.Spec.Objectives[0].Target))
l.TimeWindow = sloTimeWindowDuration
targetVector.Labels = l.NewMetricLabelGenerator()
m.TimeWindow = sloTimeWindowDuration
targetVector.Labels = m.NewMetricLabelGenerator()

// for now, total and good are required. bad is optional and is calculated as (total - good) if not provided
// TODO: validate that the SLO budgeting method is Occurrences and that the SLIs are all ratio metrics in other case throw an error
targetVectorConfig := utils.RuleConfig{
Record: "slo_target",
Expr: "",
TimeWindow: sloTimeWindowDuration,
Slo: slo,
Sli: sli,
MetricLabelCompiler: &m,
}

totalRule28Config := utils.RuleConfig{
RuleName: "sli_ratio_total",
Record: "sli_ratio_total",
Expr: "sum(increase(%s[%s]))",
TimeWindow: sloTimeWindowDuration,
Slo: slo,
Sli: sli,
LabelGenerator: l,
MetricLabelCompiler: &m,
}

goodRule28Config := utils.RuleConfig{
RuleName: "sli_ratio_total",
Record: "sli_ratio_total",
Expr: "sum(increase(%s[%s]))",
TimeWindow: sloTimeWindowDuration,
Slo: slo,
Sli: sli,
LabelGenerator: l,
MetricLabelCompiler: &m,
}

badRule28Config := utils.RuleConfig{
RuleName: "sli_ratio_total",
Record: "sli_ratio_total",
Expr: "sum(increase(%s[%s]))",
TimeWindow: sloTimeWindowDuration,
Slo: slo,
Sli: sli,
LabelGenerator: l,
MetricLabelCompiler: &m,
}

totalRuleConfig := utils.RuleConfig{
RuleName: "sli_ratio_total",
Record: "sli_ratio_total",
Expr: "sum(increase(%s[%s]))",
TimeWindow: defaultRateWindow,
Slo: slo,
Sli: sli,
SupportiveRule: &totalRule28Config,
LabelGenerator: l,
MetricLabelCompiler: nil,
MetricLabelCompiler: &m,
}

goodRuleConfig := utils.RuleConfig{
RuleName: "sli_ratio_good",
Record: "sli_ratio_good",
Expr: "sum(increase(%s[%s]))",
TimeWindow: defaultRateWindow,
Slo: slo,
Sli: sli,
SupportiveRule: &goodRule28Config,
LabelGenerator: l,
MetricLabelCompiler: nil,
MetricLabelCompiler: &m,
}

badRuleConfig := utils.RuleConfig{
RuleName: "sli_ratio_bad",
Record: "sli_ratio_bad",
Expr: "sum(increase(%s[%s]))",
TimeWindow: defaultRateWindow,
Slo: slo,
Sli: sli,
SupportiveRule: &badRule28Config,
LabelGenerator: l,
MetricLabelCompiler: nil,
MetricLabelCompiler: &m,
}

errorBudgetRuleConfig := utils.BudgetRuleConfig{
Record: "error_budget_available",
Slo: slo,
Sli: sli,
TargetRuleConfig: &targetVectorConfig,
TotalRuleConfig: &totalRuleConfig,
BadRuleConfig: &badRuleConfig,
}

configs := []utils.RuleConfig{
Expand All @@ -291,6 +302,9 @@ func (r *SLOReconciler) createPrometheusRule(slo *openslov1.SLO, sli *openslov1.
monitoringRules = append(monitoringRules, supportiveRule)
}

monitoringRules = append(monitoringRules, targetVectorConfig.NewTargetRule())
monitoringRules = append(monitoringRules, errorBudgetRuleConfig.NewBudgetRule())

rule := &monitoringv1.PrometheusRule{
TypeMeta: metav1.TypeMeta{
APIVersion: "monitoring.coreos.com/v1",
Expand Down
93 changes: 63 additions & 30 deletions internal/utils/common_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,14 @@ import (
"context"
"fmt"
openslov1 "github.com/oskoperator/osko/apis/openslo/v1"
v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"reflect"
"sigs.k8s.io/controller-runtime/pkg/client"
"time"
)

type LabelGeneratorParams struct {
Slo *openslov1.SLO
Sli *openslov1.SLI
TimeWindow string
}

type MetricLabelParams struct {
Slo *openslov1.SLO
Sli *openslov1.SLI
Expand All @@ -28,17 +22,25 @@ type MetricLabelParams struct {
type RuleConfig struct {
Sli *openslov1.SLI
Slo *openslov1.SLO
BaseRule *v1.Rule
BaseRule *monitoringv1.Rule
RuleType string
RuleName string
Record string
Expr string
RateWindow string
TimeWindow string
LabelGenerator LabelGeneratorParams
SupportiveRule *RuleConfig
MetricLabelCompiler *MetricLabelParams
}

type BudgetRuleConfig struct {
Record string
Sli *openslov1.SLI
Slo *openslov1.SLO
TotalRuleConfig *RuleConfig
BadRuleConfig *RuleConfig
TargetRuleConfig *RuleConfig
}

// UpdateCondition checks if the condition of the given type is already in the slice
// if the condition already exists and has the same status, return the unmodified conditions
// if the condition exists and has a different status, remove it and add the new one
Expand Down Expand Up @@ -89,9 +91,9 @@ func UpdateStatus(ctx context.Context, slo *openslov1.SLO, r client.Client, cond
return r.Status().Update(ctx, slo)
}

func (m MetricLabelParams) NewMetricLabelCompiler(rule *v1.Rule, window string) string {
func (m MetricLabelParams) NewMetricLabelCompiler(rule *monitoringv1.Rule, window string) string {
labelString := ""
emptyRule := v1.Rule{}
emptyRule := monitoringv1.Rule{}
if !reflect.DeepEqual(rule, emptyRule) {
windowVal := string(m.Slo.Spec.TimeWindow[0].Duration)
if window != "" {
Expand All @@ -110,45 +112,76 @@ func (m MetricLabelParams) NewMetricLabelCompiler(rule *v1.Rule, window string)
return labelString
}

func (l LabelGeneratorParams) NewMetricLabelGenerator() map[string]string {
window := string(l.Slo.Spec.TimeWindow[0].Duration)
if l.TimeWindow != "" {
window = l.TimeWindow
func (m MetricLabelParams) NewMetricLabelGenerator() map[string]string {
window := string(m.Slo.Spec.TimeWindow[0].Duration)
if m.TimeWindow != "" {
window = m.TimeWindow
}
return map[string]string{
"sli_name": l.Sli.Name,
"slo_name": l.Slo.Name,
"service": l.Slo.Spec.Service,
"sli_name": m.Sli.Name,
"slo_name": m.Slo.Name,
"service": m.Slo.Spec.Service,
"window": window,
}
}

func (c RuleConfig) NewRatioRule(window string) (v1.Rule, v1.Rule) {
rule := v1.Rule{}
rule.Record = fmt.Sprintf("osko_%s", c.RuleName)
func (c RuleConfig) NewRatioRule(window string) (monitoringv1.Rule, monitoringv1.Rule) {
rule := monitoringv1.Rule{}
rule.Record = fmt.Sprintf("osko_%s", c.Record)
expr := fmt.Sprintf(c.Expr, c.Sli.Spec.RatioMetric.Total.MetricSource.Spec, window)
if c.RuleName != "bad" && c.Expr != "" {
if c.Record != "bad" && c.Expr != "" {
expr = fmt.Sprintf("sum(increase(%s[%s]))", c.Sli.Spec.RatioMetric.Total.MetricSource.Spec, window)
}
rule.Expr = intstr.Parse(expr)
c.TimeWindow = window
c.LabelGenerator.TimeWindow = c.TimeWindow
rule.Labels = c.LabelGenerator.NewMetricLabelGenerator()
c.MetricLabelCompiler.TimeWindow = c.TimeWindow
rule.Labels = c.MetricLabelCompiler.NewMetricLabelGenerator()

supportiveRule := c.NewSupportiveRule(window, rule)

return rule, supportiveRule
}

func (c RuleConfig) NewSupportiveRule(window string, baseRule v1.Rule) v1.Rule {
rule := v1.Rule{}
rule.Record = fmt.Sprintf("osko_%s", c.RuleName)
func (c RuleConfig) NewSupportiveRule(window string, baseRule monitoringv1.Rule) monitoringv1.Rule {
rule := monitoringv1.Rule{}
rule.Record = fmt.Sprintf("osko_%s", c.Record)
labels := c.SupportiveRule.MetricLabelCompiler.NewMetricLabelCompiler(&baseRule, baseRule.Labels["window"])
expr := fmt.Sprintf("sum(increase(%s{%s}[%s])) by (service, sli_name, slo_name)", baseRule.Record, labels, c.SupportiveRule.TimeWindow)
rule.Expr = intstr.Parse(expr)

c.LabelGenerator.TimeWindow = c.SupportiveRule.TimeWindow
rule.Labels = c.LabelGenerator.NewMetricLabelGenerator()
c.MetricLabelCompiler.TimeWindow = c.SupportiveRule.TimeWindow
rule.Labels = c.MetricLabelCompiler.NewMetricLabelGenerator()

return rule
}

func (c RuleConfig) NewTargetRule() monitoringv1.Rule {
rule := monitoringv1.Rule{}
rule.Record = fmt.Sprintf("osko_%s", c.Record)
rule.Expr = intstr.Parse(fmt.Sprintf("vector(%s)", c.Slo.Spec.Objectives[0].Target))
return rule
}

func (b BudgetRuleConfig) NewBudgetRule() monitoringv1.Rule {
rule := monitoringv1.Rule{}
rule.Record = fmt.Sprintf("osko_%s", b.Record)
expr := fmt.Sprintf("(1 - %s{%s}) * (%s{%s} - %s{%s})",
b.TargetRuleConfig.Record,
b.TargetRuleConfig.MetricLabelCompiler.NewMetricLabelCompiler(nil, ""),
b.TotalRuleConfig.Record,
b.TotalRuleConfig.MetricLabelCompiler.NewMetricLabelCompiler(nil, ""),
b.BadRuleConfig.Record,
b.BadRuleConfig.MetricLabelCompiler.NewMetricLabelCompiler(nil, ""),
)
rule.Expr = intstr.Parse(expr)
return rule
}

func GetRatioRule(ruleName string, monitoringRules []monitoringv1.Rule) monitoringv1.Rule {
for _, rule := range monitoringRules {
if rule.Record == ruleName {
return rule
}
}
return monitoringv1.Rule{}
}

0 comments on commit d8a055f

Please sign in to comment.