Feature: Selectively applicable conditions and metrics

pkg/snclient/check_drivesize.go

@@ -298,43 +298,92 @@ func (l *CheckDrivesize) isExcluded(drive map[string]string, excludes []string)
 	return false
 }
 
-func (l *CheckDrivesize) addMetrics(drive string, check *CheckData, usage *disk.UsageStat, magic float64) {
+// The keyword after transformation matches the metric label added for that attribute
+// So for single conditions without a group, these keywords will be translated
+// They will then be processed during metric evaluation phase and possibly raise warning/critical/unknown.
+func (l *CheckDrivesize) transformDrivePctMetrics(driveName string, check *CheckData) {
+	for _, attribute := range check.attributes {
+		if !strings.HasSuffix(attribute.name, "_pct") {
+			continue
+		}
+
+		if attribute.unit != UPercent {
+			continue
+		}
+
+		cut, _ := strings.CutSuffix(attribute.name, "_pct")
+
+		// From:      <drive> <attribute_name>
+		// To:        <drive> <cut> %
+		check.transformKeywordsUsingAttributes(`%[2]s %[1]s`, `%[2]s %[3]s %%`, []string{attribute.name}, driveName, cut)
+	}
+}
+
+// If a condition akin to metric label is added, specialized for that drive
+// Go through the conditions again, and disable generic versions of these
+func (l *CheckDrivesize) disableGenerallizedConditionsForDrive(driveName string, entry map[string]string, check *CheckData) {
+	for _, attribute := range check.attributes {
+		if !strings.HasSuffix(attribute.name, "_pct") {
+			continue
+		}
+
+		if attribute.unit != UPercent {
+			continue
+		}
+
+		cut, _ := strings.CutSuffix(attribute.name, "_pct")
+
+		// specialized: <drive> <cut> %    generalized: <attribute_name>
+		check.disableGenerallizedConditionsUsingAttributes(entry, `%[2]s %[3]s %%`, `%[1]s`, []string{attribute.name}, driveName, cut)
+
+		// specialized: <drive> <cut> %    generalized: <attribute_name> %
+		check.disableGenerallizedConditionsUsingAttributes(entry, `%[2]s %[3]s %%`, `%[1]s %%`, []string{attribute.name}, driveName, cut)
+
+		// specialized: <drive> <cut> %    generalized: <cut>
+		check.disableGenerallizedConditionsUsingAttributes(entry, `%[2]s %[3]s %%`, `%[3]s`, []string{attribute.name}, driveName, cut)
+	}
+}
+
+func (l *CheckDrivesize) addMetrics(drive map[string]string, check *CheckData, usage *disk.UsageStat, magic float64) {
 	total := usage.Total
 	if !l.freespaceIgnoreReserved {
 		total = usage.Used + usage.Free // use this total instead of usage.Total to account in the root reserved space
 	}
 
-	if check.HasThreshold("free") || check.HasThreshold("free_pct") || check.HasThreshold("free_bytes") {
+	driveFreeMetricLabel := fmt.Sprintf("%s free %%", drive["drive"])
+	if check.HasThreshold(driveFreeMetricLabel) || check.HasThreshold("free") || check.HasThreshold("free_pct") || check.HasThreshold("free_bytes") {
 		check.warnThreshold = check.TransformMultipleKeywords([]string{"free_pct", "free_bytes"}, "free", check.warnThreshold)
 		check.critThreshold = check.TransformMultipleKeywords([]string{"free_pct", "free_bytes"}, "free", check.critThreshold)
-		check.AddBytePercentMetrics("free", drive+" free", magic*float64(usage.Free), magic*float64(total))
+		perfLabel := fmt.Sprintf("%s free", drive["drive"])
+		check.AddBytePercentMetrics("free", perfLabel, magic*float64(usage.Free), magic*float64(total), drive)
+		check.processMetricsWithSpecializedKeyword("drive", perfLabel, drive)
 	}
 
-	// convert '<drive> used_pct' keywords in conditions to '<drive> used %' as that matches the metric name
-	convertDriveUsagePctMetric1 := fmt.Sprintf("%s used_pct", drive)
-	// metrics are normally added if the operand is simply 'used' , 'used_pct' , 'used_bytes' etc. and do not have a drive prefix
-	// detect conditions where the operand is named '<drive> used %', this is the default way snclient names percent usage metrics.
-	// if there is a condition using that as an operand, add usage metrics for that drive as well. during the metrics condition checking, they will take effect.
-	// this helps to check usage metrics specific to drives.
-	driveUsagePctMetric := fmt.Sprintf("%s used %%", drive)
-
-	check.warnThreshold = check.TransformMultipleKeywords([]string{convertDriveUsagePctMetric1}, driveUsagePctMetric, check.warnThreshold)
-	check.critThreshold = check.TransformMultipleKeywords([]string{convertDriveUsagePctMetric1}, driveUsagePctMetric, check.critThreshold)
-
-	if check.HasThreshold(driveUsagePctMetric) || check.HasThreshold("used") || check.HasThreshold("used_pct") || check.HasThreshold("used_bytes") {
+	driveUsagePctMetricLabel := fmt.Sprintf("%s used %%", drive["drive"])
+	if check.HasThreshold(driveUsagePctMetricLabel) || check.HasThreshold("used") || check.HasThreshold("used_pct") || check.HasThreshold("used_bytes") {
 		check.warnThreshold = check.TransformMultipleKeywords([]string{"used_pct", "used_bytes"}, "used", check.warnThreshold)
 		check.critThreshold = check.TransformMultipleKeywords([]string{"used_pct", "used_bytes"}, "used", check.critThreshold)
-		check.AddBytePercentMetrics("used", drive+" used", magic*float64(usage.Used), magic*float64(total))
+		perfLabel := fmt.Sprintf("%s used", drive["drive"])
+		check.AddBytePercentMetrics("used", perfLabel, magic*float64(usage.Used), magic*float64(total), drive)
+		check.processMetricsWithSpecializedKeyword("drive", perfLabel, drive)
 	}
-	if check.HasThreshold("inodes") || check.HasThreshold("inodes_used") || check.HasThreshold("inodes_used_pct") {
-		check.warnThreshold = check.TransformMultipleKeywords([]string{"inodes_used_pct", "inodes_used"}, "inodes", check.warnThreshold)
-		check.critThreshold = check.TransformMultipleKeywords([]string{"inodes_used_pct", "inodes_used"}, "inodes", check.critThreshold)
-		check.AddPercentMetrics("inodes", drive+" inodes", float64(usage.InodesUsed), float64(usage.InodesTotal))
+
+	driveInodesUsedPctMetricLabel := fmt.Sprintf("%s inodes_used %%", drive["drive"])
+	if check.HasThreshold(driveInodesUsedPctMetricLabel) || check.HasThreshold("inodes") || check.HasThreshold("inodes_used") || check.HasThreshold("inodes_used_pct") {
+		check.warnThreshold = check.TransformMultipleKeywords([]string{"inodes_used_pct"}, "inodes_used", check.warnThreshold)
+		check.critThreshold = check.TransformMultipleKeywords([]string{"inodes_used_pct"}, "inodes_used", check.critThreshold)
+		perfLabel := fmt.Sprintf("%s inodes_used", drive["drive"])
+		check.AddPercentMetrics("inodes_used", perfLabel, float64(usage.InodesUsed), float64(usage.InodesTotal), drive)
+		check.processMetricsWithSpecializedKeyword("drive", perfLabel, drive)
 	}
-	if check.HasThreshold("inodes_free") || check.HasThreshold("inodes_free_pct") {
+
+	driveInodesFreePctMetricLabel := fmt.Sprintf("%s inodes_free %%", drive["drive"])
+	if check.HasThreshold(driveInodesFreePctMetricLabel) || check.HasThreshold("inodes_free") || check.HasThreshold("inodes_free_pct") {
 		check.warnThreshold = check.TransformMultipleKeywords([]string{"inodes_free_pct"}, "inodes_free", check.warnThreshold)
 		check.critThreshold = check.TransformMultipleKeywords([]string{"inodes_free_pct"}, "inodes_free", check.critThreshold)
-		check.AddPercentMetrics("inodes_free", drive+" inodes free", float64(usage.InodesFree), float64(usage.InodesTotal))
+		perfLabel := fmt.Sprintf("%s inodes_free", drive["drive"])
+		check.AddPercentMetrics("inodes_free", perfLabel, float64(usage.InodesFree), float64(usage.InodesTotal), drive)
+		check.processMetricsWithSpecializedKeyword("drive", perfLabel, drive)
 	}
 }
 
@@ -392,10 +441,10 @@ func (l *CheckDrivesize) addTotal(check *CheckData) {
 	}
 
 	if check.HasThreshold("free") || check.HasThreshold("free_bytes") {
-		check.AddBytePercentMetrics("free", drive["drive"]+" free", float64(free), float64(total))
+		check.AddBytePercentMetrics("free", drive["drive"]+" free", float64(free), float64(total), map[string]string{})
 	}
 	if check.HasThreshold("used") || check.HasThreshold("used_bytes") {
-		check.AddBytePercentMetrics("used", drive["drive"]+" used", float64(used), float64(total))
+		check.AddBytePercentMetrics("used", drive["drive"]+" used", float64(used), float64(total), map[string]string{})
 	}
 }
 
@@ -452,7 +501,11 @@ func (l *CheckDrivesize) addDriveSizeDetails(check *CheckData, drive map[string]
 		return
 	}
 
-	l.addMetrics(drive["drive"], check, usage, magic)
+	l.transformDrivePctMetrics(drive["drive"], check)
+
+	l.disableGenerallizedConditionsForDrive(drive["drive"], drive, check)
+
+	l.addMetrics(drive, check, usage, magic)
 }
 
 func (l *CheckDrivesize) getFlagNames(drive map[string]string) []string {

pkg/snclient/check_drivesize_test.go

@@ -49,7 +49,7 @@ func TestCheckDrivesize(t *testing.T) {
 	res = snc.RunCheck("check_drivesize", []string{"warn=inodes>100%", "crit=inodes>100%", "folder=" + tmpFolder})
 	assert.Equalf(t, CheckExitOK, res.State, "state OK")
 	assert.Contains(t, string(res.BuildPluginOutput()), `OK - All 1 drive`, "output matches")
-	assert.Contains(t, string(res.BuildPluginOutput()), `'`+tmpFolder+` inodes'=`, "output matches")
+	assert.Contains(t, string(res.BuildPluginOutput()), `'`+tmpFolder+` inodes_used %'=`, "output matches")
 
 	StopTestAgent(t, snc)
 }

pkg/snclient/check_memory.go

@@ -168,11 +168,11 @@ func (l *CheckMemory) addMemType(check *CheckData, name string, used, free, tota
 	if check.HasThreshold("free") || check.HasThreshold("free_pct") {
 		check.warnThreshold = check.TransformMultipleKeywords([]string{"free_pct"}, "free", check.warnThreshold)
 		check.critThreshold = check.TransformMultipleKeywords([]string{"free_pct"}, "free", check.critThreshold)
-		check.AddBytePercentMetrics("free", name+"_free", float64(free), float64(total))
+		check.AddBytePercentMetrics("free", name+"_free", float64(free), float64(total), entry)
 	}
 	if check.HasThreshold("used") || check.HasThreshold("used_pct") {
 		check.warnThreshold = check.TransformMultipleKeywords([]string{"used_pct"}, "used", check.warnThreshold)
 		check.critThreshold = check.TransformMultipleKeywords([]string{"used_pct"}, "used", check.critThreshold)
-		check.AddBytePercentMetrics("used", name, float64(used), float64(total))
+		check.AddBytePercentMetrics("used", name, float64(used), float64(total), entry)
 	}
 }

pkg/snclient/check_pagefile_windows.go

@@ -85,21 +85,21 @@ func (l *CheckPagefile) addPagefile(check *CheckData, name string, data map[stri
 
 	check.listData = append(check.listData, entry)
 	if check.HasThreshold("free") {
-		check.AddBytePercentMetrics("free", name, float64(data["AllocatedBaseSize"]-data["CurrentUsage"]), float64(data["AllocatedBaseSize"]))
+		check.AddBytePercentMetrics("free", name, float64(data["AllocatedBaseSize"]-data["CurrentUsage"]), float64(data["AllocatedBaseSize"]), entry)
 	}
 	if check.HasThreshold("used") {
-		check.AddBytePercentMetrics("used", name, float64(data["CurrentUsage"]), float64(data["AllocatedBaseSize"]))
+		check.AddBytePercentMetrics("used", name, float64(data["CurrentUsage"]), float64(data["AllocatedBaseSize"]), entry)
 	}
 	if check.HasThreshold("peak") {
-		check.AddBytePercentMetrics("peak", name, float64(data["PeakUsage"]), float64(data["AllocatedBaseSize"]))
+		check.AddBytePercentMetrics("peak", name, float64(data["PeakUsage"]), float64(data["AllocatedBaseSize"]), entry)
 	}
 	if check.HasThreshold("free_pct") {
-		check.AddPercentMetrics("free_pct", name, float64(data["AllocatedBaseSize"]-data["CurrentUsage"]), float64(data["AllocatedBaseSize"]))
+		check.AddPercentMetrics("free_pct", name, float64(data["AllocatedBaseSize"]-data["CurrentUsage"]), float64(data["AllocatedBaseSize"]), entry)
 	}
 	if check.HasThreshold("used_pct") {
-		check.AddPercentMetrics("used_pct", name, float64(data["AllocatedBaseSize"]-data["CurrentUsage"]), float64(data["AllocatedBaseSize"]))
+		check.AddPercentMetrics("used_pct", name, float64(data["AllocatedBaseSize"]-data["CurrentUsage"]), float64(data["AllocatedBaseSize"]), entry)
 	}
 	if check.HasThreshold("peak_pct") {
-		check.AddPercentMetrics("used_pct", name, float64(data["AllocatedBaseSize"]-data["PeakUsage"]), float64(data["AllocatedBaseSize"]))
+		check.AddPercentMetrics("used_pct", name, float64(data["AllocatedBaseSize"]-data["PeakUsage"]), float64(data["AllocatedBaseSize"]), entry)
 	}
 }

pkg/snclient/checkdata.go

@@ -158,7 +158,10 @@ func (cd *CheckData) Finalize() (*CheckResult, error) {
 	log.Debugf("condition critical: %s", cd.critThreshold.String())
 	log.Debugf("condition       ok: %s", cd.okThreshold.String())
 	// Run thresholds once on cd.details. This is done separately than metrics or entries
-	// This can possibly set a value to cd.details[_state]
+	// cd.details are of type map[string]string,
+	// same as elements of the slice cd.listData, but there is only one per check
+	// This can possibly set a value to cd.details[_state] , influencing check state
+	log.Tracef("checking warning, critical, and ok thresholds on check details")
 	cd.Check(cd.details, cd.warnThreshold, cd.critThreshold, cd.okThreshold)
 	log.Tracef("details:")
 	logTraceASCIIMap(cd.details)
@@ -209,7 +212,8 @@ func (cd *CheckData) finalizeOutput() (*CheckResult, error) {
 			}
 
 			// each entry in the list data is individually checked
-			// This may set "_state" of each entry
+			// This can possibly set "_state" of each entry, influencing the final state
+			log.Tracef("checking warning, critical, and ok thresholds on a check entry")
 			cd.Check(entry, cd.warnThreshold, cd.critThreshold, cd.okThreshold)
 		}
 
@@ -230,11 +234,12 @@ func (cd *CheckData) finalizeOutput() (*CheckResult, error) {
 	}
 
 	cd.result.ApplyPerfSyntax(cd.perfSyntax, cd.timezone)
-
 	// Run a separate check on the macros
+	log.Tracef("checking warning, critical, and ok thresholds on check macros")
 	cd.Check(finalMacros, cd.warnThreshold, cd.critThreshold, cd.okThreshold)
 	cd.setStateFromMaps(finalMacros)
 	// Metrics are checked last, which also sets the final state
+	log.Tracef("checking warning, critical, and ok thresholds on check metrics")
 	cd.CheckMetrics(cd.warnThreshold, cd.critThreshold, cd.okThreshold)
 
 	switch {
@@ -458,21 +463,21 @@ func (cd *CheckData) Check(data map[string]string, warnCond, critCond, okCond Co
 
 	for i := range warnCond {
 		if res, ok := warnCond[i].Match(data); res && ok {
-			log.Debugf("This data '%s' matched the WARNING Condition", warnCond[i].original)
+			log.Debugf("The given data matched the WARNING condition: '%s' ", warnCond[i].DetailedString())
 			data["_state"] = fmt.Sprintf("%d", CheckExitWarning)
 		}
 	}
 
 	for i := range critCond {
 		if res, ok := critCond[i].Match(data); res && ok {
-			log.Debugf("This data '%s' matched the CRITICAL Condition", critCond[i].original)
+			log.Debugf("This given data matched the CRITICAL condition: '%s' ", critCond[i].DetailedString())
 			data["_state"] = fmt.Sprintf("%d", CheckExitCritical)
 		}
 	}
 
 	for i := range okCond {
 		if res, ok := okCond[i].Match(data); res && ok {
-			log.Debugf("This data '%s' matched the OK Condition", okCond[i].original)
+			log.Debugf("This given data matched the OK condition: '%s' ", okCond[i].DetailedString())
 			data["_state"] = fmt.Sprintf("%d", CheckExitOK)
 		}
 	}
@@ -1293,7 +1298,7 @@ func (cd *CheckData) ExpandMetricMacros(srcThreshold ConditionList, data map[str
 	return replaced
 }
 
-func (cd *CheckData) AddBytePercentMetrics(threshold, perfLabel string, val, total float64) {
+func (cd *CheckData) AddBytePercentMetrics(threshold, perfLabel string, val, total float64, entry map[string]string) {
 	percent := float64(0)
 	if threshold == "used" {
 		percent = 100
@@ -1312,6 +1317,7 @@ func (cd *CheckData) AddBytePercentMetrics(threshold, perfLabel string, val, tot
 			Critical: cd.TransformThreshold(cd.critThreshold, threshold, perfLabel, "%", "B", total),
 			Min:      &Zero,
 			Max:      &total,
+			Entry:    entry,
 		},
 		&CheckMetric{
 			Name:     pctName,
@@ -1321,11 +1327,12 @@ func (cd *CheckData) AddBytePercentMetrics(threshold, perfLabel string, val, tot
 			Critical: cd.TransformThreshold(cd.critThreshold, threshold, pctName, "B", "%", total),
 			Min:      &Zero,
 			Max:      &Hundred,
+			Entry:    entry,
 		},
 	)
 }
 
-func (cd *CheckData) AddPercentMetrics(threshold, perfLabel string, val, total float64) {
+func (cd *CheckData) AddPercentMetrics(threshold, perfLabel string, val, total float64, entry map[string]string) {
 	percent := float64(0)
 	if strings.Contains(threshold, "used") {
 		percent = 100
@@ -1335,21 +1342,76 @@ func (cd *CheckData) AddPercentMetrics(threshold, perfLabel string, val, total f
 	if total > 0 {
 		percent = val * 100 / total
 	}
+	pctName := perfLabel + " %"
 	cd.result.Metrics = append(
 		cd.result.Metrics,
 		&CheckMetric{
-			Name:          perfLabel,
+			Name:          pctName,
 			ThresholdName: threshold,
 			Unit:          "%",
 			Value:         utils.ToPrecision(percent, 1),
 			Warning:       cd.warnThreshold,
 			Critical:      cd.critThreshold,
 			Min:           &Zero,
 			Max:           &Hundred,
+			Entry:         entry,
 		},
 	)
 }
 
+// processMetricsWithSpecializedKeyword sets the Entry reference and filters Warning/Critical thresholds of the Metric object
+// the thresholds will be filtered using the special keyword. refer to that function for more details, as it uses a heurisic to determine how to filter
+//
+//nolint:unparam // this is only used in check_drivesize for now, so keyword is always "drive"
+func (cd *CheckData) processMetricsWithSpecializedKeyword(keyword, metricName string, entry map[string]string) {
+	for _, metric := range cd.result.Metrics {
+		if !strings.HasPrefix(metric.Name, metricName) {
+			continue
+		}
+		metric.Entry = entry
+		metric.Warning = metric.Warning.filterForSpecializedKeyword(keyword, entry)
+		metric.Critical = metric.Critical.filterForSpecializedKeyword(keyword, entry)
+	}
+}
+
+// transforms keywords of ok/warn/crit thresholds, using a source keyword and target keyword
+// attribute.name of the CheckData.attributes is fed into the formats as the first argument
+// formatArgs are fed as following arguments
+func (cd *CheckData) transformKeywordsUsingAttributes(keywordSourceFormat, keywordTargetFormat string, attributeNames []string, formatArgs ...any) {
+	for _, attributeName := range attributeNames {
+		args := make([]any, len(formatArgs)+1)
+		args[0] = attributeName
+		copy(args[1:], formatArgs)
+
+		keywordSource := fmt.Sprintf(keywordSourceFormat, args...)
+		keywordTarget := fmt.Sprintf(keywordTargetFormat, args...)
+
+		log.Tracef("Transforming threshold keywords, soruceKeywords: %v , targetKeyword: %s", keywordSource, keywordTarget)
+
+		cd.warnThreshold = cd.TransformMultipleKeywords([]string{keywordSource}, keywordTarget, cd.warnThreshold)
+		cd.critThreshold = cd.TransformMultipleKeywords([]string{keywordSource}, keywordTarget, cd.critThreshold)
+		cd.okThreshold = cd.TransformMultipleKeywords([]string{keywordSource}, keywordTarget, cd.okThreshold)
+	}
+}
+
+// for a given entry, looks through the ok/warn/crit thresholds, disables conditions using generallized keywords if specialized keywords is present
+// attribute.name of the CheckData.attributes is fed into the formats as the first argument
+// formatArgs are fed as following arguments
+func (cd *CheckData) disableGenerallizedConditionsUsingAttributes(entry map[string]string, specializedKeywordFormat, generallizedKeywordFormat string, attributeNames []string, formatArgs ...any) {
+	for _, attributeName := range attributeNames {
+		args := make([]any, len(formatArgs)+1)
+		args[0] = attributeName
+		copy(args[1:], formatArgs)
+
+		specializedKeyword := fmt.Sprintf(specializedKeywordFormat, args...)
+		generallizedKeyword := fmt.Sprintf(generallizedKeywordFormat, args...)
+
+		cd.warnThreshold.disableGenerallizedConditionsForEntry(entry, []string{specializedKeyword}, []string{generallizedKeyword})
+		cd.critThreshold.disableGenerallizedConditionsForEntry(entry, []string{specializedKeyword}, []string{generallizedKeyword})
+		cd.okThreshold.disableGenerallizedConditionsForEntry(entry, []string{specializedKeyword}, []string{generallizedKeyword})
+	}
+}
+
 // expand arg definitions separated by pipe symbol
 // ex.: -w|--warning
 func (cd *CheckData) expandArgDefinitions() {