Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add new types and functions to pillar needed by newlog and edge-view #4449

Merged
merged 3 commits into from
Nov 26, 2024
Merged

Add new types and functions to pillar needed by newlog and edge-view #4449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c0697e7
add new remote log level configs and values
europaul Nov 26, 2024
047e07d
new log file prefixes and metrics for device logs
europaul Nov 26, 2024
64218a3
add function to get timestamp from gzip file name
europaul Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 36 additions & 23 deletions pkg/pillar/types/global.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,14 +287,18 @@ const (
ConsoleAccess GlobalSettingKey = "debug.enable.console"
// Shim VM VNC access global setting key
VncShimVMAccess GlobalSettingKey = "debug.enable.vnc.shim.vm"
// DefaultLogLevel global setting key
// DefaultLogLevel default level of logs produced by EVE microservices
DefaultLogLevel GlobalSettingKey = "debug.default.loglevel"
// DefaultRemoteLogLevel global setting key
// DefaultRemoteLogLevel default level of logs sent by EVE microservices to the controller
DefaultRemoteLogLevel GlobalSettingKey = "debug.default.remote.loglevel"
// SyslogLogLevel global setting key
// SyslogLogLevel level of the produced syslog messages
SyslogLogLevel GlobalSettingKey = "debug.syslog.loglevel"
// KernelLogLevel global setting key
// SyslogRemoteLogLevel level of the syslog messages sent to the controller
SyslogRemoteLogLevel GlobalSettingKey = "debug.syslog.remote.loglevel"
// KernelLogLevel level of the produced kernel messages
KernelLogLevel GlobalSettingKey = "debug.kernel.loglevel"
// KernelRemoteLogLevel level of the kernel messages sent to the controller
KernelRemoteLogLevel GlobalSettingKey = "debug.kernel.remote.loglevel"
eriknordmark marked this conversation as resolved.
Show resolved Hide resolved
// FmlCustomResolution global setting key
FmlCustomResolution GlobalSettingKey = "app.fml.resolution"

Expand Down Expand Up @@ -380,17 +384,19 @@ var (
// SyslogKernelLogLevelNum is a number representation of syslog/kernel
// loglevels.
SyslogKernelLogLevelNum = map[string]uint32{
"emerg": 0,
"alert": 1,
"crit": 2,
"critical": 2,
"err": 3,
"error": 3,
"warning": 4,
"warn": 4,
"notice": 5,
"info": 6,
"debug": 7,
"none": 0,
eriknordmark marked this conversation as resolved.
Show resolved Hide resolved
"emerg": 1,
"alert": 2,
"crit": 3,
"critical": 3,
"err": 4,
"error": 4,
"warning": 5,
"warn": 5,
"notice": 6,
"info": 7,
"debug": 8,
"all": 99,
}
// SyslogKernelDefaultLogLevel is a default loglevel for syslog and kernel.
SyslogKernelDefaultLogLevel = "info"
Expand Down Expand Up @@ -982,15 +988,17 @@ func NewConfigItemSpecMap() ConfigItemSpecMap {

// Add String Items
configItemSpecMap.AddStringItem(SSHAuthorizedKeys, "", blankValidator)
configItemSpecMap.AddStringItem(DefaultLogLevel, "info", validateLogrusLevel)
configItemSpecMap.AddStringItem(DefaultRemoteLogLevel, "info", validateLogrusLevel)
configItemSpecMap.AddStringItem(DefaultLogLevel, "info", validateLogLevel)
configItemSpecMap.AddStringItem(DefaultRemoteLogLevel, "info", validateLogLevel)
configItemSpecMap.AddStringItem(SyslogLogLevel, "info", validateSyslogKernelLevel)
configItemSpecMap.AddStringItem(KernelLogLevel, "info", validateSyslogKernelLevel)
configItemSpecMap.AddStringItem(SyslogRemoteLogLevel, "info", validateSyslogKernelLevel)
configItemSpecMap.AddStringItem(KernelRemoteLogLevel, "info", validateSyslogKernelLevel)
configItemSpecMap.AddStringItem(FmlCustomResolution, FmlResolutionUnset, blankValidator)

// Add Agent Settings
configItemSpecMap.AddAgentSettingStringItem(LogLevel, "info", validateLogrusLevel)
configItemSpecMap.AddAgentSettingStringItem(RemoteLogLevel, "info", validateLogrusLevel)
configItemSpecMap.AddAgentSettingStringItem(LogLevel, "info", validateLogLevel)
configItemSpecMap.AddAgentSettingStringItem(RemoteLogLevel, "info", validateLogLevel)

// Add NetDump settings
configItemSpecMap.AddBoolItem(NetDumpEnable, true)
Expand All @@ -1003,10 +1011,15 @@ func NewConfigItemSpecMap() ConfigItemSpecMap {
return configItemSpecMap
}

// validateLogrusLevel - Wrapper for validating logrus loglevel
func validateLogrusLevel(level string) error {
_, err := logrus.ParseLevel(level)
return err
// validateLogLevel - make sure the log level has one of the supported values
func validateLogLevel(level string) error {
switch level {
case "none", "all":
return nil
default:
_, err := logrus.ParseLevel(level)
return err
}
}

// validateSyslogKernelLevel - Wrapper for validating syslog and kernel
Expand Down
4 changes: 3 additions & 1 deletion pkg/pillar/types/global_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func TestAddStringItem(t *testing.T) {
}
for testname, test := range testMatrix {
t.Logf("Running test case %s", testname)
(&specMap).AddStringItem(test.key, test.defaultString, validateLogrusLevel)
(&specMap).AddStringItem(test.key, test.defaultString, validateLogLevel)
assert.Equal(t, test.expectedVal, specMap.GlobalSettings[test.key].StringDefault)
}
}
Expand Down Expand Up @@ -204,7 +204,9 @@ func TestNewConfigItemSpecMap(t *testing.T) {
SSHAuthorizedKeys,
DefaultLogLevel,
DefaultRemoteLogLevel,
SyslogRemoteLogLevel,
SyslogLogLevel,
KernelRemoteLogLevel,
KernelLogLevel,
FmlCustomResolution,
DisableDHCPAllOnesNetMask,
Expand Down
30 changes: 29 additions & 1 deletion pkg/pillar/types/newlogtypes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,19 @@
package types

import (
"fmt"
"strconv"
"strings"
"time"
)

const (
// DevPrefix - device log file prefix string
// DevPrefix - general file prefix string for device log files
DevPrefix = "dev.log."
// DevPrefixUpload - file prefix string for device log files to upload
DevPrefixUpload = "dev.log.upload."
// DevPrefixKeep - file prefix string for device log files to keep on device
DevPrefixKeep = "dev.log.keep."
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, are we going to change the directories naming scheme?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no the directories themselves, but the files inside of them

// AppPrefix - app log file prefix string
AppPrefix = "app."
// AppSuffix - app log file suffix string, the appuuid is between the AppPrefix and AppSuffix
Expand Down Expand Up @@ -65,6 +72,8 @@ type NewlogMetrics struct {
NumKmessages uint64 // total input kmessages
NumSyslogMessages uint64 // total input syslog message
DevTop10InputBytesPCT map[string]uint32 // top 10 sources device log input in percentage
TotalSizeLogs uint64 // total size of logs on device
OldestSavedDeviceLog time.Time // timestamp of the latest device log saved on device

// upload latency
Latency cloudDelay
Expand All @@ -75,3 +84,22 @@ type NewlogMetrics struct {
DevMetrics logfileMetrics // Device metrics
AppMetrics logfileMetrics // App metrics
}

// GetTimestampFromGzipName - get timestamp from gzip file name
func GetTimestampFromGzipName(fName string) (time.Time, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then this should also replace this func, I guess?

eve/pkg/pillar/cmd/loguploader/loguploader.go

Line 739 in 63c8aae

func getTimeNumber(isApp bool, fName string) (bool, int) {

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exactly, I'm doing this in the main PR #4413

// here are example file names:
// app.6656f860-7563-4bbf-8bba-051f5942982b.log.1730464687367.gz
// dev.log.keep.1730404601953.gz
// dev.log.upload.1730404601953.gz
// the timestamp is the number between the last two dots
nameParts := strings.Split(fName, ".")
if len(nameParts) < 2 {
return time.Time{}, fmt.Errorf("getTimestampFromGzipName: invalid log file name %s", fName)
}
timeStr := nameParts[len(nameParts)-2]
fTime, err := strconv.Atoi(timeStr)
if err != nil {
return time.Time{}, fmt.Errorf("getTimestampFromGzipName: %w", err)
}
return time.Unix(0, int64(fTime)*int64(time.Millisecond)), nil
}
Loading