diff --git a/internal/alert/alert.go b/internal/alert/alert.go index 222de0d..a005100 100644 --- a/internal/alert/alert.go +++ b/internal/alert/alert.go @@ -242,9 +242,31 @@ func (e *Engine) evalMetric(metric, resource string, value, warn, crit float64, e.states[key] = st } st.lastValue = value + st.updateThresholdState(value >= warn, value >= crit, now) - aboveWarn := value >= warn - aboveCrit := value >= crit + held := func(since time.Time) bool { return now.Sub(since) >= e.forDur } + next := nextLevel(st.active, + st.critAbove && held(st.critSince), // critConfirmed + st.warnAbove && held(st.warnSince), // warnConfirmed + !st.critAbove && held(st.critSince), // belowCritConfirmed + !st.warnAbove && held(st.warnSince), // belowWarnConfirmed + ) + if next == st.active { + return nil + } + prev := st.active + st.active = next + st.activeSince = now + ev := e.recordEvent(st, prev, next, value, now) + return &ev +} + +// updateThresholdState records how long the value has continuously been on +// its current side of the warn/crit thresholds, resetting the "since" clock +// whenever it crosses over. Tracking the two boundaries independently means +// "value has been >= warn continuously for the debounce window" holds even +// while the value oscillates in and out of the crit band. +func (st *metricState) updateThresholdState(aboveWarn, aboveCrit bool, now time.Time) { if !st.initialized || st.warnAbove != aboveWarn { st.warnAbove = aboveWarn st.warnSince = now @@ -254,47 +276,36 @@ func (e *Engine) evalMetric(metric, resource string, value, warn, crit float64, st.critSince = now } st.initialized = true +} - held := func(since time.Time) bool { return now.Sub(since) >= e.forDur } - critConfirmed := st.critAbove && held(st.critSince) - warnConfirmed := st.warnAbove && held(st.warnSince) - belowCritConfirmed := !st.critAbove && held(st.critSince) - belowWarnConfirmed := !st.warnAbove && held(st.warnSince) - - next := st.active - switch st.active { +// nextLevel applies the debounce state machine's transition table: a level +// only changes once the corresponding signal has been confirmed (held past +// the debounce window). +func nextLevel(current Level, critConfirmed, warnConfirmed, belowCritConfirmed, belowWarnConfirmed bool) Level { + switch current { case LevelOK: switch { case critConfirmed: - next = LevelCrit + return LevelCrit case warnConfirmed: - next = LevelWarn + return LevelWarn } case LevelWarn: switch { case critConfirmed: - next = LevelCrit + return LevelCrit case belowWarnConfirmed: - next = LevelOK + return LevelOK } case LevelCrit: if belowCritConfirmed { if belowWarnConfirmed { - next = LevelOK - } else { - next = LevelWarn + return LevelOK } + return LevelWarn } } - - if next != st.active { - prev := st.active - st.active = next - st.activeSince = now - ev := e.recordEvent(st, prev, next, value, now) - return &ev - } - return nil + return current } // pruneDisks drops disk states whose mountpoint is absent from the latest diff --git a/internal/collector/disk.go b/internal/collector/disk.go index bef0bfd..6d779df 100644 --- a/internal/collector/disk.go +++ b/internal/collector/disk.go @@ -192,18 +192,7 @@ func (c *DiskCollector) Collect() ([]Disk, error) { return nil, err } - // /proc/mounts can list the same mountpoint several times (overmounts, - // some bind-mount setups). Only the last mount is visible at the path - // — and the one statfs reports on — so keep the last entry per - // mountpoint, preserving first-seen order for stable output. - order := make([]string, 0, len(entries)) - byMountpoint := make(map[string]mountEntry, len(entries)) - for _, e := range entries { - if _, seen := byMountpoint[e.mountpoint]; !seen { - order = append(order, e.mountpoint) - } - byMountpoint[e.mountpoint] = e - } + order, byMountpoint := dedupeMounts(entries) now := c.clock() var disks []Disk @@ -212,46 +201,78 @@ func (c *DiskCollector) Collect() ([]Disk, error) { if c.excludedFSType[e.fstype] { continue } - if until, bad := c.badUntil[e.mountpoint]; bad { - if now.Before(until) { - continue - } - delete(c.badUntil, e.mountpoint) + if disk, ok := c.collectDisk(e, now); ok { + disks = append(disks, disk) } - buf, err := c.statfsWithTimeout(e.mountpoint) - if err != nil { - if errors.Is(err, errStatfsTimeout) { - if c.badUntil == nil { - c.badUntil = make(map[string]time.Time) - } - c.badUntil[e.mountpoint] = now.Add(c.statfsCooldown) - } - continue + } + return disks, nil +} + +// dedupeMounts collapses /proc/mounts entries to the last entry per +// mountpoint. /proc/mounts can list the same mountpoint several times +// (overmounts, some bind-mount setups); only the last mount is visible at +// the path — and the one statfs reports on. First-seen order is preserved +// for stable output. +func dedupeMounts(entries []mountEntry) (order []string, byMountpoint map[string]mountEntry) { + order = make([]string, 0, len(entries)) + byMountpoint = make(map[string]mountEntry, len(entries)) + for _, e := range entries { + if _, seen := byMountpoint[e.mountpoint]; !seen { + order = append(order, e.mountpoint) } - total := uint64(buf.Blocks) * uint64(buf.Bsize) - if total == 0 { - continue + byMountpoint[e.mountpoint] = e + } + return order, byMountpoint +} + +// collectDisk statfs's a single mountpoint and reports its usage. ok is +// false when the mountpoint should be skipped: it's on the bad-mount +// cooldown, statfs failed, or it reports zero size. +func (c *DiskCollector) collectDisk(e mountEntry, now time.Time) (disk Disk, ok bool) { + if until, bad := c.badUntil[e.mountpoint]; bad { + if now.Before(until) { + return Disk{}, false } - free := uint64(buf.Bfree) * uint64(buf.Bsize) - avail := uint64(buf.Bavail) * uint64(buf.Bsize) - used := total - free - // df-compatible percentage: used / (used + avail). Bfree includes - // blocks reserved for root (typically 5% on ext4) that services - // cannot write to, so a Blocks-based percentage under-reports - // fullness — showing ~95% while df (and failing writes) already - // say 100%. - var usedPercent float64 - if denom := used + avail; denom > 0 { - usedPercent = float64(used) / float64(denom) * 100 + delete(c.badUntil, e.mountpoint) + } + buf, err := c.statfsWithTimeout(e.mountpoint) + if err != nil { + if errors.Is(err, errStatfsTimeout) { + c.markBad(e.mountpoint, now) } - disks = append(disks, Disk{ - Mountpoint: e.mountpoint, - Device: e.device, - FSType: e.fstype, - TotalBytes: total, - UsedBytes: used, - UsedPercent: usedPercent, - }) + return Disk{}, false } - return disks, nil + total := uint64(buf.Blocks) * uint64(buf.Bsize) + if total == 0 { + return Disk{}, false + } + free := uint64(buf.Bfree) * uint64(buf.Bsize) + avail := uint64(buf.Bavail) * uint64(buf.Bsize) + used := total - free + // df-compatible percentage: used / (used + avail). Bfree includes + // blocks reserved for root (typically 5% on ext4) that services + // cannot write to, so a Blocks-based percentage under-reports + // fullness — showing ~95% while df (and failing writes) already + // say 100%. + var usedPercent float64 + if denom := used + avail; denom > 0 { + usedPercent = float64(used) / float64(denom) * 100 + } + return Disk{ + Mountpoint: e.mountpoint, + Device: e.device, + FSType: e.fstype, + TotalBytes: total, + UsedBytes: used, + UsedPercent: usedPercent, + }, true +} + +// markBad puts mountpoint on cooldown after a statfs timeout so subsequent +// Collect calls skip it instead of blocking again for statfsTimeout. +func (c *DiskCollector) markBad(mountpoint string, now time.Time) { + if c.badUntil == nil { + c.badUntil = make(map[string]time.Time) + } + c.badUntil[mountpoint] = now.Add(c.statfsCooldown) } diff --git a/internal/collector/persist.go b/internal/collector/persist.go index d632584..3bf4bcd 100644 --- a/internal/collector/persist.go +++ b/internal/collector/persist.go @@ -222,27 +222,14 @@ func decodeHistory(data []byte) (History, error) { list := make([]historySeries, 0, count) for i := uint32(0); i < count && d.err == nil; i++ { - kind := d.uint8() - keyLen := d.uint16() - if d.err == nil && int(keyLen) > maxKeyLen { - return History{}, fmt.Errorf("history file series key length %d exceeds limit %d", keyLen, maxKeyLen) + s, err, ok := d.decodeSeries() + if err != nil { + return History{}, err } - key := string(d.take(int(keyLen))) - n := d.uint32() - if d.err == nil && n > maxPointsPerSeries { - return History{}, fmt.Errorf("history file series declares %d points (limit %d)", n, maxPointsPerSeries) - } - raw := d.take(int(n) * pointSize) - if d.err != nil { + if !ok { break } - points := make([]HistoryPoint, n) - for j := range points { - ms := int64(binary.LittleEndian.Uint64(raw[j*pointSize:])) - bits := binary.LittleEndian.Uint64(raw[j*pointSize+8:]) - points[j] = HistoryPoint{Timestamp: time.UnixMilli(ms), Value: math.Float64frombits(bits)} - } - list = append(list, historySeries{kind: kind, key: key, points: points}) + list = append(list, s) } if d.err != nil { return History{}, d.err @@ -253,6 +240,39 @@ func decodeHistory(data []byte) (History, error) { return seriesToHistory(list), nil } +// decodeSeries decodes one series record (kind, key, and its points) from +// d. A non-nil err is a fatal format violation (a declared limit exceeded) +// that should abort decoding immediately. ok is false when the sticky +// decoder error was set instead (e.g. truncated data); the caller reports +// that via d.err after the loop. +func (d *historyDecoder) decodeSeries() (s historySeries, err error, ok bool) { + kind := d.uint8() + keyLen := d.uint16() + if d.err == nil && int(keyLen) > maxKeyLen { + return historySeries{}, fmt.Errorf("history file series key length %d exceeds limit %d", keyLen, maxKeyLen), false + } + key := string(d.take(int(keyLen))) + n := d.uint32() + if d.err == nil && n > maxPointsPerSeries { + return historySeries{}, fmt.Errorf("history file series declares %d points (limit %d)", n, maxPointsPerSeries), false + } + raw := d.take(int(n) * pointSize) + if d.err != nil { + return historySeries{}, nil, false + } + return historySeries{kind: kind, key: key, points: decodeHistoryPoints(raw, n)}, nil, true +} + +func decodeHistoryPoints(raw []byte, n uint32) []HistoryPoint { + points := make([]HistoryPoint, n) + for j := range points { + ms := int64(binary.LittleEndian.Uint64(raw[j*pointSize:])) + bits := binary.LittleEndian.Uint64(raw[j*pointSize+8:]) + points[j] = HistoryPoint{Timestamp: time.UnixMilli(ms), Value: math.Float64frombits(bits)} + } + return points +} + // writeFileAtomic writes data to path via a temp file in the same // directory plus rename, so a crash mid-write never leaves a partially // written history file behind, and creates the parent directory if needed. diff --git a/internal/web/assets/app.js b/internal/web/assets/app.js index ca4165d..9ce5217 100644 --- a/internal/web/assets/app.js +++ b/internal/web/assets/app.js @@ -77,9 +77,9 @@ function applyTheme(theme) { if (theme) { - document.documentElement.setAttribute('data-theme', theme); + document.documentElement.dataset.theme = theme; } else { - document.documentElement.removeAttribute('data-theme'); + delete document.documentElement.dataset.theme; } updateThemeToggle(); // Repaint canvas widgets that cached the previous palette's colors. diff --git a/internal/web/assets/index.html b/internal/web/assets/index.html index ecc6c87..44b8b2d 100644 --- a/internal/web/assets/index.html +++ b/internal/web/assets/index.html @@ -84,7 +84,7 @@