Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 37 additions & 26 deletions internal/alert/alert.go
Original file line number Diff line number Diff line change
Expand Up @@ -242,9 +242,31 @@ func (e *Engine) evalMetric(metric, resource string, value, warn, crit float64,
e.states[key] = st
}
st.lastValue = value
st.updateThresholdState(value >= warn, value >= crit, now)

aboveWarn := value >= warn
aboveCrit := value >= crit
held := func(since time.Time) bool { return now.Sub(since) >= e.forDur }
next := nextLevel(st.active,
st.critAbove && held(st.critSince), // critConfirmed
st.warnAbove && held(st.warnSince), // warnConfirmed
!st.critAbove && held(st.critSince), // belowCritConfirmed
!st.warnAbove && held(st.warnSince), // belowWarnConfirmed
)
if next == st.active {
return nil
}
prev := st.active
st.active = next
st.activeSince = now
ev := e.recordEvent(st, prev, next, value, now)
return &ev
}

// updateThresholdState records how long the value has continuously been on
// its current side of the warn/crit thresholds, resetting the "since" clock
// whenever it crosses over. Tracking the two boundaries independently means
// "value has been >= warn continuously for the debounce window" holds even
// while the value oscillates in and out of the crit band.
func (st *metricState) updateThresholdState(aboveWarn, aboveCrit bool, now time.Time) {
if !st.initialized || st.warnAbove != aboveWarn {
st.warnAbove = aboveWarn
st.warnSince = now
Expand All @@ -254,47 +276,36 @@ func (e *Engine) evalMetric(metric, resource string, value, warn, crit float64,
st.critSince = now
}
st.initialized = true
}

held := func(since time.Time) bool { return now.Sub(since) >= e.forDur }
critConfirmed := st.critAbove && held(st.critSince)
warnConfirmed := st.warnAbove && held(st.warnSince)
belowCritConfirmed := !st.critAbove && held(st.critSince)
belowWarnConfirmed := !st.warnAbove && held(st.warnSince)

next := st.active
switch st.active {
// nextLevel applies the debounce state machine's transition table: a level
// only changes once the corresponding signal has been confirmed (held past
// the debounce window).
func nextLevel(current Level, critConfirmed, warnConfirmed, belowCritConfirmed, belowWarnConfirmed bool) Level {
switch current {
case LevelOK:
switch {
case critConfirmed:
next = LevelCrit
return LevelCrit
case warnConfirmed:
next = LevelWarn
return LevelWarn
}
case LevelWarn:
switch {
case critConfirmed:
next = LevelCrit
return LevelCrit
case belowWarnConfirmed:
next = LevelOK
return LevelOK
}
case LevelCrit:
if belowCritConfirmed {
if belowWarnConfirmed {
next = LevelOK
} else {
next = LevelWarn
return LevelOK
}
return LevelWarn
}
}

if next != st.active {
prev := st.active
st.active = next
st.activeSince = now
ev := e.recordEvent(st, prev, next, value, now)
return &ev
}
return nil
return current
}

// pruneDisks drops disk states whose mountpoint is absent from the latest
Expand Down
119 changes: 70 additions & 49 deletions internal/collector/disk.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,18 +192,7 @@ func (c *DiskCollector) Collect() ([]Disk, error) {
return nil, err
}

// /proc/mounts can list the same mountpoint several times (overmounts,
// some bind-mount setups). Only the last mount is visible at the path
// β€” and the one statfs reports on β€” so keep the last entry per
// mountpoint, preserving first-seen order for stable output.
order := make([]string, 0, len(entries))
byMountpoint := make(map[string]mountEntry, len(entries))
for _, e := range entries {
if _, seen := byMountpoint[e.mountpoint]; !seen {
order = append(order, e.mountpoint)
}
byMountpoint[e.mountpoint] = e
}
order, byMountpoint := dedupeMounts(entries)

now := c.clock()
var disks []Disk
Expand All @@ -212,46 +201,78 @@ func (c *DiskCollector) Collect() ([]Disk, error) {
if c.excludedFSType[e.fstype] {
continue
}
if until, bad := c.badUntil[e.mountpoint]; bad {
if now.Before(until) {
continue
}
delete(c.badUntil, e.mountpoint)
if disk, ok := c.collectDisk(e, now); ok {
disks = append(disks, disk)
}
buf, err := c.statfsWithTimeout(e.mountpoint)
if err != nil {
if errors.Is(err, errStatfsTimeout) {
if c.badUntil == nil {
c.badUntil = make(map[string]time.Time)
}
c.badUntil[e.mountpoint] = now.Add(c.statfsCooldown)
}
continue
}
return disks, nil
}

// dedupeMounts collapses /proc/mounts entries to the last entry per
// mountpoint. /proc/mounts can list the same mountpoint several times
// (overmounts, some bind-mount setups); only the last mount is visible at
// the path β€” and the one statfs reports on. First-seen order is preserved
// for stable output.
func dedupeMounts(entries []mountEntry) (order []string, byMountpoint map[string]mountEntry) {
order = make([]string, 0, len(entries))
byMountpoint = make(map[string]mountEntry, len(entries))
for _, e := range entries {
if _, seen := byMountpoint[e.mountpoint]; !seen {
order = append(order, e.mountpoint)
}
total := uint64(buf.Blocks) * uint64(buf.Bsize)
if total == 0 {
continue
byMountpoint[e.mountpoint] = e
}
return order, byMountpoint
}

// collectDisk statfs's a single mountpoint and reports its usage. ok is
// false when the mountpoint should be skipped: it's on the bad-mount
// cooldown, statfs failed, or it reports zero size.
func (c *DiskCollector) collectDisk(e mountEntry, now time.Time) (disk Disk, ok bool) {
if until, bad := c.badUntil[e.mountpoint]; bad {
if now.Before(until) {
return Disk{}, false
}
free := uint64(buf.Bfree) * uint64(buf.Bsize)
avail := uint64(buf.Bavail) * uint64(buf.Bsize)
used := total - free
// df-compatible percentage: used / (used + avail). Bfree includes
// blocks reserved for root (typically 5% on ext4) that services
// cannot write to, so a Blocks-based percentage under-reports
// fullness β€” showing ~95% while df (and failing writes) already
// say 100%.
var usedPercent float64
if denom := used + avail; denom > 0 {
usedPercent = float64(used) / float64(denom) * 100
delete(c.badUntil, e.mountpoint)
}
buf, err := c.statfsWithTimeout(e.mountpoint)
if err != nil {
if errors.Is(err, errStatfsTimeout) {
c.markBad(e.mountpoint, now)
}
disks = append(disks, Disk{
Mountpoint: e.mountpoint,
Device: e.device,
FSType: e.fstype,
TotalBytes: total,
UsedBytes: used,
UsedPercent: usedPercent,
})
return Disk{}, false
}
return disks, nil
total := uint64(buf.Blocks) * uint64(buf.Bsize)
if total == 0 {
return Disk{}, false
}
free := uint64(buf.Bfree) * uint64(buf.Bsize)
avail := uint64(buf.Bavail) * uint64(buf.Bsize)
used := total - free
// df-compatible percentage: used / (used + avail). Bfree includes
// blocks reserved for root (typically 5% on ext4) that services
// cannot write to, so a Blocks-based percentage under-reports
// fullness β€” showing ~95% while df (and failing writes) already
// say 100%.
var usedPercent float64
if denom := used + avail; denom > 0 {
usedPercent = float64(used) / float64(denom) * 100
}
return Disk{
Mountpoint: e.mountpoint,
Device: e.device,
FSType: e.fstype,
TotalBytes: total,
UsedBytes: used,
UsedPercent: usedPercent,
}, true
}

// markBad puts mountpoint on cooldown after a statfs timeout so subsequent
// Collect calls skip it instead of blocking again for statfsTimeout.
func (c *DiskCollector) markBad(mountpoint string, now time.Time) {
if c.badUntil == nil {
c.badUntil = make(map[string]time.Time)
}
c.badUntil[mountpoint] = now.Add(c.statfsCooldown)
}
56 changes: 38 additions & 18 deletions internal/collector/persist.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,27 +222,14 @@ func decodeHistory(data []byte) (History, error) {

list := make([]historySeries, 0, count)
for i := uint32(0); i < count && d.err == nil; i++ {
kind := d.uint8()
keyLen := d.uint16()
if d.err == nil && int(keyLen) > maxKeyLen {
return History{}, fmt.Errorf("history file series key length %d exceeds limit %d", keyLen, maxKeyLen)
s, err, ok := d.decodeSeries()
if err != nil {
return History{}, err
}
key := string(d.take(int(keyLen)))
n := d.uint32()
if d.err == nil && n > maxPointsPerSeries {
return History{}, fmt.Errorf("history file series declares %d points (limit %d)", n, maxPointsPerSeries)
}
raw := d.take(int(n) * pointSize)
if d.err != nil {
if !ok {
break
}
points := make([]HistoryPoint, n)
for j := range points {
ms := int64(binary.LittleEndian.Uint64(raw[j*pointSize:]))
bits := binary.LittleEndian.Uint64(raw[j*pointSize+8:])
points[j] = HistoryPoint{Timestamp: time.UnixMilli(ms), Value: math.Float64frombits(bits)}
}
list = append(list, historySeries{kind: kind, key: key, points: points})
list = append(list, s)
}
if d.err != nil {
return History{}, d.err
Expand All @@ -253,6 +240,39 @@ func decodeHistory(data []byte) (History, error) {
return seriesToHistory(list), nil
}

// decodeSeries decodes one series record (kind, key, and its points) from
// d. A non-nil err is a fatal format violation (a declared limit exceeded)
// that should abort decoding immediately. ok is false when the sticky
// decoder error was set instead (e.g. truncated data); the caller reports
// that via d.err after the loop.
func (d *historyDecoder) decodeSeries() (s historySeries, err error, ok bool) {
kind := d.uint8()
keyLen := d.uint16()
if d.err == nil && int(keyLen) > maxKeyLen {
return historySeries{}, fmt.Errorf("history file series key length %d exceeds limit %d", keyLen, maxKeyLen), false
}
key := string(d.take(int(keyLen)))
n := d.uint32()
if d.err == nil && n > maxPointsPerSeries {
return historySeries{}, fmt.Errorf("history file series declares %d points (limit %d)", n, maxPointsPerSeries), false
}
raw := d.take(int(n) * pointSize)
if d.err != nil {
return historySeries{}, nil, false
}
return historySeries{kind: kind, key: key, points: decodeHistoryPoints(raw, n)}, nil, true
}

func decodeHistoryPoints(raw []byte, n uint32) []HistoryPoint {
points := make([]HistoryPoint, n)
for j := range points {
ms := int64(binary.LittleEndian.Uint64(raw[j*pointSize:]))
bits := binary.LittleEndian.Uint64(raw[j*pointSize+8:])
points[j] = HistoryPoint{Timestamp: time.UnixMilli(ms), Value: math.Float64frombits(bits)}
}
return points
}

// writeFileAtomic writes data to path via a temp file in the same
// directory plus rename, so a crash mid-write never leaves a partially
// written history file behind, and creates the parent directory if needed.
Expand Down
4 changes: 2 additions & 2 deletions internal/web/assets/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@

function applyTheme(theme) {
if (theme) {
document.documentElement.setAttribute('data-theme', theme);
document.documentElement.dataset.theme = theme;
} else {
document.documentElement.removeAttribute('data-theme');
delete document.documentElement.dataset.theme;
}
updateThemeToggle();
// Repaint canvas widgets that cached the previous palette's colors.
Expand Down
2 changes: 1 addition & 1 deletion internal/web/assets/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ <h2 role="button" tabindex="0" aria-label="Open temperature detail"><span class=
<!-- Column 3: memory, storage & network -->
<div class="column">
<section class="card accent-memory card-clickable" id="card-memory" data-metric="memory">
<h2 role="button" tabindex="0" aria-label="Open memory usage detail"><span class="card-icon">🧠</span> Memory &amp; Swap</h2>
<h2 role="button" tabindex="0" aria-label="Open Memory &amp; Swap detail"><span class="card-icon">🧠</span> Memory &amp; Swap</h2>
<div class="bar-row">
<div class="bar-label">
<span class="bar-name">RAM</span>
Expand Down
4 changes: 2 additions & 2 deletions internal/web/assets/theme-init.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
// render-blocking, it still runs before the first paint.
(function () {
try {
var stored = localStorage.getItem('pimonitor-theme');
const stored = localStorage.getItem('pimonitor-theme');
if (stored === 'light' || stored === 'dark') {
document.documentElement.setAttribute('data-theme', stored);
document.documentElement.dataset.theme = stored;
}
} catch (e) {}
})();
2 changes: 1 addition & 1 deletion sonar-project.properties
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
sonar.projectKey=networlddev_PiMonitor
sonar.projectKey=LarsLaskowski_PiMonitor
sonar.organization=networlddev


Expand Down
Loading