rewrite as a daemon with shutdown on low battery
the oneshot+timer produced 8367 journal lines a day of systemd start/stop noise. now Type=notify with an internal ticker, logging transitions only. hardware sits behind sensor.Sensor; internal/monitor is pure, so the shutdown state machine is tested without a pi. four guards, each tested, because a false poweroff of a box with no physical access is worse than a missed one: - a failed read resets the arming window rather than pausing it - voltage must actually fall across the window, else it is a stuck sensor. this is the only defence against the ac line reading 0 while mains is connected, so it is load-bearing - a settle period stops a restart loop acting on early readings - flapping ac cannot accumulate verified on hardware 2026-08-09: after poweroff on battery the x1208 starts the pi again when mains returns, with no button press. that was the risk that could have made this feature unsafe. a failed tick rewrites the previous sample with sensor_healthy 0 and the failure counter, so last_update still freezes for the staleness alert while the failure stays visible. module path moved to git.mial.net. debian units and the compose overlay deleted; deployment lives in the infra repo.
This commit is contained in:
@@ -0,0 +1,257 @@
|
||||
// Package monitor holds the daemon's decision logic.
|
||||
//
|
||||
// Everything here is pure: it takes a reading and an error, and returns what
|
||||
// to log, what to write, and whether to act. It performs no I/O and reads no
|
||||
// clock. That is what makes the shutdown path testable without a Pi.
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.mial.net/mokhtar/x1208-exporter/internal/sensor"
|
||||
)
|
||||
|
||||
// Event names a state transition. The daemon logs transitions only: a healthy
|
||||
// tick produces no output at all.
|
||||
type Event string
|
||||
|
||||
const (
|
||||
EventStartup Event = "startup"
|
||||
EventACChanged Event = "ac_changed"
|
||||
EventErrorEnter Event = "error_enter"
|
||||
EventErrorExit Event = "error_exit"
|
||||
EventShutdownArmed Event = "shutdown_armed"
|
||||
EventShutdownDisarmed Event = "shutdown_disarmed"
|
||||
EventShutdownInhibited Event = "shutdown_inhibited"
|
||||
EventShutdownTriggered Event = "shutdown_triggered"
|
||||
)
|
||||
|
||||
// Log is one line the daemon must emit.
|
||||
type Log struct {
|
||||
Event Event
|
||||
Message string
|
||||
}
|
||||
|
||||
// Config controls the shutdown policy. The zero value disables shutdown.
|
||||
type Config struct {
|
||||
// Shutdown enables the poweroff decision. Default off: the daemon is an
|
||||
// exporter first, and actuation is opt-in.
|
||||
Shutdown bool
|
||||
|
||||
// Volts and SOC are the low-battery thresholds. Either one can arm.
|
||||
Volts float64
|
||||
SOC float64
|
||||
|
||||
// Samples is how many consecutive qualifying ticks must pass before the
|
||||
// daemon acts.
|
||||
Samples int
|
||||
|
||||
// MinUptimeTicks suppresses shutdown for the first N ticks after start.
|
||||
// Without this a restart loop could power the machine off using readings
|
||||
// taken before the hardware settled.
|
||||
MinUptimeTicks int
|
||||
|
||||
// RequireDischarge demands that cell voltage actually fall across the
|
||||
// arming window before the daemon acts.
|
||||
//
|
||||
// This is the only check that distinguishes a real discharge from a stuck
|
||||
// sensor. A wedged fuel gauge repeats one plausible low value forever, and
|
||||
// consecutive samples of a persistent fault are not independent evidence.
|
||||
//
|
||||
// It is LOAD-BEARING, not a refinement. It is the sole defence against the
|
||||
// worst remaining failure: the AC line falsely reading 0 while mains is
|
||||
// actually connected. On mains the charger holds cell voltage flat or
|
||||
// rising, so a false "on battery" cannot satisfy this check. Turn it off
|
||||
// and a single stuck GPIO line can power the machine off during normal
|
||||
// operation.
|
||||
RequireDischarge bool
|
||||
}
|
||||
|
||||
// Result is the outcome of one tick.
|
||||
type Result struct {
|
||||
Logs []Log
|
||||
|
||||
// Sample is the most recent COMPLETE reading, which is not necessarily
|
||||
// this tick's. HaveSample is false until the first successful read.
|
||||
Sample sensor.Reading
|
||||
HaveSample bool
|
||||
|
||||
// Healthy reports whether this tick's read succeeded.
|
||||
Healthy bool
|
||||
Failures uint64
|
||||
|
||||
// Shutdown is true on the single tick where the daemon decides to power
|
||||
// the machine off.
|
||||
Shutdown bool
|
||||
}
|
||||
|
||||
// Monitor tracks state across ticks.
|
||||
type Monitor struct {
|
||||
cfg Config
|
||||
|
||||
started bool
|
||||
ticks int
|
||||
|
||||
lastAC bool
|
||||
haveAC bool
|
||||
|
||||
inError bool
|
||||
errStreak int
|
||||
failures uint64
|
||||
last sensor.Reading
|
||||
haveLast bool
|
||||
armed bool
|
||||
armCount int
|
||||
armVolts float64
|
||||
inhibited bool
|
||||
firedShut bool
|
||||
}
|
||||
|
||||
// New returns a Monitor with the given policy.
|
||||
func New(cfg Config) *Monitor { return &Monitor{cfg: cfg} }
|
||||
|
||||
// Tick advances the state machine by one reading.
|
||||
//
|
||||
// Pass the error from the sensor as err; reading is ignored when err is
|
||||
// non-nil. Tick never blocks and never panics.
|
||||
func (m *Monitor) Tick(reading sensor.Reading, err error) Result {
|
||||
m.ticks++
|
||||
res := Result{Failures: m.failures}
|
||||
|
||||
if !m.started {
|
||||
m.started = true
|
||||
res.Logs = append(res.Logs, Log{EventStartup, m.startupMessage()})
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return m.tickFailed(res, err)
|
||||
}
|
||||
return m.tickSucceeded(res, reading)
|
||||
}
|
||||
|
||||
func (m *Monitor) startupMessage() string {
|
||||
if !m.cfg.Shutdown {
|
||||
return "x1208-exporter started; shutdown-on-battery disabled"
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"x1208-exporter started; shutdown-on-battery ENABLED below %.2f V or %.0f%% for %d ticks (require-discharge=%t)",
|
||||
m.cfg.Volts, m.cfg.SOC, m.cfg.Samples, m.cfg.RequireDischarge)
|
||||
}
|
||||
|
||||
func (m *Monitor) tickFailed(res Result, err error) Result {
|
||||
m.failures++
|
||||
m.errStreak++
|
||||
res.Failures = m.failures
|
||||
res.Healthy = false
|
||||
res.Sample = m.last
|
||||
res.HaveSample = m.haveLast
|
||||
|
||||
if !m.inError {
|
||||
m.inError = true
|
||||
res.Logs = append(res.Logs, Log{EventErrorEnter, "UPS read failed: " + err.Error()})
|
||||
}
|
||||
|
||||
// A failed tick RESETS the arming window rather than pausing it.
|
||||
//
|
||||
// Pausing would let "low, low, thirty minutes of errors, low, low" reach
|
||||
// the sample count. That is not a sustained low battery and must not power
|
||||
// the machine off.
|
||||
res.Logs = append(res.Logs, m.disarm("sensor read failed")...)
|
||||
return res
|
||||
}
|
||||
|
||||
func (m *Monitor) tickSucceeded(res Result, r sensor.Reading) Result {
|
||||
if m.inError {
|
||||
res.Logs = append(res.Logs, Log{EventErrorExit,
|
||||
fmt.Sprintf("UPS read recovered after %d failed ticks", m.errStreak)})
|
||||
m.inError = false
|
||||
m.errStreak = 0
|
||||
}
|
||||
|
||||
if m.haveAC && m.lastAC != r.ACPresent {
|
||||
res.Logs = append(res.Logs, Log{EventACChanged, acMessage(r.ACPresent)})
|
||||
}
|
||||
m.lastAC = r.ACPresent
|
||||
m.haveAC = true
|
||||
|
||||
m.last = r
|
||||
m.haveLast = true
|
||||
|
||||
res.Healthy = true
|
||||
res.Sample = r
|
||||
res.HaveSample = true
|
||||
|
||||
logs, shutdown := m.evaluateShutdown(r)
|
||||
res.Logs = append(res.Logs, logs...)
|
||||
res.Shutdown = shutdown
|
||||
return res
|
||||
}
|
||||
|
||||
func acMessage(present bool) string {
|
||||
if present {
|
||||
return "AC power restored"
|
||||
}
|
||||
return "AC power lost; running on battery"
|
||||
}
|
||||
|
||||
func (m *Monitor) evaluateShutdown(r sensor.Reading) ([]Log, bool) {
|
||||
if !m.cfg.Shutdown || m.firedShut {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
qualifies := !r.ACPresent && (r.Volts < m.cfg.Volts || r.SOC < m.cfg.SOC)
|
||||
if !qualifies {
|
||||
return m.disarm("battery recovered or AC returned"), false
|
||||
}
|
||||
|
||||
if m.ticks <= m.cfg.MinUptimeTicks {
|
||||
if m.inhibited {
|
||||
return nil, false
|
||||
}
|
||||
m.inhibited = true
|
||||
return []Log{{EventShutdownInhibited, fmt.Sprintf(
|
||||
"low battery within the first %d ticks after start; shutdown suppressed",
|
||||
m.cfg.MinUptimeTicks)}}, false
|
||||
}
|
||||
|
||||
if !m.armed {
|
||||
m.armed = true
|
||||
m.armCount = 1
|
||||
m.armVolts = r.Volts
|
||||
return []Log{{EventShutdownArmed, fmt.Sprintf(
|
||||
"low battery on cells (%.3f V, %.1f%%); shutdown in %d more qualifying ticks",
|
||||
r.Volts, r.SOC, m.cfg.Samples-1)}}, false
|
||||
}
|
||||
|
||||
m.armCount++
|
||||
if m.armCount < m.cfg.Samples {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if m.cfg.RequireDischarge && r.Volts >= m.armVolts {
|
||||
if m.inhibited {
|
||||
return nil, false
|
||||
}
|
||||
m.inhibited = true
|
||||
return []Log{{EventShutdownInhibited, fmt.Sprintf(
|
||||
"battery low but voltage did not fall over %d ticks (%.3f V then %.3f V); "+
|
||||
"treating as a stuck sensor, not a discharge",
|
||||
m.armCount, m.armVolts, r.Volts)}}, false
|
||||
}
|
||||
|
||||
m.firedShut = true
|
||||
return []Log{{EventShutdownTriggered, fmt.Sprintf(
|
||||
"powering off: on battery at %.3f V / %.1f%% for %d consecutive ticks",
|
||||
r.Volts, r.SOC, m.armCount)}}, true
|
||||
}
|
||||
|
||||
func (m *Monitor) disarm(reason string) []Log {
|
||||
m.inhibited = false
|
||||
if !m.armed {
|
||||
m.armCount = 0
|
||||
return nil
|
||||
}
|
||||
m.armed = false
|
||||
m.armCount = 0
|
||||
return []Log{{EventShutdownDisarmed, "shutdown disarmed: " + reason}}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.mial.net/mokhtar/x1208-exporter/internal/sensor"
|
||||
)
|
||||
|
||||
func shutdownConfig() Config {
|
||||
return Config{
|
||||
Shutdown: true,
|
||||
Volts: 3.5,
|
||||
SOC: 15,
|
||||
Samples: 4,
|
||||
MinUptimeTicks: 2,
|
||||
RequireDischarge: true,
|
||||
}
|
||||
}
|
||||
|
||||
// onBattery returns a low reading that falls by 10 mV per step, which is what
|
||||
// a real discharge looks like.
|
||||
func onBattery(step int) sensor.Reading {
|
||||
return sensor.Reading{
|
||||
ACPresent: false,
|
||||
Volts: 3.40 - float64(step)*0.01,
|
||||
SOC: 10,
|
||||
At: time.Unix(1754661600+int64(step)*30, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func onMains() sensor.Reading {
|
||||
return sensor.Reading{ACPresent: true, Volts: 4.1, SOC: 100, At: time.Unix(1754661600, 0)}
|
||||
}
|
||||
|
||||
func events(r Result) []Event {
|
||||
out := make([]Event, 0, len(r.Logs))
|
||||
for _, l := range r.Logs {
|
||||
out = append(out, l.Event)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contains(list []Event, want Event) bool {
|
||||
for _, e := range list {
|
||||
if e == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// settle runs enough healthy mains ticks to clear the startup inhibition.
|
||||
func settle(t *testing.T, m *Monitor) {
|
||||
t.Helper()
|
||||
for i := 0; i <= m.cfg.MinUptimeTicks; i++ {
|
||||
if res := m.Tick(onMains(), nil); res.Shutdown {
|
||||
t.Fatal("shutdown fired while on mains")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthyTicksLogNothingAfterStartup(t *testing.T) {
|
||||
m := New(Config{})
|
||||
|
||||
first := m.Tick(onMains(), nil)
|
||||
if !contains(events(first), EventStartup) {
|
||||
t.Error("first tick must log startup")
|
||||
}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
if res := m.Tick(onMains(), nil); len(res.Logs) != 0 {
|
||||
t.Errorf("healthy tick %d logged %v, want nothing", i, events(res))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestACTransitionLogsOncePerChange(t *testing.T) {
|
||||
m := New(Config{})
|
||||
m.Tick(onMains(), nil)
|
||||
|
||||
res := m.Tick(onBattery(0), nil)
|
||||
if !contains(events(res), EventACChanged) {
|
||||
t.Fatal("losing AC must log a transition")
|
||||
}
|
||||
if res := m.Tick(onBattery(1), nil); len(res.Logs) != 0 {
|
||||
t.Errorf("staying on battery logged %v, want nothing", events(res))
|
||||
}
|
||||
if res := m.Tick(onMains(), nil); !contains(events(res), EventACChanged) {
|
||||
t.Error("regaining AC must log a transition")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedTickKeepsPreviousSampleAndCounts(t *testing.T) {
|
||||
m := New(Config{})
|
||||
m.Tick(onMains(), nil)
|
||||
|
||||
res := m.Tick(sensor.Reading{}, errors.New("i2c wedged"))
|
||||
if res.Healthy {
|
||||
t.Error("failed tick must report unhealthy")
|
||||
}
|
||||
if res.Failures != 1 {
|
||||
t.Errorf("failures = %d, want 1", res.Failures)
|
||||
}
|
||||
if !res.HaveSample || res.Sample.Volts != 4.1 {
|
||||
t.Error("failed tick must carry the previous complete sample forward")
|
||||
}
|
||||
if !contains(events(res), EventErrorEnter) {
|
||||
t.Error("first failure must log error_enter")
|
||||
}
|
||||
|
||||
if res := m.Tick(sensor.Reading{}, errors.New("still wedged")); contains(events(res), EventErrorEnter) {
|
||||
t.Error("a continuing error must not log error_enter again")
|
||||
}
|
||||
if res := m.Tick(onMains(), nil); !contains(events(res), EventErrorExit) {
|
||||
t.Error("recovery must log error_exit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownDisabledByDefault(t *testing.T) {
|
||||
m := New(Config{})
|
||||
for i := 0; i < 20; i++ {
|
||||
if m.Tick(onBattery(i), nil).Shutdown {
|
||||
t.Fatal("shutdown fired with an empty Config; it must be opt-in")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownFiresOnSustainedDischarge(t *testing.T) {
|
||||
m := New(shutdownConfig())
|
||||
settle(t, m)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
if m.Tick(onBattery(i), nil).Shutdown {
|
||||
t.Fatalf("shutdown fired on qualifying tick %d, before the sample count", i+1)
|
||||
}
|
||||
}
|
||||
res := m.Tick(onBattery(3), nil)
|
||||
if !res.Shutdown {
|
||||
t.Fatal("shutdown must fire on the fourth consecutive qualifying tick")
|
||||
}
|
||||
if !contains(events(res), EventShutdownTriggered) {
|
||||
t.Error("firing must log shutdown_triggered")
|
||||
}
|
||||
if m.Tick(onBattery(4), nil).Shutdown {
|
||||
t.Error("shutdown must fire only once")
|
||||
}
|
||||
}
|
||||
|
||||
// A wedged sensor repeats one plausible low value. Consecutive identical
|
||||
// samples are one fault observed four times, not four confirmations.
|
||||
func TestStuckSensorNeverTriggersShutdown(t *testing.T) {
|
||||
m := New(shutdownConfig())
|
||||
settle(t, m)
|
||||
|
||||
stuck := sensor.Reading{ACPresent: false, Volts: 3.40, SOC: 10, At: time.Unix(1754661600, 0)}
|
||||
for i := 0; i < 50; i++ {
|
||||
if m.Tick(stuck, nil).Shutdown {
|
||||
t.Fatalf("shutdown fired at tick %d on a constant voltage", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// "low, low, long outage, low, low" is not a sustained low battery.
|
||||
func TestErrorResetsArmingWindow(t *testing.T) {
|
||||
m := New(shutdownConfig())
|
||||
settle(t, m)
|
||||
|
||||
m.Tick(onBattery(0), nil)
|
||||
m.Tick(onBattery(1), nil)
|
||||
|
||||
res := m.Tick(sensor.Reading{}, errors.New("bus error"))
|
||||
if !contains(events(res), EventShutdownDisarmed) {
|
||||
t.Error("a failed tick must disarm, not pause, the shutdown window")
|
||||
}
|
||||
|
||||
for i := 2; i < 5; i++ {
|
||||
if m.Tick(onBattery(i), nil).Shutdown {
|
||||
t.Fatalf("shutdown fired at tick %d; the window must restart after an error", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestACReturnDisarms(t *testing.T) {
|
||||
m := New(shutdownConfig())
|
||||
settle(t, m)
|
||||
|
||||
m.Tick(onBattery(0), nil)
|
||||
m.Tick(onBattery(1), nil)
|
||||
|
||||
res := m.Tick(onMains(), nil)
|
||||
if !contains(events(res), EventShutdownDisarmed) {
|
||||
t.Error("AC returning must disarm")
|
||||
}
|
||||
for i := 2; i < 5; i++ {
|
||||
if m.Tick(onBattery(i), nil).Shutdown {
|
||||
t.Fatalf("shutdown fired at tick %d; the window must restart after AC returned", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A flapping AC line must not accumulate toward a poweroff.
|
||||
func TestFlappingACNeverAccumulates(t *testing.T) {
|
||||
m := New(shutdownConfig())
|
||||
settle(t, m)
|
||||
|
||||
for i := 0; i < 40; i++ {
|
||||
var res Result
|
||||
if i%2 == 0 {
|
||||
res = m.Tick(onBattery(i), nil)
|
||||
} else {
|
||||
res = m.Tick(onMains(), nil)
|
||||
}
|
||||
if res.Shutdown {
|
||||
t.Fatalf("shutdown fired at tick %d on a flapping AC line", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A restart loop must not power the machine off using readings taken before
|
||||
// the hardware settled.
|
||||
func TestStartupInhibitionBlocksImmediateShutdown(t *testing.T) {
|
||||
m := New(shutdownConfig())
|
||||
for i := 0; i < 2; i++ {
|
||||
if m.Tick(onBattery(i), nil).Shutdown {
|
||||
t.Fatalf("shutdown fired at tick %d, inside the startup inhibition", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The worst remaining failure: the AC line reads 0 while mains is actually
|
||||
// connected. The charger then holds cell voltage flat or rising, which is what
|
||||
// the discharge guard exists to catch. Without it a single stuck GPIO line
|
||||
// powers the machine off during normal operation.
|
||||
func TestFalseACLossWhileChargingNeverTriggersShutdown(t *testing.T) {
|
||||
m := New(shutdownConfig())
|
||||
settle(t, m)
|
||||
|
||||
for i := 0; i < 60; i++ {
|
||||
r := sensor.Reading{
|
||||
ACPresent: false, // the line is lying
|
||||
Volts: 3.40 + float64(i)*0.005, // but the charger is working
|
||||
SOC: 10,
|
||||
}
|
||||
if m.Tick(r, nil).Shutdown {
|
||||
t.Fatalf("shutdown fired at tick %d while voltage was RISING; "+
|
||||
"a false AC reading must not power the machine off", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Guard against a future change that quietly drops the discharge requirement:
|
||||
// with it disabled, the same false reading DOES power the machine off. This
|
||||
// test documents the consequence rather than endorsing the setting.
|
||||
func TestWithoutDischargeGuardAFalseACReadingIsEnough(t *testing.T) {
|
||||
cfg := shutdownConfig()
|
||||
cfg.RequireDischarge = false
|
||||
m := New(cfg)
|
||||
settle(t, m)
|
||||
|
||||
fired := false
|
||||
for i := 0; i < 10 && !fired; i++ {
|
||||
r := sensor.Reading{ACPresent: false, Volts: 3.40, SOC: 10}
|
||||
fired = m.Tick(r, nil).Shutdown
|
||||
}
|
||||
if !fired {
|
||||
t.Error("expected the unguarded config to act on a constant low reading; " +
|
||||
"if this changed, update the RequireDischarge documentation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLowSOCAloneCanArm(t *testing.T) {
|
||||
m := New(shutdownConfig())
|
||||
settle(t, m)
|
||||
|
||||
// Voltage stays above the threshold; only state of charge is low.
|
||||
for i := 0; i < 3; i++ {
|
||||
r := sensor.Reading{ACPresent: false, Volts: 3.9 - float64(i)*0.01, SOC: 5}
|
||||
if m.Tick(r, nil).Shutdown {
|
||||
t.Fatalf("fired early at tick %d", i)
|
||||
}
|
||||
}
|
||||
r := sensor.Reading{ACPresent: false, Volts: 3.87, SOC: 5}
|
||||
if !m.Tick(r, nil).Shutdown {
|
||||
t.Error("a sustained low state of charge must trigger shutdown")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user