rewrite as a daemon with shutdown on low battery
the oneshot+timer produced 8367 journal lines a day of systemd start/stop noise. now Type=notify with an internal ticker, logging transitions only. hardware sits behind sensor.Sensor; internal/monitor is pure, so the shutdown state machine is tested without a pi. four guards, each tested, because a false poweroff of a box with no physical access is worse than a missed one: - a failed read resets the arming window rather than pausing it - voltage must actually fall across the window, else it is a stuck sensor. this is the only defence against the ac line reading 0 while mains is connected, so it is load-bearing - a settle period stops a restart loop acting on early readings - flapping ac cannot accumulate verified on hardware 2026-08-09: after poweroff on battery the x1208 starts the pi again when mains returns, with no button press. that was the risk that could have made this feature unsafe. a failed tick rewrites the previous sample with sensor_healthy 0 and the failure counter, so last_update still freezes for the staleness alert while the failure stays visible. module path moved to git.mial.net. debian units and the compose overlay deleted; deployment lives in the infra repo.
This commit is contained in:
@@ -1,181 +1,270 @@
|
||||
// Command x1208-exporter reads the Geekworm X1208 UPS HAT and writes a
|
||||
// Prometheus textfile for node-exporter's textfile collector.
|
||||
//
|
||||
// It runs as a long-running daemon under systemd (Type=notify). It replaces an
|
||||
// earlier oneshot-plus-timer, which produced 8367 journal lines a day of
|
||||
// systemd start/stop noise and a matching amount of SD card writeback.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/warthog618/go-gpiocdev"
|
||||
"golang.org/x/sys/unix"
|
||||
"github.com/coreos/go-systemd/v22/daemon"
|
||||
|
||||
"git.mial.net/mokhtar/x1208-exporter/internal/max17040"
|
||||
"git.mial.net/mokhtar/x1208-exporter/internal/monitor"
|
||||
"git.mial.net/mokhtar/x1208-exporter/internal/pld"
|
||||
"git.mial.net/mokhtar/x1208-exporter/internal/sensor"
|
||||
"git.mial.net/mokhtar/x1208-exporter/internal/textfile"
|
||||
)
|
||||
|
||||
const (
|
||||
i2cDev = "/dev/i2c-1"
|
||||
gpioChip = "gpiochip0"
|
||||
gpioPLDPin = 6
|
||||
maxAddr = 0x36
|
||||
regVoltage = 0x02
|
||||
regCapacity = 0x04
|
||||
i2cSlave = 0x0703 // I2C_SLAVE
|
||||
maxRetries = 3
|
||||
)
|
||||
// version is set at build time with -ldflags "-X main.version=vX.Y.Z".
|
||||
var version = "dev"
|
||||
|
||||
func readI2CWord(fd int, reg byte) (uint16, error) {
|
||||
if _, err := unix.Write(fd, []byte{reg}); err != nil {
|
||||
return 0, fmt.Errorf("write reg: %w", err)
|
||||
}
|
||||
buf := make([]byte, 2)
|
||||
n, err := unix.Read(fd, buf)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("read: %w", err)
|
||||
}
|
||||
if n != 2 {
|
||||
return 0, fmt.Errorf("short read: %d bytes", n)
|
||||
}
|
||||
return binary.BigEndian.Uint16(buf), nil
|
||||
type options struct {
|
||||
out string
|
||||
interval time.Duration
|
||||
readTimeout time.Duration
|
||||
i2cDevice string
|
||||
gpioChip string
|
||||
gpioLine int
|
||||
fake bool
|
||||
|
||||
shutdown bool
|
||||
shutdownVolts float64
|
||||
shutdownSOC float64
|
||||
shutdownSamples int
|
||||
shutdownSettle int
|
||||
shutdownDischarge bool
|
||||
shutdownCmd string
|
||||
|
||||
showVersion bool
|
||||
}
|
||||
|
||||
func readI2CWordRetry(fd int, reg byte, validate func(uint16) bool) (uint16, error) {
|
||||
var lastErr error
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
v, err := readI2CWord(fd, reg)
|
||||
if err == nil && validate(v) {
|
||||
return v, nil
|
||||
}
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
lastErr = fmt.Errorf("read out of range: 0x%04x", v)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
return 0, lastErr
|
||||
}
|
||||
func parseFlags() *options {
|
||||
o := &options{}
|
||||
flag.StringVar(&o.out, "out", "/var/lib/node-exporter/textfiles/x1208.prom", "textfile path")
|
||||
flag.DurationVar(&o.interval, "interval", 30*time.Second, "sample interval")
|
||||
flag.DurationVar(&o.readTimeout, "read-timeout", 5*time.Second, "give up on a sensor read after this long")
|
||||
flag.StringVar(&o.i2cDevice, "i2c-device", max17040.DefaultDevice, "i2c device holding the fuel gauge")
|
||||
flag.StringVar(&o.gpioChip, "gpio-chip", pld.DefaultChip, "gpio chip carrying the AC-present line")
|
||||
flag.IntVar(&o.gpioLine, "gpio-line", pld.DefaultLine, "gpio line number for AC-present")
|
||||
flag.BoolVar(&o.fake, "fake", false, "use a scripted sensor instead of the HAT (development only)")
|
||||
|
||||
func readBattery() (voltage float64, percent float64, err error) {
|
||||
fd, err := unix.Open(i2cDev, unix.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("open %s: %w", i2cDev, err)
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
flag.BoolVar(&o.shutdown, "shutdown", false, "power the machine off on a sustained low battery")
|
||||
flag.Float64Var(&o.shutdownVolts, "shutdown-voltage", 3.5, "cell voltage that qualifies as low")
|
||||
flag.Float64Var(&o.shutdownSOC, "shutdown-soc", 15, "state of charge percent that qualifies as low")
|
||||
flag.IntVar(&o.shutdownSamples, "shutdown-samples", 4, "consecutive qualifying samples before acting")
|
||||
flag.IntVar(&o.shutdownSettle, "shutdown-settle-ticks", 4, "suppress shutdown for this many ticks after start")
|
||||
flag.BoolVar(&o.shutdownDischarge, "shutdown-require-discharge", true,
|
||||
"require cell voltage to fall across the window; blocks a stuck sensor from powering the machine off")
|
||||
flag.StringVar(&o.shutdownCmd, "shutdown-cmd", "/run/current-system/sw/bin/systemctl poweroff",
|
||||
"command run to power the machine off")
|
||||
|
||||
if err := unix.IoctlSetInt(fd, i2cSlave, maxAddr); err != nil {
|
||||
return 0, 0, fmt.Errorf("ioctl I2C_SLAVE: %w", err)
|
||||
}
|
||||
|
||||
rawV, err := readI2CWordRetry(fd, regVoltage, func(v uint16) bool {
|
||||
f := float64(v) * 1.25 / 1000 / 16
|
||||
return f > 2.0 && f < 5.0
|
||||
})
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("voltage: %w", err)
|
||||
}
|
||||
voltage = float64(rawV) * 1.25 / 1000 / 16
|
||||
|
||||
rawC, err := readI2CWordRetry(fd, regCapacity, func(v uint16) bool {
|
||||
return float64(v)/256 <= 110
|
||||
})
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("capacity: %w", err)
|
||||
}
|
||||
percent = float64(rawC) / 256
|
||||
return voltage, percent, nil
|
||||
}
|
||||
|
||||
func readACPower() (int, error) {
|
||||
line, err := gpiocdev.RequestLine(gpioChip, gpioPLDPin,
|
||||
gpiocdev.AsInput,
|
||||
gpiocdev.WithConsumer("x1208-exporter"))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("request gpio: %w", err)
|
||||
}
|
||||
defer line.Close()
|
||||
|
||||
v, err := line.Value()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("read gpio: %w", err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func writeAtomic(path string, content string) error {
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, ".x1208.*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer os.Remove(tmpPath)
|
||||
if _, err := tmp.WriteString(content); err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Chmod(0o644); err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmpPath, path)
|
||||
flag.BoolVar(&o.showVersion, "version", false, "print the version and exit")
|
||||
flag.Parse()
|
||||
return o
|
||||
}
|
||||
|
||||
func main() {
|
||||
out := flag.String("out", "/var/lib/node-exporter/textfiles/x1208.prom", "output prom file")
|
||||
flag.Parse()
|
||||
|
||||
var (
|
||||
ac int
|
||||
voltage float64
|
||||
percent float64
|
||||
acErr, batErr error
|
||||
)
|
||||
|
||||
ac, acErr = readACPower()
|
||||
voltage, percent, batErr = readBattery()
|
||||
|
||||
if acErr != nil && batErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "all reads failed: ac=%v bat=%v\n", acErr, batErr)
|
||||
os.Exit(1)
|
||||
opts := parseFlags()
|
||||
if opts.showVersion {
|
||||
fmt.Println(version)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now().Unix()
|
||||
var content string
|
||||
content += "# HELP rpi_ups_last_update_seconds Unix timestamp of last successful exporter run.\n"
|
||||
content += "# TYPE rpi_ups_last_update_seconds gauge\n"
|
||||
content += fmt.Sprintf("rpi_ups_last_update_seconds %d\n", now)
|
||||
|
||||
if acErr == nil {
|
||||
content += "# HELP rpi_ups_ac_power AC adapter present (1=plugged in, 0=on battery).\n"
|
||||
content += "# TYPE rpi_ups_ac_power gauge\n"
|
||||
content += fmt.Sprintf("rpi_ups_ac_power %d\n", ac)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "ac read failed: %v\n", acErr)
|
||||
}
|
||||
|
||||
if batErr == nil {
|
||||
content += "# HELP rpi_ups_voltage_volts Battery cell voltage from MAX17040 fuel gauge.\n"
|
||||
content += "# TYPE rpi_ups_voltage_volts gauge\n"
|
||||
content += fmt.Sprintf("rpi_ups_voltage_volts %.3f\n", voltage)
|
||||
content += "# HELP rpi_ups_battery_percent Battery state of charge from MAX17040 fuel gauge.\n"
|
||||
content += "# TYPE rpi_ups_battery_percent gauge\n"
|
||||
content += fmt.Sprintf("rpi_ups_battery_percent %.2f\n", percent)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "battery read failed: %v\n", batErr)
|
||||
}
|
||||
|
||||
if err := writeAtomic(*out, content); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "write %s: %v\n", *out, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if acErr != nil || batErr != nil {
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
if err := run(opts, log); err != nil {
|
||||
log.Error("exporter failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(opts *options, log *slog.Logger) error {
|
||||
var src sensor.Sensor
|
||||
if opts.fake {
|
||||
src = sensor.Discharging(3, 40)
|
||||
log.Warn("using the fake sensor; no hardware is being read")
|
||||
} else {
|
||||
src = sensor.NewHardware(opts.i2cDevice, opts.gpioChip, opts.gpioLine)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
mon := monitor.New(monitor.Config{
|
||||
Shutdown: opts.shutdown,
|
||||
Volts: opts.shutdownVolts,
|
||||
SOC: opts.shutdownSOC,
|
||||
Samples: opts.shutdownSamples,
|
||||
MinUptimeTicks: opts.shutdownSettle,
|
||||
RequireDischarge: opts.shutdownDischarge,
|
||||
})
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
// lastTick records when the sample loop last completed a full pass. The
|
||||
// watchdog pings from this, not from its own liveness — a watchdog
|
||||
// goroutine that reports health while the sample loop is wedged is worse
|
||||
// than no watchdog at all.
|
||||
var lastTick atomic.Int64
|
||||
lastTick.Store(time.Now().UnixNano())
|
||||
|
||||
stopWatchdog := startWatchdog(ctx, &lastTick, opts.interval, log)
|
||||
defer stopWatchdog()
|
||||
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyReady); err != nil {
|
||||
log.Warn("could not notify systemd of readiness", "err", err)
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(opts.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
tick(ctx, src, mon, opts, log)
|
||||
lastTick.Store(time.Now().UnixNano())
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Info("shutting down on signal")
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tick performs one sample, writes the textfile, emits transition logs, and
|
||||
// acts on a shutdown decision.
|
||||
func tick(ctx context.Context, src sensor.Sensor, mon *monitor.Monitor, opts *options, log *slog.Logger) {
|
||||
reading, err := readWithTimeout(ctx, src, opts.readTimeout)
|
||||
res := mon.Tick(reading, err)
|
||||
|
||||
for _, entry := range res.Logs {
|
||||
switch entry.Event {
|
||||
case monitor.EventErrorEnter, monitor.EventShutdownArmed,
|
||||
monitor.EventShutdownInhibited, monitor.EventShutdownTriggered:
|
||||
log.Warn(entry.Message, "event", string(entry.Event))
|
||||
default:
|
||||
log.Info(entry.Message, "event", string(entry.Event))
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing to publish until the first complete sample. Writing a zeroed
|
||||
// file would report 1970 as the last update and a flat battery.
|
||||
if res.HaveSample {
|
||||
content := textfile.Render(res.Sample, res.Healthy, res.Failures)
|
||||
if err := textfile.WriteAtomic(opts.out, content); err != nil {
|
||||
log.Error("could not write textfile", "path", opts.out, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
if res.Shutdown {
|
||||
powerOff(opts.shutdownCmd, log)
|
||||
}
|
||||
}
|
||||
|
||||
// readWithTimeout bounds a sensor read.
|
||||
//
|
||||
// A blocked i2c or GPIO operation would otherwise stall the loop until the
|
||||
// systemd watchdog killed the whole process. The read goroutine is abandoned
|
||||
// rather than cancelled: the kernel owns the call, and a leaked goroutine per
|
||||
// wedged read is a smaller problem than a stalled sample loop.
|
||||
func readWithTimeout(ctx context.Context, src sensor.Sensor, timeout time.Duration) (sensor.Reading, error) {
|
||||
type outcome struct {
|
||||
reading sensor.Reading
|
||||
err error
|
||||
}
|
||||
done := make(chan outcome, 1)
|
||||
go func() {
|
||||
r, err := src.Read()
|
||||
done <- outcome{r, err}
|
||||
}()
|
||||
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case o := <-done:
|
||||
return o.reading, o.err
|
||||
case <-timer.C:
|
||||
return sensor.Reading{}, fmt.Errorf("sensor read exceeded %s", timeout)
|
||||
case <-ctx.Done():
|
||||
return sensor.Reading{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// startWatchdog pings systemd only while the sample loop is making progress.
|
||||
//
|
||||
// The cadence comes from WATCHDOG_USEC, halved, as systemd documents. If the
|
||||
// unit sets no watchdog, this does nothing.
|
||||
func startWatchdog(ctx context.Context, lastTick *atomic.Int64, interval time.Duration, log *slog.Logger) func() {
|
||||
timeout, err := daemon.SdWatchdogEnabled(false)
|
||||
if err != nil || timeout == 0 {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
// A tick is considered fresh for two intervals, so one slow sample does
|
||||
// not trip the watchdog while a genuinely wedged loop still does.
|
||||
stale := 2 * interval
|
||||
if stale >= timeout {
|
||||
log.Warn("sample interval is too long for the configured watchdog; "+
|
||||
"systemd will restart the daemon during normal operation",
|
||||
"interval", interval, "watchdog", timeout)
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(timeout / 2)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
last := time.Unix(0, lastTick.Load())
|
||||
if time.Since(last) > stale {
|
||||
// Deliberately silent: withholding the ping IS the
|
||||
// report. systemd restarts the daemon.
|
||||
continue
|
||||
}
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyWatchdog); err != nil {
|
||||
log.Warn("watchdog ping failed", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { close(done) }
|
||||
}
|
||||
|
||||
func powerOff(command string, log *slog.Logger) {
|
||||
fields := strings.Fields(command)
|
||||
if len(fields) == 0 {
|
||||
log.Error("shutdown requested but -shutdown-cmd is empty")
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyStopping); err != nil {
|
||||
log.Warn("could not notify systemd that we are stopping", "err", err)
|
||||
}
|
||||
|
||||
out, err := exec.Command(fields[0], fields[1:]...).CombinedOutput()
|
||||
if err != nil {
|
||||
// Do not retry. A poweroff that fails needs a human, and a retry loop
|
||||
// would fork repeatedly while the battery drains.
|
||||
log.Error("poweroff command failed",
|
||||
"cmd", command, "err", err, "output", strings.TrimSpace(string(out)))
|
||||
return
|
||||
}
|
||||
log.Warn("poweroff requested", "cmd", command)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user