// Command x1208-exporter reads the Geekworm X1208 UPS HAT and writes a // Prometheus textfile for node-exporter's textfile collector. // // It runs as a long-running daemon under systemd (Type=notify). It replaces an // earlier oneshot-plus-timer, which produced 8367 journal lines a day of // systemd start/stop noise and a matching amount of SD card writeback. package main import ( "context" "flag" "fmt" "log/slog" "os" "os/exec" "os/signal" "strings" "sync/atomic" "syscall" "time" "github.com/coreos/go-systemd/v22/daemon" "git.mial.net/mokhtar/x1208-exporter/internal/max17040" "git.mial.net/mokhtar/x1208-exporter/internal/monitor" "git.mial.net/mokhtar/x1208-exporter/internal/pld" "git.mial.net/mokhtar/x1208-exporter/internal/sensor" "git.mial.net/mokhtar/x1208-exporter/internal/textfile" ) // version is set at build time with -ldflags "-X main.version=vX.Y.Z". var version = "dev" type options struct { out string interval time.Duration readTimeout time.Duration i2cDevice string gpioChip string gpioLine int fake bool shutdown bool shutdownVolts float64 shutdownSOC float64 shutdownSamples int shutdownSettle int shutdownDischarge bool shutdownCmd string showVersion bool } func parseFlags() *options { o := &options{} flag.StringVar(&o.out, "out", "/var/lib/node-exporter/textfiles/x1208.prom", "textfile path") flag.DurationVar(&o.interval, "interval", 30*time.Second, "sample interval") flag.DurationVar(&o.readTimeout, "read-timeout", 5*time.Second, "give up on a sensor read after this long") flag.StringVar(&o.i2cDevice, "i2c-device", max17040.DefaultDevice, "i2c device holding the fuel gauge") flag.StringVar(&o.gpioChip, "gpio-chip", pld.DefaultChip, "gpio chip carrying the AC-present line") flag.IntVar(&o.gpioLine, "gpio-line", pld.DefaultLine, "gpio line number for AC-present") flag.BoolVar(&o.fake, "fake", false, "use a scripted sensor instead of the HAT (development only)") flag.BoolVar(&o.shutdown, "shutdown", false, "power the machine off on a sustained low battery") flag.Float64Var(&o.shutdownVolts, "shutdown-voltage", 3.5, "cell voltage that qualifies as low") flag.Float64Var(&o.shutdownSOC, "shutdown-soc", 15, "state of charge percent that qualifies as low") flag.IntVar(&o.shutdownSamples, "shutdown-samples", 4, "consecutive qualifying samples before acting") flag.IntVar(&o.shutdownSettle, "shutdown-settle-ticks", 4, "suppress shutdown for this many ticks after start") flag.BoolVar(&o.shutdownDischarge, "shutdown-require-discharge", true, "require cell voltage to fall across the window; blocks a stuck sensor from powering the machine off") flag.StringVar(&o.shutdownCmd, "shutdown-cmd", "/run/current-system/sw/bin/systemctl poweroff", "command run to power the machine off") flag.BoolVar(&o.showVersion, "version", false, "print the version and exit") flag.Parse() return o } func main() { opts := parseFlags() if opts.showVersion { fmt.Println(version) return } log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) if err := run(opts, log); err != nil { log.Error("exporter failed", "err", err) os.Exit(1) } } func run(opts *options, log *slog.Logger) error { var src sensor.Sensor if opts.fake { src = sensor.Discharging(3, 40) log.Warn("using the fake sensor; no hardware is being read") } else { src = sensor.NewHardware(opts.i2cDevice, opts.gpioChip, opts.gpioLine) } defer src.Close() mon := monitor.New(monitor.Config{ Shutdown: opts.shutdown, Volts: opts.shutdownVolts, SOC: opts.shutdownSOC, Samples: opts.shutdownSamples, MinUptimeTicks: opts.shutdownSettle, RequireDischarge: opts.shutdownDischarge, }) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() // lastTick records when the sample loop last completed a full pass. The // watchdog pings from this, not from its own liveness — a watchdog // goroutine that reports health while the sample loop is wedged is worse // than no watchdog at all. var lastTick atomic.Int64 lastTick.Store(time.Now().UnixNano()) stopWatchdog := startWatchdog(ctx, &lastTick, opts.interval, log) defer stopWatchdog() if _, err := daemon.SdNotify(false, daemon.SdNotifyReady); err != nil { log.Warn("could not notify systemd of readiness", "err", err) } ticker := time.NewTicker(opts.interval) defer ticker.Stop() for { tick(ctx, src, mon, opts, log) lastTick.Store(time.Now().UnixNano()) select { case <-ctx.Done(): log.Info("shutting down on signal") return nil case <-ticker.C: } } } // tick performs one sample, writes the textfile, emits transition logs, and // acts on a shutdown decision. func tick(ctx context.Context, src sensor.Sensor, mon *monitor.Monitor, opts *options, log *slog.Logger) { reading, err := readWithTimeout(ctx, src, opts.readTimeout) res := mon.Tick(reading, err) for _, entry := range res.Logs { switch entry.Event { case monitor.EventErrorEnter, monitor.EventShutdownArmed, monitor.EventShutdownInhibited, monitor.EventShutdownTriggered: log.Warn(entry.Message, "event", string(entry.Event)) default: log.Info(entry.Message, "event", string(entry.Event)) } } // Nothing to publish until the first complete sample. Writing a zeroed // file would report 1970 as the last update and a flat battery. if res.HaveSample { content := textfile.Render(res.Sample, res.Healthy, res.Failures) if err := textfile.WriteAtomic(opts.out, content); err != nil { log.Error("could not write textfile", "path", opts.out, "err", err) } } if res.Shutdown { if err := powerOff(opts.shutdownCmd, log); err != nil { // Re-arm. A failed poweroff must not count as a completed one, or // the battery reaches cell cutoff and the HAT hard-cuts the // machine — the exact outcome this feature prevents. mon.ActuationFailed() log.Error("poweroff failed; will retry on the next qualifying tick", "attempt", mon.Retries(), "err", err) } } } // readWithTimeout bounds a sensor read. // // A blocked i2c or GPIO operation would otherwise stall the loop until the // systemd watchdog killed the whole process. The read goroutine is abandoned // rather than cancelled: the kernel owns the call, and a leaked goroutine per // wedged read is a smaller problem than a stalled sample loop. func readWithTimeout(ctx context.Context, src sensor.Sensor, timeout time.Duration) (sensor.Reading, error) { type outcome struct { reading sensor.Reading err error } done := make(chan outcome, 1) go func() { r, err := src.Read() done <- outcome{r, err} }() timer := time.NewTimer(timeout) defer timer.Stop() select { case o := <-done: return o.reading, o.err case <-timer.C: return sensor.Reading{}, fmt.Errorf("sensor read exceeded %s", timeout) case <-ctx.Done(): return sensor.Reading{}, ctx.Err() } } // startWatchdog pings systemd only while the sample loop is making progress. // // The cadence comes from WATCHDOG_USEC, halved, as systemd documents. If the // unit sets no watchdog, this does nothing. func startWatchdog(ctx context.Context, lastTick *atomic.Int64, interval time.Duration, log *slog.Logger) func() { timeout, err := daemon.SdWatchdogEnabled(false) if err != nil || timeout == 0 { return func() {} } // A tick is considered fresh for two intervals, so one slow sample does // not trip the watchdog while a genuinely wedged loop still does. stale := 2 * interval if stale >= timeout { log.Warn("sample interval is too long for the configured watchdog; "+ "systemd will restart the daemon during normal operation", "interval", interval, "watchdog", timeout) } ticker := time.NewTicker(timeout / 2) done := make(chan struct{}) go func() { defer ticker.Stop() for { select { case <-ctx.Done(): return case <-done: return case <-ticker.C: last := time.Unix(0, lastTick.Load()) if time.Since(last) > stale { // Deliberately silent: withholding the ping IS the // report. systemd restarts the daemon. continue } if _, err := daemon.SdNotify(false, daemon.SdNotifyWatchdog); err != nil { log.Warn("watchdog ping failed", "err", err) } } } }() return func() { close(done) } } // powerOff runs the configured poweroff command and reports whether it // succeeded. The caller re-arms on failure. func powerOff(command string, log *slog.Logger) error { fields := strings.Fields(command) if len(fields) == 0 { return fmt.Errorf("-shutdown-cmd is empty") } // Deliberately NOT sending SdNotifyStopping before the attempt. Announcing // a stop that then fails leaves systemd believing the unit is going away // while it keeps running. logind ends the session on a successful poweroff // regardless. out, err := exec.Command(fields[0], fields[1:]...).CombinedOutput() if err != nil { if trimmed := strings.TrimSpace(string(out)); trimmed != "" { return fmt.Errorf("%s: %w: %s", command, err, trimmed) } return fmt.Errorf("%s: %w", command, err) } log.Warn("poweroff requested", "cmd", command) return nil }