Commit b95ae48

Eric Bower  ·  2026-05-06 20:18:57 -0400 EDT
parent b95ae48 
feat: init
3 files changed,  +1178, -0
A go.mod
A go.sum
A main.go
A go.mod
+5, -0
......@@ -0,0 +1,5 @@
1+module github.com/picosh/ci
2+
3+go 1.26.2
4+
5+require github.com/golang-cz/devslog v0.0.15
A go.sum
+2, -0
......@@ -0,0 +1,2 @@
1+github.com/golang-cz/devslog v0.0.15 h1:ejoBLTCwJHWGbAmDf2fyTJJQO3AkzcPjw8SC9LaOQMI=
2+github.com/golang-cz/devslog v0.0.15/go.mod h1:bSe5bm0A7Nyfqtijf1OMNgVJHlWEuVSXnkuASiE1vV8=
A main.go
+1171, -0
......@@ -0,0 +1,1171 @@
1+package main
2+
3+import (
4+ "bufio"
5+ "context"
6+ "crypto/sha256"
7+ "encoding/json"
8+ "flag"
9+ "fmt"
10+ "html/template"
11+ "io"
12+ "log/slog"
13+ "os"
14+ "os/exec"
15+ "path/filepath"
16+ "strings"
17+ "time"
18+
19+ "github.com/golang-cz/devslog"
20+)
21+
22+type WorkspaceFactory func(cfg *Cfg, logger *slog.Logger, source string) Workspace
23+
24+func defaultWorkspaceFactory(cfg *Cfg, logger *slog.Logger, source string) Workspace {
25+ return &WorkspaceRsync{
26+ Cfg: cfg,
27+ Logger: logger,
28+ Source: source,
29+ }
30+}
31+
32+type Cfg struct {
33+ Logger *slog.Logger
34+ Ctx context.Context
35+ Cancel context.CancelFunc
36+ KeyLocation string
37+ CertificateLocation string
38+ ArtifactDir string
39+ Event string // event JSON passed via --event flag
40+ EventSource io.ReadCloser // when set, used directly as the event source (for testing)
41+ MonitorInterval time.Duration
42+ NewWorkspace WorkspaceFactory
43+ StatusOutput io.Writer // where status JSONL is written (default: os.Stdout)
44+ StatusFilter string // "terminal" (default) or "all"
45+}
46+
47+type Event struct {
48+ Type string `json:"type"`
49+ Name string `json:"name"`
50+ Workspace string `json:"workspace"`
51+ ArtifactDest string `json:"artifact_dest"`
52+ ArtifactURL string `json:"artifact_url"` // public URL for artifacts (e.g. https://{user}-artifacts.pgs.sh/{repo})
53+}
54+
55+func NewCfg() *Cfg {
56+ var keyLoc, certLoc, artifactDir, event string
57+ var monitorInterval time.Duration
58+ var logLevel string
59+ var structured bool
60+ flag.StringVar(&keyLoc, "pk", "", "ssh private key used to authenticate with pico services")
61+ flag.StringVar(&certLoc, "ck", "", "ssh certificate public key used to authenticate with pico services (only required if using ssh certificates)")
62+ flag.StringVar(&artifactDir, "artifact-dir", "/tmp/pico-ci-artifacts", "local directory to stage artifacts")
63+ flag.StringVar(&event, "event", "", "event JSON to run (alternative to reading from stdin)")
64+ flag.DurationVar(&monitorInterval, "monitor-interval", 5*time.Second, "interval for monitoring zmx sessions")
65+ flag.StringVar(&logLevel, "log-level", "info", "log level: debug, info, warn, error")
66+ flag.BoolVar(&structured, "structured", false, "use structured key=value log output")
67+ var statusFilter string
68+ flag.StringVar(&statusFilter, "status-filter", "terminal", "status output filter: terminal (default) or all")
69+ flag.Parse()
70+
71+ logger := newLogger("ci", logLevel, structured)
72+ ctx, cancel := context.WithCancel(context.Background())
73+ return &Cfg{
74+ NewWorkspace: defaultWorkspaceFactory,
75+ Logger: logger.With("key_loc", keyLoc, "cert_loc", certLoc),
76+ Ctx: ctx,
77+ Cancel: cancel,
78+ KeyLocation: keyLoc,
79+ CertificateLocation: certLoc,
80+ ArtifactDir: artifactDir,
81+ Event: event,
82+ MonitorInterval: monitorInterval,
83+ StatusFilter: statusFilter,
84+ }
85+}
86+
87+func parseLogLevel(s string) slog.Level {
88+ switch strings.ToLower(s) {
89+ case "debug":
90+ return slog.LevelDebug
91+ case "warn":
92+ return slog.LevelWarn
93+ case "error":
94+ return slog.LevelError
95+ default:
96+ return slog.LevelInfo
97+ }
98+}
99+
100+func newLogger(space string, levelStr string, structured bool) *slog.Logger {
101+ lvl := parseLogLevel(levelStr)
102+ if structured {
103+ return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
104+ Level: lvl,
105+ })).With("service", space)
106+ }
107+ return slog.New(devslog.NewHandler(os.Stderr, &devslog.Options{
108+ HandlerOptions: &slog.HandlerOptions{
109+ Level: lvl,
110+ },
111+ })).With("service", space)
112+}
113+
114+func main() {
115+ cfg := NewCfg()
116+ cmd := flag.Arg(0)
117+
118+ cfg.Logger.Debug("setting up ci", "cfg", cfg)
119+ cfg.Logger.Debug("running cmd", "cmd", cmd)
120+
121+ switch cmd {
122+ case "runner":
123+ cfg.Logger.Debug("starting runner")
124+ if err := RunRunner(cfg); err != nil {
125+ cfg.Logger.Error("runner failed", "err", err)
126+ os.Exit(1)
127+ }
128+ case "cancel":
129+ cfg.Logger.Debug("starting cancel handler")
130+ if err := runCancel(cfg); err != nil {
131+ cfg.Logger.Error("cancel failed", "err", err)
132+ os.Exit(1)
133+ }
134+ case "gc":
135+ cfg.Logger.Debug("starting garbage collection")
136+ if err := runGC(cfg); err != nil {
137+ cfg.Logger.Error("gc failed", "err", err)
138+ os.Exit(1)
139+ }
140+ case "monitor":
141+ cfg.Logger.Debug("starting monitor")
142+ if err := runMonitor(cfg); err != nil {
143+ cfg.Logger.Error("monitor failed", "err", err)
144+ os.Exit(1)
145+ }
146+ case "status":
147+ cfg.Logger.Debug("starting status updater")
148+ case "orca":
149+ cfg.Logger.Debug("starting orchestrator")
150+ default:
151+ cfg.Logger.Error("must provide command: runner, cancel, gc, monitor, status, or orca")
152+ os.Exit(1)
153+ }
154+}
155+
156+func RunRunner(cfg *Cfg) error {
157+ var payload string
158+ if cfg.EventSource != nil {
159+ data, err := io.ReadAll(cfg.EventSource)
160+ if err != nil {
161+ return fmt.Errorf("read event source: %w", err)
162+ }
163+ payload = strings.TrimSpace(string(data))
164+ } else if cfg.Event != "" {
165+ payload = cfg.Event
166+ } else {
167+ data, err := io.ReadAll(os.Stdin)
168+ if err != nil {
169+ return fmt.Errorf("read stdin: %w", err)
170+ }
171+ payload = strings.TrimSpace(string(data))
172+ }
173+
174+ var eventData Event
175+ if err := json.Unmarshal([]byte(payload), &eventData); err != nil {
176+ return fmt.Errorf("unmarshal event: %w", err)
177+ }
178+
179+ cfg.Logger.Info("received event", "type", eventData.Type, "repo", eventData.Name, "workspace", eventData.Workspace)
180+
181+ return eventHandler(cfg, &eventData)
182+}
183+
184+type Workspace interface {
185+ Setup() error
186+ Cleanup() error
187+ GetDir() string
188+}
189+
190+type WorkspaceRsync struct {
191+ Cfg *Cfg
192+ Logger *slog.Logger
193+ Source string
194+ Dest string
195+}
196+
197+func (w *WorkspaceRsync) Setup() error {
198+ tempDir, err := os.MkdirTemp("", "pico-ci-*")
199+ if err != nil {
200+ return err
201+ }
202+ w.Dest = tempDir
203+
204+ log := w.Logger.With("source", w.Source, "dest", w.Dest)
205+ log.Debug("syncing workspace via rsync")
206+
207+ var cmd *exec.Cmd
208+ if w.Cfg.KeyLocation != "" {
209+ sshcmd := fmt.Sprintf(
210+ "-i %s -o IdentitiesOnly=yes -o CertificateFile %s",
211+ w.Cfg.KeyLocation,
212+ w.Cfg.CertificateLocation,
213+ )
214+ cmd = exec.Command("rsync", "-e", sshcmd, "-rv", `--exclude="/.git"`, w.Source+"/", w.Dest+"/")
215+ } else {
216+ cmd = exec.Command("rsync", "-rv", `--exclude="/.git"`, w.Source+"/", w.Dest+"/")
217+ }
218+ return runCmd(cmd, log)
219+}
220+
221+func (w *WorkspaceRsync) Cleanup() error {
222+ // return os.RemoveAll(w.Dest)
223+ return nil
224+}
225+
226+func (w *WorkspaceRsync) GetDir() string {
227+ return w.Dest
228+}
229+
230+type JobEngine struct {
231+ Wk Workspace
232+ Logger *slog.Logger
233+ Cfg *Cfg
234+ Ev *Event
235+ JobID string
236+}
237+
238+type SessionInfo struct {
239+ Name string `json:"name"`
240+ Short string `json:"short"`
241+ PID string `json:"pid"`
242+ Clients string `json:"clients"`
243+ Created string `json:"created"`
244+ StartDir string `json:"start_dir"`
245+ Ended string `json:"ended"`
246+ ExitCode string `json:"exit_code"`
247+}
248+
249+type StatusPayload struct {
250+ Name string `json:"name"`
251+ JobID string `json:"job_id"`
252+ Status string `json:"status"`
253+ ExitCode *int `json:"exit_code"`
254+ Duration string `json:"duration,omitempty"`
255+ StartedAt string `json:"started_at,omitempty"`
256+ EndedAt string `json:"ended_at,omitempty"`
257+ ArtifactURL string `json:"artifact_url,omitempty"`
258+ SessionCount int `json:"session_count"`
259+ Sessions []SessionInfo `json:"sessions"`
260+}
261+
262+func (eng *JobEngine) Setup() error {
263+ return eng.Wk.Setup()
264+}
265+
266+func (eng *JobEngine) Run() error {
267+ manifest, err := eng.getManifest()
268+ if err != nil {
269+ return err
270+ }
271+
272+ prefix := fmt.Sprintf("ci.%s.%s.", eng.Ev.Name, eng.JobID)
273+
274+ log := eng.Logger.With("manifest", manifest, "prefix", prefix)
275+ log.Debug("starting runner session", "session", prefix+"runner")
276+
277+ evStr := fmt.Sprintf("PICO_CI_EVENT=%s", eng.Ev.Type)
278+ jobStr := fmt.Sprintf("ZMX_SESSION_PREFIX=%s", prefix)
279+
280+ cmd := exec.Command("zmx", "run", "runner", "-d", "bash", manifest)
281+ cmd.Env = append(os.Environ(), evStr, jobStr)
282+ cmd.Dir = eng.Wk.GetDir()
283+
284+ if err := cmd.Run(); err != nil {
285+ return fmt.Errorf("start runner session: %w", err)
286+ }
287+
288+ return nil
289+}
290+
291+func (eng *JobEngine) Cleanup() error {
292+ return eng.Wk.Cleanup()
293+}
294+
295+func (eng *JobEngine) getManifest() (string, error) {
296+ fnames := []string{"pico.sh"}
297+ for _, manifest := range fnames {
298+ path := filepath.Join(eng.Wk.GetDir(), manifest)
299+ if _, err := os.Stat(path); err == nil {
300+ return path, nil
301+ }
302+ }
303+ return "", fmt.Errorf("manifest not found in %s", eng.Wk.GetDir())
304+}
305+
306+func eventHandler(cfg *Cfg, eventData *Event) error {
307+ log := cfg.Logger.With("repo", eventData.Name, "type", eventData.Type)
308+ log.Info("processing event", "workspace", eventData.Workspace)
309+
310+ // Cancel any existing job for this repo before starting a new one
311+ cancelRunningJobs(cfg, log, eventData.Name)
312+
313+ jobID := generateJobID(eventData.Name, eventData.Workspace)
314+ log = log.With("job_id", jobID)
315+ log.Info("starting job", "session_prefix", fmt.Sprintf("ci.%s.%s", eventData.Name, jobID))
316+
317+ wk := cfg.NewWorkspace(cfg, log, eventData.Workspace)
318+ eng := &JobEngine{
319+ Logger: log,
320+ Cfg: cfg,
321+ Wk: wk,
322+ Ev: eventData,
323+ JobID: jobID,
324+ }
325+ defer func() {
326+ if err := eng.Cleanup(); err != nil {
327+ cfg.Logger.Error("engine cleanup", "err", err)
328+ }
329+ }()
330+
331+ log.Info("cloning workspace", "source", eventData.Workspace)
332+ if err := eng.Setup(); err != nil {
333+ return fmt.Errorf("setup: %w", err)
334+ }
335+ log.Info("workspace ready", "dir", eng.Wk.GetDir())
336+
337+ // Store the event in the artifact directory so the monitor can access it
338+ eventBytes, _ := json.Marshal(eventData)
339+ eventDir := filepath.Join(cfg.ArtifactDir, eventData.Name, jobID)
340+ if err := os.MkdirAll(eventDir, 0755); err != nil {
341+ log.Error("create event dir", "err", err)
342+ } else {
343+ if err := os.WriteFile(filepath.Join(eventDir, "event.json"), eventBytes, 0644); err != nil {
344+ log.Error("write event", "err", err)
345+ }
346+ }
347+
348+ log.Info("launching job sessions")
349+ if err := eng.Run(); err != nil {
350+ return fmt.Errorf("run: %w", err)
351+ }
352+
353+ log.Info("job launched successfully — monitor will track progress", "artifact_dir", cfg.ArtifactDir, "artifact_dest", eventData.ArtifactDest)
354+ log.Info("follow the runner live", "command", fmt.Sprintf("zmx tail ci.%s.%s.runner", eventData.Name, jobID))
355+ return nil
356+}
357+
358+// runMonitor is a long-lived daemon that polls all ci.* zmx sessions,
359+// writes status as JSONL to stdout, stages artifacts, and syncs to destination.
360+// Logs go to stderr. Compose with shell tools to route status:
361+//
362+// pico-ci monitor | ssh pipe.pico.sh "pub build.status -b=false"
363+// pico-ci monitor | while read -r line; do curl -sd"$line" $WEBHOOK; done
364+// pico-ci monitor > status.jsonl
365+func runMonitor(cfg *Cfg) error {
366+ log := cfg.Logger.With("cmd", "monitor")
367+
368+ output := cfg.StatusOutput
369+ if output == nil {
370+ output = os.Stdout
371+ }
372+
373+ ticker := time.NewTicker(cfg.MonitorInterval)
374+ defer ticker.Stop()
375+
376+ log.Debug("monitor started", "interval", cfg.MonitorInterval, "artifact_dir", cfg.ArtifactDir)
377+ log.Debug("monitoring ci.* sessions for job status, writing status to stdout")
378+
379+ for {
380+ select {
381+ case <-cfg.Ctx.Done():
382+ log.Debug("context cancelled, stopping monitor")
383+ return cfg.Ctx.Err()
384+ case <-ticker.C:
385+ }
386+
387+ if err := monitorTick(cfg, log, output); err != nil {
388+ log.Error("monitor tick", "err", err)
389+ }
390+ }
391+}
392+
393+// monitorTick performs a single monitoring pass over all ci.* sessions.
394+func monitorTick(cfg *Cfg, log *slog.Logger, output io.Writer) error {
395+ // a. zmx list → filter ci.* sessions
396+ listOutput, err := exec.Command("zmx", "list").CombinedOutput()
397+ if err != nil {
398+ return fmt.Errorf("zmx list: %w", err)
399+ }
400+ sessions := parseZMXList(string(listOutput))
401+ ciSessions := filterCISessions(sessions)
402+
403+ if len(ciSessions) == 0 {
404+ log.Debug("no ci.* sessions found")
405+ return nil
406+ }
407+
408+ log.Debug("found ci sessions", "count", len(ciSessions))
409+
410+ // b. Group by job prefix: ci.<name>.<jobID>.
411+ groups := groupSessionsByJob(ciSessions)
412+
413+ // c. Process each job group
414+ for prefix, group := range groups {
415+ name, jobID := parseJobPrefix(prefix)
416+ if name == "" {
417+ continue
418+ }
419+
420+ log := log.With("repo", name, "job_id", jobID)
421+
422+ // Fetch and stage history for every session at each tick,
423+ // not just when the job completes. This gives live progress
424+ // snapshots while the job is running.
425+ for _, s := range group {
426+ html, err := fetchHistoryHTML(s.Name)
427+ if err != nil {
428+ log.Error("fetch history html", "session", s.Name, "err", err)
429+ continue
430+ }
431+ if err := stageArtifact(cfg.ArtifactDir, name, jobID, s.Short, html, ".html"); err != nil {
432+ log.Error("stage html artifact", "session", s.Short, "err", err)
433+ }
434+
435+ plain, err := fetchHistoryPlain(s.Name)
436+ if err != nil {
437+ log.Error("fetch history plain", "session", s.Name, "err", err)
438+ continue
439+ }
440+ if err := stageArtifact(cfg.ArtifactDir, name, jobID, s.Short, plain, ".txt"); err != nil {
441+ log.Error("stage txt artifact", "session", s.Short, "err", err)
442+ }
443+ }
444+
445+ // Generate and stage job index landing pages
446+ indexHTML, indexTXT := generateJobIndex(name, jobID, group)
447+ if err := stageArtifact(cfg.ArtifactDir, name, jobID, "index", indexHTML, ".html"); err != nil {
448+ log.Error("stage index.html", "err", err)
449+ }
450+ if err := stageArtifact(cfg.ArtifactDir, name, jobID, "index", indexTXT, ".txt"); err != nil {
451+ log.Error("stage index.txt", "err", err)
452+ }
453+
454+ // Load event to get artifact URL
455+ eventData, _ := loadEvent(cfg.ArtifactDir, name, jobID)
456+
457+ // Build artifact URL: artifact_url/jobID/index.html
458+ var artifactURL string
459+ if eventData.ArtifactURL != "" {
460+ artifactURL = fmt.Sprintf("%s/%s/index.html", strings.TrimRight(eventData.ArtifactURL, "/"), jobID)
461+ }
462+
463+ // Compute job-level timing from session timestamps
464+ startedAt, endedAt, duration := computeJobTiming(group)
465+
466+ if allCompleted(group) {
467+ // Check sentinel — publish terminal status exactly once
468+ sentinel := filepath.Join(cfg.ArtifactDir, name, jobID, "published.json")
469+ if _, err := os.Stat(sentinel); err == nil {
470+ log.Debug("terminal status already published, skipping", "sentinel", sentinel)
471+ continue
472+ }
473+
474+ log.Debug("job completed, publishing final status", "sessions", len(group))
475+ exitCode, status := resolveJobExitCode(group)
476+ log.Info("job finished", "status", status, "exit_code", exitCode)
477+ payload := StatusPayload{
478+ Name: name,
479+ JobID: jobID,
480+ Status: status,
481+ ExitCode: &exitCode,
482+ Duration: duration,
483+ StartedAt: startedAt,
484+ EndedAt: endedAt,
485+ ArtifactURL: artifactURL,
486+ SessionCount: len(group),
487+ Sessions: group,
488+ }
489+ if err := publishStatus(output, payload); err != nil {
490+ log.Error("publish final status", "err", err)
491+ }
492+ // Write sentinel so we don't re-publish on subsequent ticks
493+ if err := os.WriteFile(sentinel, []byte(status), 0644); err != nil {
494+ log.Error("write published sentinel", "err", err)
495+ }
496+ } else {
497+ log.Debug("job still running", "sessions", len(group))
498+ // Publish running status only when filter is "all"
499+ if cfg.StatusFilter != "terminal" {
500+ payload := StatusPayload{
501+ Name: name,
502+ JobID: jobID,
503+ Status: "running",
504+ ExitCode: nil,
505+ Duration: duration,
506+ StartedAt: startedAt,
507+ ArtifactURL: artifactURL,
508+ SessionCount: len(group),
509+ Sessions: group,
510+ }
511+ if err := publishStatus(output, payload); err != nil {
512+ log.Error("publish status", "err", err)
513+ }
514+ }
515+ }
516+ }
517+
518+ // d. Sync artifacts once per tick
519+ if err := syncArtifacts(cfg, log); err != nil {
520+ log.Error("sync artifacts", "err", err)
521+ }
522+
523+ return nil
524+}
525+
526+// loadEvent reads the event.json for a job from the artifact directory.
527+func loadEvent(dir, name, jobID string) (Event, error) {
528+ var event Event
529+ data, err := os.ReadFile(filepath.Join(dir, name, jobID, "event.json"))
530+ if err != nil {
531+ return event, err
532+ }
533+ return event, json.Unmarshal(data, &event)
534+}
535+
536+// computeJobTiming derives started_at, ended_at, and duration from session timestamps.
537+// started_at is the earliest session creation time, ended_at is the latest session end time.
538+// For running jobs, ended_at is empty and duration is computed against now.
539+func computeJobTiming(sessions []SessionInfo) (startedAt, endedAt, duration string) {
540+ var earliestCreated, latestEnded int64
541+ hasCreated, hasEnded := false, false
542+
543+ for _, s := range sessions {
544+ if s.Created != "" {
545+ var c int64
546+ if _, err := fmt.Sscanf(s.Created, "%d", &c); err == nil {
547+ if !hasCreated || c < earliestCreated {
548+ earliestCreated = c
549+ }
550+ hasCreated = true
551+ }
552+ }
553+ if s.Ended != "" {
554+ var e int64
555+ if _, err := fmt.Sscanf(s.Ended, "%d", &e); err == nil {
556+ if !hasEnded || e > latestEnded {
557+ latestEnded = e
558+ }
559+ hasEnded = true
560+ }
561+ }
562+ }
563+
564+ if hasCreated {
565+ startedAt = time.Unix(earliestCreated, 0).UTC().Format(time.RFC3339)
566+ }
567+ if hasEnded {
568+ endedAt = time.Unix(latestEnded, 0).UTC().Format(time.RFC3339)
569+ }
570+
571+ if hasCreated && hasEnded {
572+ duration = fmtDurationTs(earliestCreated, latestEnded)
573+ } else if hasCreated {
574+ duration = fmtDurationTs(earliestCreated, time.Now().Unix())
575+ }
576+
577+ return startedAt, endedAt, duration
578+}
579+
580+// fmtDurationTs formats the duration between two unix timestamps.
581+func fmtDurationTs(started, ended int64) string {
582+ d := time.Duration(ended-started) * time.Second
583+ if d < 0 {
584+ return "—"
585+ }
586+ if d >= time.Minute {
587+ return fmt.Sprintf("%dm%ds", d/time.Minute, d%time.Minute/time.Second)
588+ }
589+ secs := float64(d) / float64(time.Second)
590+ if secs >= 10 {
591+ return fmt.Sprintf("%ds", int(secs))
592+ }
593+ return fmt.Sprintf("%.1fs", secs)
594+}
595+
596+// filterCISessions returns only sessions with ci. prefix.
597+func filterCISessions(sessions []SessionInfo) []SessionInfo {
598+ var filtered []SessionInfo
599+ for _, s := range sessions {
600+ if strings.HasPrefix(s.Name, "ci.") {
601+ filtered = append(filtered, s)
602+ }
603+ }
604+ return filtered
605+}
606+
607+// groupSessionsByJob groups sessions by their job prefix (ci.<name>.<jobID>.).
608+func groupSessionsByJob(sessions []SessionInfo) map[string][]SessionInfo {
609+ groups := make(map[string][]SessionInfo)
610+ for _, s := range sessions {
611+ prefix := extractJobPrefix(s.Name)
612+ if prefix == "" {
613+ continue
614+ }
615+ // Set the Short name
616+ s.Short = strings.TrimPrefix(s.Name, prefix)
617+ groups[prefix] = append(groups[prefix], s)
618+ }
619+ return groups
620+}
621+
622+// parseJobPrefix extracts name and jobID from a prefix like "ci.<name>.<jobID>.".
623+func parseJobPrefix(prefix string) (name, jobID string) {
624+ // ci.name.jobID. -> ["ci", "name", "jobID", ""]
625+ parts := strings.Split(prefix, ".")
626+ if len(parts) < 4 {
627+ return "", ""
628+ }
629+ return parts[1], parts[2]
630+}
631+
632+// resolveJobExitCode determines the job's exit code from its sessions.
633+// Defensive: if any child session failed, the job failed regardless of the
634+// runner's exit code. This protects against bad pico.sh scripts that exit 0
635+// without waiting for children.
636+func resolveJobExitCode(sessions []SessionInfo) (int, string) {
637+ var runnerExit *int
638+ var worstChild *int // highest non-zero child exit code
639+
640+ for _, s := range sessions {
641+ if s.ExitCode == "" {
642+ continue
643+ }
644+ var code int
645+ if _, err := fmt.Sscanf(s.ExitCode, "%d", &code); err != nil {
646+ continue
647+ }
648+
649+ if strings.HasSuffix(s.Name, ".runner") {
650+ runnerExit = &code
651+ } else if code != 0 {
652+ if worstChild == nil || code > *worstChild {
653+ worstChild = &code
654+ }
655+ }
656+ }
657+
658+ // Any child failure overrides the runner — defensive against bad scripts
659+ if worstChild != nil {
660+ return *worstChild, "failed"
661+ }
662+ if runnerExit != nil && *runnerExit != 0 {
663+ return *runnerExit, "failed"
664+ }
665+ return 0, "success"
666+}
667+
668+func generateJobID(name, workspace string) string {
669+ return jobIDFor(name, workspace, time.Now().UnixNano())
670+}
671+
672+func jobIDFor(name, workspace string, ts int64) string {
673+ h := sha256.Sum256([]byte(name + workspace + fmt.Sprintf("%d", ts)))
674+ return fmt.Sprintf("%x", h[:4])
675+}
676+
677+func shortSessionName(session, prefix string) string {
678+ return strings.TrimPrefix(session, prefix)
679+}
680+
681+func parseZMXList(output string) []SessionInfo {
682+ var sessions []SessionInfo
683+ lines := strings.Split(strings.TrimSpace(output), "\n")
684+ for _, line := range lines {
685+ line = strings.TrimSpace(line)
686+ if line == "" {
687+ continue
688+ }
689+ // Strip leading arrow/space prefix
690+ line = strings.TrimPrefix(line, "→ ")
691+ line = strings.TrimSpace(line)
692+
693+ fields := strings.FieldsFunc(line, func(r rune) bool {
694+ return r == '\t'
695+ })
696+
697+ var si SessionInfo
698+ for _, field := range fields {
699+ parts := strings.SplitN(field, "=", 2)
700+ if len(parts) != 2 {
701+ continue
702+ }
703+ switch parts[0] {
704+ case "name":
705+ si.Name = parts[1]
706+ case "pid":
707+ si.PID = parts[1]
708+ case "clients":
709+ si.Clients = parts[1]
710+ case "created":
711+ si.Created = parts[1]
712+ case "start_dir":
713+ si.StartDir = parts[1]
714+ case "ended":
715+ si.Ended = parts[1]
716+ case "exit_code":
717+ si.ExitCode = parts[1]
718+ }
719+ }
720+ if si.Name != "" {
721+ sessions = append(sessions, si)
722+ }
723+ }
724+ return sessions
725+}
726+
727+func fetchHistoryHTML(sessionName string) (string, error) {
728+ cmd := exec.Command("zmx", "history", sessionName, "--html")
729+ output, err := cmd.Output()
730+ if err != nil {
731+ return "", err
732+ }
733+ return string(output), nil
734+}
735+
736+func fetchHistoryPlain(sessionName string) (string, error) {
737+ cmd := exec.Command("zmx", "history", sessionName, "--plain")
738+ output, err := cmd.Output()
739+ if err != nil {
740+ return "", err
741+ }
742+ return string(output), nil
743+}
744+
745+func publishStatus(w io.Writer, payload StatusPayload) error {
746+ data, err := json.Marshal(payload)
747+ if err != nil {
748+ return err
749+ }
750+ _, err = w.Write(append(data, '\n'))
751+ return err
752+}
753+
754+func stageArtifact(dir, name, jobID, short, content, ext string) error {
755+ path := filepath.Join(dir, name, jobID, short+ext)
756+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
757+ return err
758+ }
759+ return os.WriteFile(path, []byte(content), 0644)
760+}
761+
762+func syncArtifacts(cfg *Cfg, log *slog.Logger) error {
763+ entries, err := os.ReadDir(cfg.ArtifactDir)
764+ if err != nil {
765+ return fmt.Errorf("read artifact dir: %w", err)
766+ }
767+
768+ for _, entry := range entries {
769+ if !entry.IsDir() {
770+ continue
771+ }
772+ repoEntries, err := os.ReadDir(filepath.Join(cfg.ArtifactDir, entry.Name()))
773+ if err != nil {
774+ log.Error("read repo dir", "repo", entry.Name(), "err", err)
775+ continue
776+ }
777+ for _, repoEntry := range repoEntries {
778+ if !repoEntry.IsDir() {
779+ continue
780+ }
781+ eventPath := filepath.Join(cfg.ArtifactDir, entry.Name(), repoEntry.Name(), "event.json")
782+ data, err := os.ReadFile(eventPath)
783+ if err != nil {
784+ continue // no event yet, skip
785+ }
786+ var event Event
787+ if err := json.Unmarshal(data, &event); err != nil || event.ArtifactDest == "" {
788+ continue
789+ }
790+ log.Debug("syncing artifacts", "repo", entry.Name(), "job_id", repoEntry.Name(), "dest", event.ArtifactDest)
791+ sshArgs := fmt.Sprintf(
792+ "-i %s -o IdentitiesOnly=yes -o CertificateFile %s",
793+ cfg.KeyLocation,
794+ cfg.CertificateLocation,
795+ )
796+ srcDir := filepath.Join(cfg.ArtifactDir, entry.Name(), repoEntry.Name())
797+ cmd := exec.Command("rsync", "-e", sshArgs, "-rv", srcDir+"/", event.ArtifactDest)
798+ if err := runCmd(cmd, log); err != nil {
799+ log.Error("sync artifacts", "repo", entry.Name(), "job_id", repoEntry.Name(), "err", err)
800+ }
801+ }
802+ }
803+ return nil
804+}
805+
806+func filterSessions(sessions []SessionInfo, prefix string) []SessionInfo {
807+ var filtered []SessionInfo
808+ for _, s := range sessions {
809+ if strings.HasPrefix(s.Name, prefix) {
810+ cs := s
811+ cs.Short = shortSessionName(s.Name, prefix)
812+ filtered = append(filtered, cs)
813+ }
814+ }
815+ return filtered
816+}
817+
818+func allCompleted(sessions []SessionInfo) bool {
819+ for _, s := range sessions {
820+ if s.Ended == "" {
821+ return false
822+ }
823+ }
824+ return true
825+}
826+
827+func runCmd(cmd *exec.Cmd, log *slog.Logger) error {
828+ stdout, err := cmd.StdoutPipe()
829+ if err != nil {
830+ return err
831+ }
832+
833+ stderr, err := cmd.StderrPipe()
834+ if err != nil {
835+ return err
836+ }
837+
838+ if err := cmd.Start(); err != nil {
839+ return err
840+ }
841+
842+ go func() {
843+ scanner := bufio.NewScanner(stdout)
844+ for scanner.Scan() {
845+ log.Debug("cmd stdout", "text", scanner.Text())
846+ }
847+ }()
848+
849+ go func() {
850+ scanner := bufio.NewScanner(stderr)
851+ for scanner.Scan() {
852+ log.Error("cmd stderr", "text", scanner.Text())
853+ }
854+ }()
855+
856+ return cmd.Wait()
857+}
858+
859+// runCancel reads an event from stdin and cancels any running job with matching name+type.
860+func runCancel(cfg *Cfg) error {
861+ log := cfg.Logger.With("cmd", "cancel")
862+
863+ // Read event from stdin
864+ scanner := bufio.NewScanner(os.Stdin)
865+ if !scanner.Scan() {
866+ return fmt.Errorf("no input on stdin")
867+ }
868+
869+ var event Event
870+ if err := json.Unmarshal([]byte(scanner.Text()), &event); err != nil {
871+ return fmt.Errorf("unmarshal event: %w", err)
872+ }
873+
874+ log = log.With("repo", event.Name, "type", event.Type)
875+ log.Info("cancelling running jobs for repo")
876+
877+ cancelRunningJobs(cfg, log, event.Name)
878+ return nil
879+}
880+
881+// cancelRunningJobs finds and cancels all running jobs for a given repo name.
882+// It kills the runner sessions (which cascades to children). The monitor will
883+// detect the ended sessions and publish cancelled status on its next tick.
884+func cancelRunningJobs(cfg *Cfg, log *slog.Logger, name string) {
885+ runnerSessions, _ := findRunningJobs(name)
886+ if len(runnerSessions) == 0 {
887+ log.Debug("no running jobs to cancel")
888+ return
889+ }
890+
891+ log.Debug("found running jobs to cancel", "count", len(runnerSessions))
892+
893+ for _, runnerName := range runnerSessions {
894+ jobID := extractJobID(runnerName)
895+ log := log.With("job_id", jobID)
896+
897+ log.Debug("cancelling job", "runner", runnerName)
898+ if err := killSessions([]string{runnerName}); err != nil {
899+ log.Error("kill runner session", "err", err)
900+ continue
901+ }
902+ log.Debug("cancelled runner session")
903+ }
904+}
905+
906+// findRunningJobs finds all active runner sessions for a given name.
907+// Returns runner session names and all sessions for reference.
908+func findRunningJobs(name string) ([]string, []SessionInfo) {
909+ listOutput, err := exec.Command("zmx", "list").CombinedOutput()
910+ if err != nil {
911+ return nil, nil
912+ }
913+
914+ sessions := parseZMXList(string(listOutput))
915+ var runners []string
916+ for _, s := range sessions {
917+ // Match ci.<name>.*.runner sessions that are still active (no ended)
918+ if strings.HasPrefix(s.Name, "ci."+name+".") && strings.HasSuffix(s.Name, ".runner") && s.Ended == "" {
919+ runners = append(runners, s.Name)
920+ }
921+ }
922+ return runners, sessions
923+}
924+
925+// extractJobID extracts the jobID from a runner session name like ci.<name>.<jobID>.runner.
926+func extractJobID(runnerName string) string {
927+ // ci.<name>.<jobID>.runner
928+ // Remove "ci." prefix and ".runner" suffix, then take the part after the first "."
929+ name := strings.TrimSuffix(runnerName, ".runner")
930+ name = strings.TrimPrefix(name, "ci.")
931+ // name is now <name>.<jobID>, take the jobID part
932+ parts := strings.SplitN(name, ".", 2)
933+ if len(parts) == 2 {
934+ return parts[1]
935+ }
936+ return ""
937+}
938+
939+// killSessions kills zmx sessions by name.
940+func killSessions(names []string) error {
941+ if len(names) == 0 {
942+ return nil
943+ }
944+ args := append([]string{"kill"}, names...)
945+ cmd := exec.Command("zmx", args...)
946+ output, err := cmd.CombinedOutput()
947+ if err != nil {
948+ return fmt.Errorf("zmx kill: %s: %w", string(output), err)
949+ }
950+ return nil
951+}
952+
953+// runGC deletes completed CI zmx sessions that are not part of a running job.
954+func runGC(cfg *Cfg) error {
955+ log := cfg.Logger.With("cmd", "gc")
956+ log.Debug("running garbage collection")
957+
958+ listOutput, err := exec.Command("zmx", "list").CombinedOutput()
959+ if err != nil {
960+ return fmt.Errorf("zmx list: %w", err)
961+ }
962+
963+ sessions := parseZMXList(string(listOutput))
964+
965+ // Group ci. sessions by job prefix: ci.<name>.<jobID>.
966+ groups := make(map[string][]SessionInfo)
967+ for _, s := range sessions {
968+ if !strings.HasPrefix(s.Name, "ci.") {
969+ continue
970+ }
971+ // Extract job prefix: ci.<name>.<jobID>.
972+ prefix := extractJobPrefix(s.Name)
973+ if prefix == "" {
974+ continue
975+ }
976+ groups[prefix] = append(groups[prefix], s)
977+ }
978+
979+ // For each group, if all sessions are completed, kill them.
980+ var toKill []string
981+ for prefix, group := range groups {
982+ allDone := true
983+ for _, s := range group {
984+ if s.Ended == "" {
985+ allDone = false
986+ break
987+ }
988+ }
989+ if allDone {
990+ log.Debug("completed job, scheduling for gc", "prefix", prefix, "sessions", len(group))
991+ toKill = append(toKill, group[0].Name)
992+ }
993+ }
994+
995+ if len(toKill) == 0 {
996+ log.Debug("no sessions to garbage collect")
997+ return nil
998+ }
999+
1000+ if err := killSessions(toKill); err != nil {
1001+ return fmt.Errorf("kill sessions: %w", err)
1002+ }
1003+
1004+ log.Debug("garbage collection complete", "killed", len(toKill))
1005+ return nil
1006+}
1007+
1008+// extractJobPrefix extracts the job prefix from a session name.
1009+// ci.<name>.<jobID>.<step> -> ci.<name>.<jobID>.
1010+func extractJobPrefix(sessionName string) string {
1011+ // Find the third dot (after ci.<name>.<jobID>)
1012+ // ci.name.jobID.step -> ci.name.jobID.
1013+ parts := strings.Split(sessionName, ".")
1014+ if len(parts) < 4 {
1015+ return ""
1016+ }
1017+ return parts[0] + "." + parts[1] + "." + parts[2] + "."
1018+}
1019+
1020+// sessionRow holds display info for a single session in the job index.
1021+type sessionRow struct {
1022+ Name string
1023+ Short string
1024+ Status string
1025+ ExitCode string
1026+ Ended string
1027+ Duration string
1028+}
1029+
1030+// generateJobIndex produces HTML and plain-text index pages listing all
1031+// sessions for a job with links and metadata (status, exit code, ended at).
1032+func generateJobIndex(name, jobID string, sessions []SessionInfo) (htmlContent, txtContent string) {
1033+ rows := make([]sessionRow, 0, len(sessions))
1034+ for _, s := range sessions {
1035+ row := sessionRow{
1036+ Name: s.Name,
1037+ Short: s.Short,
1038+ }
1039+ if s.Ended == "" {
1040+ row.Status = "running"
1041+ row.Duration = fmtDuration(s.Created, fmt.Sprintf("%d", time.Now().Unix()))
1042+ } else if s.ExitCode == "0" {
1043+ row.Status = "success"
1044+ row.ExitCode = "0"
1045+ } else {
1046+ row.Status = "failed"
1047+ row.ExitCode = s.ExitCode
1048+ }
1049+ row.Ended = s.Ended
1050+ row.Duration = fmtDuration(s.Created, s.Ended)
1051+ rows = append(rows, row)
1052+ }
1053+
1054+ // Resolve overall job status
1055+ jobStatus := "success"
1056+ hasRunning := false
1057+ for _, r := range rows {
1058+ if r.Status == "running" {
1059+ hasRunning = true
1060+ break
1061+ }
1062+ if r.Status == "failed" {
1063+ jobStatus = "failed"
1064+ }
1065+ }
1066+ if hasRunning {
1067+ jobStatus = "running"
1068+ }
1069+
1070+ // HTML index
1071+ tmpl, err := template.New("index").Parse(indexHTMLTemplate)
1072+ if err == nil {
1073+ var buf strings.Builder
1074+ if err := tmpl.Execute(&buf, struct {
1075+ Name string
1076+ JobID string
1077+ JobStatus string
1078+ Rows []sessionRow
1079+ }{Name: name, JobID: jobID, JobStatus: jobStatus, Rows: rows}); err == nil {
1080+ htmlContent = buf.String()
1081+ }
1082+ }
1083+
1084+ // Plain-text index
1085+ var buf strings.Builder
1086+ statusIcon := map[string]string{"success": "\u2705", "failed": "\u274c", "running": "\u23f3"}
1087+ icon := statusIcon[jobStatus]
1088+ fmt.Fprintf(&buf, "Job: %s (%s) %s\n", name, jobID, icon)
1089+ fmt.Fprintf(&buf, "Sessions: %d\n", len(rows))
1090+ fmt.Fprintln(&buf, strings.Repeat("-", 70))
1091+ for _, r := range rows {
1092+ rIcon := statusIcon[r.Status]
1093+ fmt.Fprintf(&buf, " %s %-20s exit: %-5s duration: %-8s ended: %s\n",
1094+ rIcon, r.Short, r.ExitCode, r.Duration, r.Ended)
1095+ }
1096+ fmt.Fprintln(&buf, strings.Repeat("-", 60))
1097+ txtContent = buf.String()
1098+
1099+ return htmlContent, txtContent
1100+}
1101+
1102+const indexHTMLTemplate = `<!DOCTYPE html>
1103+<html lang="en">
1104+<head>
1105+<meta charset="UTF-8">
1106+<meta name="viewport" content="width=device-width, initial-scale=1.0">
1107+<title>CI Job: {{.Name}} ({{.JobID}})</title>
1108+<style>
1109+ body { font-family: system-ui, -apple-system, sans-serif; margin: 2rem; background: #fafafa; color: #222; }
1110+ h1 { font-size: 1.4rem; margin-bottom: 0.25rem; display: flex; align-items: center; gap: 0.75rem; }
1111+ .meta { color: #666; margin-bottom: 1.5rem; font-size: 0.9rem; }
1112+ table { border-collapse: collapse; width: 100%; max-width: 800px; }
1113+ th { text-align: left; padding: 0.5rem 0.75rem; border-bottom: 2px solid #ccc; font-size: 0.8rem; text-transform: uppercase; color: #555; }
1114+ td { padding: 0.5rem 0.75rem; border-bottom: 1px solid #eee; }
1115+ a { color: #0066cc; text-decoration: none; }
1116+ a:hover { text-decoration: underline; }
1117+ .pill { display: inline-block; padding: 0.2rem 0.65rem; border-radius: 999px; font-size: 0.75rem; font-weight: 700; text-transform: uppercase; letter-spacing: 0.03em; }
1118+ .pill-success { background: #d4edda; color: #155724; }
1119+ .pill-failed { background: #f8d7da; color: #721c24; }
1120+ .pill-running { background: #cce5ff; color: #004085; }
1121+</style>
1122+</head>
1123+<body>
1124+<h1>{{.Name}} <span class="pill pill-{{.JobStatus}}">{{.JobStatus}}</span></h1>
1125+<p class="meta">Job ID: {{.JobID}} &middot; {{len .Rows}} session(s)</p>
1126+<table>
1127+<thead>
1128+<tr><th>Session</th><th>Status</th><th>Exit Code</th><th>Duration</th><th>Ended</th><th>Artifacts</th></tr>
1129+</thead>
1130+<tbody>
1131+{{range .Rows}}
1132+<tr>
1133+ <td><strong>{{.Short}}</strong></td>
1134+ <td><span class="pill pill-{{.Status}}">{{.Status}}</span></td>
1135+ <td>{{.ExitCode}}</td>
1136+ <td>{{.Duration}}</td>
1137+ <td>{{.Ended}}</td>
1138+ <td><a href="{{.Short}}.html">html</a> / <a href="{{.Short}}.txt">txt</a></td>
1139+</tr>
1140+{{end}}
1141+</tbody>
1142+</table>
1143+</body>
1144+</html>`
1145+
1146+// fmtDuration formats the duration between two unix timestamp strings.
1147+// Returns human-readable strings like "2m34s", "1.5s", or "—" if invalid.
1148+func fmtDuration(created, ended string) string {
1149+ if created == "" || ended == "" {
1150+ return "—"
1151+ }
1152+ var c, e int64
1153+ if _, err := fmt.Sscanf(created, "%d", &c); err != nil {
1154+ return "—"
1155+ }
1156+ if _, err := fmt.Sscanf(ended, "%d", &e); err != nil {
1157+ return "—"
1158+ }
1159+ duration := time.Duration(e-c) * time.Second
1160+ if duration < 0 {
1161+ return "—"
1162+ }
1163+ if duration >= time.Minute {
1164+ return fmt.Sprintf("%dm%ds", duration/time.Minute, duration%time.Minute/time.Second)
1165+ }
1166+ secs := float64(duration) / float64(time.Second)
1167+ if secs >= 10 {
1168+ return fmt.Sprintf("%ds", int(secs))
1169+ }
1170+ return fmt.Sprintf("%.1fs", secs)
1171+}