Commit 8759922
Eric Bower
·
2026-08-09 10:53:11 -0400 EDT
parent a9e9d52
fix: random monitoring issues
2 files changed,
+41,
-18
M
main.go
M
main.go
+23,
-12
| ... | ... | @@ -73,6 +73,7 @@ type Cfg struct { | |
| 73 | 73 | HumanOutput bool // human-readable output instead of JSONL / slog | |
| 74 | 74 | Wait bool // block until job completes, print history and summary | |
| 75 | 75 | EnvVars envList // custom environment variables passed via -e / -env | |
| 76 | + | SessionPrefix string // session prefix filter for monitor (default: "ci.") | |
| 76 | 77 | } | |
| 77 | 78 | ||
| 78 | 79 | type Event struct { |
| ... | ... | @@ -93,10 +94,12 @@ func NewCfg() (*Cfg, string, bool) { | |
| 93 | 94 | var gcInterval time.Duration | |
| 94 | 95 | var logLevel string | |
| 95 | 96 | var envVars envList | |
| 97 | + | var sessionPrefix string | |
| 96 | 98 | flag.StringVar(&keyLoc, "pk", "", "ssh private key used to authenticate with pico services") | |
| 97 | 99 | flag.StringVar(&certLoc, "ck", "", "ssh certificate public key used to authenticate with pico services (only required if using ssh certificates)") | |
| 98 | 100 | flag.StringVar(&artifactDir, "artifact-dir", "/tmp/pici-artifacts", "local directory to stage artifacts") | |
| 99 | 101 | flag.StringVar(&event, "event", "", "event JSON to run (alternative to reading from stdin)") | |
| 102 | + | flag.StringVar(&sessionPrefix, "prefix", "ci.", "session prefix filter for monitor (default: ci.)") | |
| 100 | 103 | flag.DurationVar(&monitorInterval, "monitor-interval", 5*time.Second, "interval for monitoring zmx sessions") | |
| 101 | 104 | flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute, "interval for garbage collection in monitor (0 to disable)") | |
| 102 | 105 | flag.StringVar(&logLevel, "log-level", "info", "log level: debug, info, warn, error") |
| ... | ... | @@ -134,6 +137,7 @@ func NewCfg() (*Cfg, string, bool) { | |
| 134 | 137 | HumanOutput: human, | |
| 135 | 138 | Wait: wait, | |
| 136 | 139 | EnvVars: envVars, | |
| 140 | + | SessionPrefix: sessionPrefix, | |
| 137 | 141 | }, cmd, wantHelp | |
| 138 | 142 | } | |
| 139 | 143 |
| ... | ... | @@ -614,8 +618,8 @@ func (eng *JobEngine) Setup() error { | |
| 614 | 618 | ||
| 615 | 619 | func (eng *JobEngine) Run(manifest string) error { | |
| 616 | 620 | domain := "ci" | |
| 617 | - | if eng.Ev != nil && eng.Ev.Type == "local" { | |
| 618 | - | domain = "local" | |
| 621 | + | if eng.Ev != nil && eng.Ev.Type != "" { | |
| 622 | + | domain = eng.Ev.Type | |
| 619 | 623 | } | |
| 620 | 624 | prefix := fmt.Sprintf("%s.%s.%s.", domain, eng.Ev.Name, eng.JobID) | |
| 621 | 625 | // Child sessions use ".step." sub-prefix so zmx wait "*" inside pico.sh |
| ... | ... | @@ -693,8 +697,8 @@ func eventHandler(cfg *Cfg, eventData *Event) error { | |
| 693 | 697 | ||
| 694 | 698 | eventBytes, _ := json.Marshal(eventData) | |
| 695 | 699 | domain := "ci" | |
| 696 | - | if eventData.Type == "local" { | |
| 697 | - | domain = "local" | |
| 700 | + | if eventData.Type != "" { | |
| 701 | + | domain = eventData.Type | |
| 698 | 702 | } | |
| 699 | 703 | fmt.Fprintf(os.Stdout, "🚀 starting job %s.%s.%s\n", domain, eventData.Name, jobID) //nolint:errcheck | |
| 700 | 704 | fmt.Fprintf(os.Stdout, " event: type=%s name=%s workspace=%s\n", eventData.Type, eventData.Name, eventData.Workspace) //nolint:errcheck |
| ... | ... | @@ -804,8 +808,8 @@ See: https://github.com/picosh/pici | |
| 804 | 808 | } | |
| 805 | 809 | ||
| 806 | 810 | domain = "ci" | |
| 807 | - | if eventData.Type == "local" { | |
| 808 | - | domain = "local" | |
| 811 | + | if eventData.Type != "" { | |
| 812 | + | domain = eventData.Type | |
| 809 | 813 | } | |
| 810 | 814 | session := fmt.Sprintf("%s.%s.%s.runner", domain, eventData.Name, jobID) | |
| 811 | 815 | fmt.Fprintf(os.Stdout, " zmx tail %s\n", session) //nolint:errcheck |
| ... | ... | @@ -818,8 +822,8 @@ See: https://github.com/picosh/pici | |
| 818 | 822 | // progress to stdout, then dumps session history and a final summary. | |
| 819 | 823 | func waitAndReport(cfg *Cfg, log *slog.Logger, name, jobID, eventType string) error { | |
| 820 | 824 | domain := "ci" | |
| 821 | - | if eventType == "local" { | |
| 822 | - | domain = "local" | |
| 825 | + | if eventType != "" { | |
| 826 | + | domain = eventType | |
| 823 | 827 | } | |
| 824 | 828 | prefix := domain + "." + name + "." + jobID + "." | |
| 825 | 829 | interval := cfg.MonitorInterval |
| ... | ... | @@ -1180,7 +1184,7 @@ func monitorTick(cfg *Cfg, log *slog.Logger, output io.Writer, jobStates map[str | |
| 1180 | 1184 | return fmt.Errorf("zmx list: %w", err) | |
| 1181 | 1185 | } | |
| 1182 | 1186 | sessions := parseZMXList(string(listOutput)) | |
| 1183 | - | ciSessions := filterCISessions(sessions) | |
| 1187 | + | ciSessions := filterCISessions(cfg, sessions) | |
| 1184 | 1188 | ||
| 1185 | 1189 | if len(ciSessions) == 0 { | |
| 1186 | 1190 | log.Debug("no ci.* sessions found") |
| ... | ... | @@ -1420,11 +1424,18 @@ func fmtDurationTs(started, ended int64) string { | |
| 1420 | 1424 | return fmt.Sprintf("%.1fs", secs) | |
| 1421 | 1425 | } | |
| 1422 | 1426 | ||
| 1423 | - | // filterCISessions returns only sessions with ci. prefix. | |
| 1424 | - | func filterCISessions(sessions []SessionInfo) []SessionInfo { | |
| 1427 | + | // filterCISessions returns only sessions matching cfg.SessionPrefix (default "ci."). | |
| 1428 | + | func filterCISessions(cfg *Cfg, sessions []SessionInfo) []SessionInfo { | |
| 1429 | + | prefix := "ci." | |
| 1430 | + | if cfg != nil && cfg.SessionPrefix != "" { | |
| 1431 | + | prefix = cfg.SessionPrefix | |
| 1432 | + | } | |
| 1433 | + | if !strings.HasSuffix(prefix, ".") { | |
| 1434 | + | prefix = prefix + "." | |
| 1435 | + | } | |
| 1425 | 1436 | var filtered []SessionInfo | |
| 1426 | 1437 | for _, s := range sessions { | |
| 1427 | - | if strings.HasPrefix(s.Name, "ci.") { | |
| 1438 | + | if strings.HasPrefix(s.Name, prefix) { | |
| 1428 | 1439 | filtered = append(filtered, s) | |
| 1429 | 1440 | } | |
| 1430 | 1441 | } |
+18,
-6
| ... | ... | @@ -48,10 +48,10 @@ zmx run step2 echo "hello from step2" | |
| 48 | 48 | ctx, cancel := context.WithCancel(context.Background()) | |
| 49 | 49 | testJobID := fmt.Sprintf("j-%d", time.Now().UnixNano()%100000000) | |
| 50 | 50 | t.Cleanup(func() { | |
| 51 | - | _ = exec.Command("zmx", "kill", "-f", fmt.Sprintf("ci.test-repo.%s", testJobID)).Run() | |
| 51 | + | _ = exec.Command("zmx", "kill", "-f", fmt.Sprintf("test-ci.test-repo.%s", testJobID)).Run() | |
| 52 | 52 | }) | |
| 53 | 53 | event := Event{ | |
| 54 | - | Type: "ci", | |
| 54 | + | Type: "test-ci", | |
| 55 | 55 | Name: "test-repo", | |
| 56 | 56 | JobID: testJobID, | |
| 57 | 57 | Workspace: workspaceDir, |
| ... | ... | @@ -68,6 +68,7 @@ zmx run step2 echo "hello from step2" | |
| 68 | 68 | NewWorkspace: defaultWorkspaceFactory, | |
| 69 | 69 | StatusOutput: statusBuf, | |
| 70 | 70 | IncludeRunning: true, | |
| 71 | + | SessionPrefix: "test-ci.", | |
| 71 | 72 | } | |
| 72 | 73 | ||
| 73 | 74 | // 3. Run the runner (fire-and-forget, exits quickly) |
| ... | ... | @@ -184,8 +185,8 @@ zmx run step2 echo "hello from step2" | |
| 184 | 185 | if finalPayload == nil { | |
| 185 | 186 | t.Fatal("no final payload") | |
| 186 | 187 | } | |
| 187 | - | // Filter to only sessions with our job prefix to avoid picking up unrelated ci.* sessions | |
| 188 | - | expectedPrefix := fmt.Sprintf("ci.%s.%s.", finalPayload.Name, finalPayload.JobID) | |
| 188 | + | // Filter to only sessions with our job prefix to avoid picking up unrelated test-ci.* sessions | |
| 189 | + | expectedPrefix := fmt.Sprintf("test-ci.%s.%s.", finalPayload.Name, finalPayload.JobID) | |
| 189 | 190 | sessions := make([]SessionInfo, 0, len(finalPayload.Sessions)) | |
| 190 | 191 | for _, s := range finalPayload.Sessions { | |
| 191 | 192 | if strings.HasPrefix(s.Name, expectedPrefix) { |
| ... | ... | @@ -756,6 +757,13 @@ func TestDetectGitCommit(t *testing.T) { | |
| 756 | 757 | } | |
| 757 | 758 | ||
| 758 | 759 | func TestWaitAndReport_Cancellation(t *testing.T) { | |
| 760 | + | if testing.Short() { | |
| 761 | + | t.Skip("skip integration test") | |
| 762 | + | } | |
| 763 | + | if _, err := exec.LookPath("zmx"); err != nil { | |
| 764 | + | t.Skip("zmx not found, skipping integration test") | |
| 765 | + | } | |
| 766 | + | ||
| 759 | 767 | ctx, cancel := context.WithCancel(context.Background()) | |
| 760 | 768 | cfg := &Cfg{ | |
| 761 | 769 | Ctx: ctx, |
| ... | ... | @@ -764,9 +772,13 @@ func TestWaitAndReport_Cancellation(t *testing.T) { | |
| 764 | 772 | } | |
| 765 | 773 | ||
| 766 | 774 | jobID := fmt.Sprintf("canceltest-%d", time.Now().UnixNano()) | |
| 767 | - | prefix := "local.testrepo." + jobID + "." | |
| 775 | + | prefix := "test-local.testrepo." + jobID + "." | |
| 768 | 776 | runnerSession := prefix + "runner" | |
| 769 | 777 | ||
| 778 | + | t.Cleanup(func() { | |
| 779 | + | _ = exec.Command("zmx", "kill", "-f", runnerSession).Run() | |
| 780 | + | }) | |
| 781 | + | ||
| 770 | 782 | cmd := exec.Command("zmx", "run", runnerSession, "-d", "sleep", "30") | |
| 771 | 783 | if err := cmd.Run(); err != nil { | |
| 772 | 784 | t.Fatalf("failed to start zmx session: %v", err) |
| ... | ... | @@ -777,7 +789,7 @@ func TestWaitAndReport_Cancellation(t *testing.T) { | |
| 777 | 789 | cancel() | |
| 778 | 790 | }() | |
| 779 | 791 | ||
| 780 | - | err := waitAndReport(cfg, nil, "testrepo", jobID, "local") | |
| 792 | + | err := waitAndReport(cfg, nil, "testrepo", jobID, "test-local") | |
| 781 | 793 | if err == nil { | |
| 782 | 794 | t.Error("expected error when waitAndReport is cancelled, got nil") | |
| 783 | 795 | } |