Commit 8759922

Eric Bower  ·  2026-08-09 10:53:11 -0400 EDT
parent a9e9d52
fix: random monitoring issues
2 files changed,  +41, -18
+23, -12
......@@ -73,6 +73,7 @@ type Cfg struct {
7373 HumanOutput bool // human-readable output instead of JSONL / slog
7474 Wait bool // block until job completes, print history and summary
7575 EnvVars envList // custom environment variables passed via -e / -env
76+ SessionPrefix string // session prefix filter for monitor (default: "ci.")
7677 }
7778
7879 type Event struct {
......@@ -93,10 +94,12 @@ func NewCfg() (*Cfg, string, bool) {
9394 var gcInterval time.Duration
9495 var logLevel string
9596 var envVars envList
97+ var sessionPrefix string
9698 flag.StringVar(&keyLoc, "pk", "", "ssh private key used to authenticate with pico services")
9799 flag.StringVar(&certLoc, "ck", "", "ssh certificate public key used to authenticate with pico services (only required if using ssh certificates)")
98100 flag.StringVar(&artifactDir, "artifact-dir", "/tmp/pici-artifacts", "local directory to stage artifacts")
99101 flag.StringVar(&event, "event", "", "event JSON to run (alternative to reading from stdin)")
102+ flag.StringVar(&sessionPrefix, "prefix", "ci.", "session prefix filter for monitor (default: ci.)")
100103 flag.DurationVar(&monitorInterval, "monitor-interval", 5*time.Second, "interval for monitoring zmx sessions")
101104 flag.DurationVar(&gcInterval, "gc-interval", 10*time.Minute, "interval for garbage collection in monitor (0 to disable)")
102105 flag.StringVar(&logLevel, "log-level", "info", "log level: debug, info, warn, error")
......@@ -134,6 +137,7 @@ func NewCfg() (*Cfg, string, bool) {
134137 HumanOutput: human,
135138 Wait: wait,
136139 EnvVars: envVars,
140+ SessionPrefix: sessionPrefix,
137141 }, cmd, wantHelp
138142 }
139143
......@@ -614,8 +618,8 @@ func (eng *JobEngine) Setup() error {
614618
615619 func (eng *JobEngine) Run(manifest string) error {
616620 domain := "ci"
617- if eng.Ev != nil && eng.Ev.Type == "local" {
618- domain = "local"
621+ if eng.Ev != nil && eng.Ev.Type != "" {
622+ domain = eng.Ev.Type
619623 }
620624 prefix := fmt.Sprintf("%s.%s.%s.", domain, eng.Ev.Name, eng.JobID)
621625 // Child sessions use ".step." sub-prefix so zmx wait "*" inside pico.sh
......@@ -693,8 +697,8 @@ func eventHandler(cfg *Cfg, eventData *Event) error {
693697
694698 eventBytes, _ := json.Marshal(eventData)
695699 domain := "ci"
696- if eventData.Type == "local" {
697- domain = "local"
700+ if eventData.Type != "" {
701+ domain = eventData.Type
698702 }
699703 fmt.Fprintf(os.Stdout, "🚀 starting job %s.%s.%s\n", domain, eventData.Name, jobID) //nolint:errcheck
700704 fmt.Fprintf(os.Stdout, " event: type=%s name=%s workspace=%s\n", eventData.Type, eventData.Name, eventData.Workspace) //nolint:errcheck
......@@ -804,8 +808,8 @@ See: https://github.com/picosh/pici
804808 }
805809
806810 domain = "ci"
807- if eventData.Type == "local" {
808- domain = "local"
811+ if eventData.Type != "" {
812+ domain = eventData.Type
809813 }
810814 session := fmt.Sprintf("%s.%s.%s.runner", domain, eventData.Name, jobID)
811815 fmt.Fprintf(os.Stdout, " zmx tail %s\n", session) //nolint:errcheck
......@@ -818,8 +822,8 @@ See: https://github.com/picosh/pici
818822 // progress to stdout, then dumps session history and a final summary.
819823 func waitAndReport(cfg *Cfg, log *slog.Logger, name, jobID, eventType string) error {
820824 domain := "ci"
821- if eventType == "local" {
822- domain = "local"
825+ if eventType != "" {
826+ domain = eventType
823827 }
824828 prefix := domain + "." + name + "." + jobID + "."
825829 interval := cfg.MonitorInterval
......@@ -1180,7 +1184,7 @@ func monitorTick(cfg *Cfg, log *slog.Logger, output io.Writer, jobStates map[str
11801184 return fmt.Errorf("zmx list: %w", err)
11811185 }
11821186 sessions := parseZMXList(string(listOutput))
1183- ciSessions := filterCISessions(sessions)
1187+ ciSessions := filterCISessions(cfg, sessions)
11841188
11851189 if len(ciSessions) == 0 {
11861190 log.Debug("no ci.* sessions found")
......@@ -1420,11 +1424,18 @@ func fmtDurationTs(started, ended int64) string {
14201424 return fmt.Sprintf("%.1fs", secs)
14211425 }
14221426
1423-// filterCISessions returns only sessions with ci. prefix.
1424-func filterCISessions(sessions []SessionInfo) []SessionInfo {
1427+// filterCISessions returns only sessions matching cfg.SessionPrefix (default "ci.").
1428+func filterCISessions(cfg *Cfg, sessions []SessionInfo) []SessionInfo {
1429+ prefix := "ci."
1430+ if cfg != nil && cfg.SessionPrefix != "" {
1431+ prefix = cfg.SessionPrefix
1432+ }
1433+ if !strings.HasSuffix(prefix, ".") {
1434+ prefix = prefix + "."
1435+ }
14251436 var filtered []SessionInfo
14261437 for _, s := range sessions {
1427- if strings.HasPrefix(s.Name, "ci.") {
1438+ if strings.HasPrefix(s.Name, prefix) {
14281439 filtered = append(filtered, s)
14291440 }
14301441 }
+18, -6
......@@ -48,10 +48,10 @@ zmx run step2 echo "hello from step2"
4848 ctx, cancel := context.WithCancel(context.Background())
4949 testJobID := fmt.Sprintf("j-%d", time.Now().UnixNano()%100000000)
5050 t.Cleanup(func() {
51- _ = exec.Command("zmx", "kill", "-f", fmt.Sprintf("ci.test-repo.%s", testJobID)).Run()
51+ _ = exec.Command("zmx", "kill", "-f", fmt.Sprintf("test-ci.test-repo.%s", testJobID)).Run()
5252 })
5353 event := Event{
54- Type: "ci",
54+ Type: "test-ci",
5555 Name: "test-repo",
5656 JobID: testJobID,
5757 Workspace: workspaceDir,
......@@ -68,6 +68,7 @@ zmx run step2 echo "hello from step2"
6868 NewWorkspace: defaultWorkspaceFactory,
6969 StatusOutput: statusBuf,
7070 IncludeRunning: true,
71+ SessionPrefix: "test-ci.",
7172 }
7273
7374 // 3. Run the runner (fire-and-forget, exits quickly)
......@@ -184,8 +185,8 @@ zmx run step2 echo "hello from step2"
184185 if finalPayload == nil {
185186 t.Fatal("no final payload")
186187 }
187- // Filter to only sessions with our job prefix to avoid picking up unrelated ci.* sessions
188- expectedPrefix := fmt.Sprintf("ci.%s.%s.", finalPayload.Name, finalPayload.JobID)
188+ // Filter to only sessions with our job prefix to avoid picking up unrelated test-ci.* sessions
189+ expectedPrefix := fmt.Sprintf("test-ci.%s.%s.", finalPayload.Name, finalPayload.JobID)
189190 sessions := make([]SessionInfo, 0, len(finalPayload.Sessions))
190191 for _, s := range finalPayload.Sessions {
191192 if strings.HasPrefix(s.Name, expectedPrefix) {
......@@ -756,6 +757,13 @@ func TestDetectGitCommit(t *testing.T) {
756757 }
757758
758759 func TestWaitAndReport_Cancellation(t *testing.T) {
760+ if testing.Short() {
761+ t.Skip("skip integration test")
762+ }
763+ if _, err := exec.LookPath("zmx"); err != nil {
764+ t.Skip("zmx not found, skipping integration test")
765+ }
766+
759767 ctx, cancel := context.WithCancel(context.Background())
760768 cfg := &Cfg{
761769 Ctx: ctx,
......@@ -764,9 +772,13 @@ func TestWaitAndReport_Cancellation(t *testing.T) {
764772 }
765773
766774 jobID := fmt.Sprintf("canceltest-%d", time.Now().UnixNano())
767- prefix := "local.testrepo." + jobID + "."
775+ prefix := "test-local.testrepo." + jobID + "."
768776 runnerSession := prefix + "runner"
769777
778+ t.Cleanup(func() {
779+ _ = exec.Command("zmx", "kill", "-f", runnerSession).Run()
780+ })
781+
770782 cmd := exec.Command("zmx", "run", runnerSession, "-d", "sleep", "30")
771783 if err := cmd.Run(); err != nil {
772784 t.Fatalf("failed to start zmx session: %v", err)
......@@ -777,7 +789,7 @@ func TestWaitAndReport_Cancellation(t *testing.T) {
777789 cancel()
778790 }()
779791
780- err := waitAndReport(cfg, nil, "testrepo", jobID, "local")
792+ err := waitAndReport(cfg, nil, "testrepo", jobID, "test-local")
781793 if err == nil {
782794 t.Error("expected error when waitAndReport is cancelled, got nil")
783795 }