Commit 818edb5

Eric Bower  ·  2026-08-15 09:39:08 -0400 EDT
parent 51628ad
fix: report proper error exit codes when job fails
2 files changed,  +186, -4
+41, -4
......@@ -7,6 +7,7 @@ import (
77 "crypto/sha256"
88 "embed"
99 "encoding/json"
10+ "errors"
1011 "flag"
1112 "fmt"
1213 "html/template"
......@@ -76,6 +77,16 @@ type Cfg struct {
7677 SessionPrefix string // session prefix filter for monitor (default: "ci.")
7778 }
7879
80+// JobFailedError indicates that one or more sessions within a job failed,
81+// carrying the resolved non-zero exit code.
82+type JobFailedError struct {
83+ ExitCode int
84+}
85+
86+func (e *JobFailedError) Error() string {
87+ return fmt.Sprintf("job failed with exit code %d", e.ExitCode)
88+}
89+
7990 type Event struct {
8091 Type string `json:"type"`
8192 Name string `json:"name"`
......@@ -321,6 +332,11 @@ func main() {
321332 case "runner":
322333 cfg.Logger.Debug("starting runner")
323334 if err := RunRunner(cfg); err != nil {
335+ var failedErr *JobFailedError
336+ if errors.As(err, &failedErr) {
337+ cfg.Logger.Error("runner failed", "err", err, "exit_code", failedErr.ExitCode)
338+ os.Exit(failedErr.ExitCode)
339+ }
324340 cfg.Logger.Error("runner failed", "err", err)
325341 os.Exit(1)
326342 }
......@@ -354,12 +370,22 @@ func main() {
354370 dest = flag.Arg(0)
355371 }
356372 if err := runLocal(cfg, dest); err != nil {
373+ var failedErr *JobFailedError
374+ if errors.As(err, &failedErr) {
375+ cfg.Logger.Error("local run failed", "err", err, "exit_code", failedErr.ExitCode)
376+ os.Exit(failedErr.ExitCode)
377+ }
357378 cfg.Logger.Error("local run failed", "err", err)
358379 os.Exit(1)
359380 }
360381 default:
361382 dest := cmd
362383 if err := runLocal(cfg, dest); err != nil {
384+ var failedErr *JobFailedError
385+ if errors.As(err, &failedErr) {
386+ cfg.Logger.Error("local run failed", "err", err, "exit_code", failedErr.ExitCode)
387+ os.Exit(failedErr.ExitCode)
388+ }
363389 cfg.Logger.Error("local run failed", "err", err)
364390 os.Exit(1)
365391 }
......@@ -822,9 +848,11 @@ func eventHandler(cfg *Cfg, eventData *Event) error {
822848 domain := getDomain(eventData.Type)
823849 fmt.Fprintf(os.Stdout, "🚀 starting job %s.%s.%s\n", domain, eventData.Name, jobID) //nolint:errcheck
824850 fmt.Fprintf(os.Stdout, " event: type=%s name=%s workspace=%s\n", eventData.Type, eventData.Name, eventData.Workspace) //nolint:errcheck
825- fmt.Fprintf(os.Stdout, " %s\n", string(eventBytes)) //nolint:errcheck
826-
827- wk := cfg.NewWorkspace(cfg, log, eventData.Workspace)
851+ newWk := cfg.NewWorkspace
852+ if newWk == nil {
853+ newWk = defaultWorkspaceFactory
854+ }
855+ wk := newWk(cfg, log, eventData.Workspace)
828856 eng := &JobEngine{
829857 Logger: log,
830858 Cfg: cfg,
......@@ -1112,6 +1140,10 @@ func waitAndReport(cfg *Cfg, log *slog.Logger, name, jobID, eventType string) er
11121140 fmt.Fprintf(os.Stdout, "%s job failed: exit %d (%s)\n", icon, exitCode, duration) //nolint:errcheck
11131141 }
11141142
1143+ if exitCode != 0 {
1144+ return &JobFailedError{ExitCode: exitCode}
1145+ }
1146+
11151147 return nil
11161148 }
11171149
......@@ -2454,8 +2486,9 @@ func runLocal(cfg *Cfg, dest string) error {
24542486 }
24552487
24562488 // Wait for completion & print live progress
2489+ var waitErr error
24572490 if err := waitAndReport(cfg, log, eventData.Name, jobID, eventData.Type); err != nil {
2458- return fmt.Errorf("wait: %w", err)
2491+ waitErr = err
24592492 }
24602493
24612494 // Generate and stage full HTML/txt artifacts and index
......@@ -2491,6 +2524,10 @@ func runLocal(cfg *Cfg, dest string) error {
24912524 _ = writePublishedSentinel(cfg.ArtifactDir, eventData.Name, jobID, status, exitCode)
24922525 }
24932526
2527+ if waitErr != nil {
2528+ return waitErr
2529+ }
2530+
24942531 return nil
24952532 }
24962533
+145, -0
......@@ -5,6 +5,7 @@ import (
55 "bytes"
66 "context"
77 "encoding/json"
8+ "errors"
89 "fmt"
910 "io"
1011 "log/slog"
......@@ -1081,3 +1082,147 @@ func TestGenerateJobIndex_WalksSubdirectories(t *testing.T) {
10811082 t.Errorf("txt index missing nested artifact build/output.json: %s", txtContent)
10821083 }
10831084 }
1085+
1086+func TestWaitAndReport_ExitCodeFailure(t *testing.T) {
1087+ if _, err := exec.LookPath("zmx"); err != nil {
1088+ t.Skip("zmx not found, skipping test")
1089+ }
1090+
1091+ cfg := &Cfg{
1092+ MonitorInterval: 100 * time.Millisecond,
1093+ ArtifactDir: t.TempDir(),
1094+ }
1095+
1096+ jobID := fmt.Sprintf("failtest-%d", time.Now().UnixNano())
1097+ prefix := "local.testrepo." + jobID + "."
1098+ runnerSession := prefix + "runner"
1099+
1100+ t.Cleanup(func() {
1101+ _ = exec.Command("zmx", "kill", "-f", runnerSession).Run()
1102+ })
1103+
1104+ // Start a zmx session that exits with code 42
1105+ cmd := exec.Command("zmx", "run", runnerSession, "-d", "bash", "-c", "exit 42")
1106+ if err := cmd.Run(); err != nil {
1107+ t.Fatalf("failed to start zmx session: %v", err)
1108+ }
1109+
1110+ err := waitAndReport(cfg, nil, "testrepo", jobID, "local")
1111+ if err == nil {
1112+ t.Fatal("expected error from waitAndReport for failed job, got nil")
1113+ }
1114+
1115+ var jobFailedErr *JobFailedError
1116+ if !errors.As(err, &jobFailedErr) {
1117+ t.Fatalf("expected error of type *JobFailedError, got %T: %v", err, err)
1118+ }
1119+ if jobFailedErr.ExitCode != 42 {
1120+ t.Errorf("expected exit code 42, got %d", jobFailedErr.ExitCode)
1121+ }
1122+}
1123+
1124+func TestWaitAndReport_Success(t *testing.T) {
1125+ if _, err := exec.LookPath("zmx"); err != nil {
1126+ t.Skip("zmx not found, skipping test")
1127+ }
1128+
1129+ cfg := &Cfg{
1130+ MonitorInterval: 100 * time.Millisecond,
1131+ ArtifactDir: t.TempDir(),
1132+ }
1133+
1134+ jobID := fmt.Sprintf("successtest-%d", time.Now().UnixNano())
1135+ prefix := "local.testrepo." + jobID + "."
1136+ runnerSession := prefix + "runner"
1137+
1138+ t.Cleanup(func() {
1139+ _ = exec.Command("zmx", "kill", "-f", runnerSession).Run()
1140+ })
1141+
1142+ // Start a zmx session that exits with code 0
1143+ cmd := exec.Command("zmx", "run", runnerSession, "-d", "bash", "-c", "exit 0")
1144+ if err := cmd.Run(); err != nil {
1145+ t.Fatalf("failed to start zmx session: %v", err)
1146+ }
1147+
1148+ err := waitAndReport(cfg, nil, "testrepo", jobID, "local")
1149+ if err != nil {
1150+ t.Fatalf("expected nil error for successful job, got %v", err)
1151+ }
1152+}
1153+
1154+func TestRunLocal_ExitCodeFailure(t *testing.T) {
1155+ if _, err := exec.LookPath("zmx"); err != nil {
1156+ t.Skip("zmx not found, skipping test")
1157+ }
1158+
1159+ tempDir := t.TempDir()
1160+ origWd, _ := os.Getwd()
1161+ defer func() { _ = os.Chdir(origWd) }()
1162+ _ = os.Chdir(tempDir)
1163+
1164+ picoContent := `#!/usr/bin/env bash
1165+exit 7
1166+`
1167+ if err := os.WriteFile("pico.sh", []byte(picoContent), 0755); err != nil {
1168+ t.Fatal(err)
1169+ }
1170+
1171+ artifactDir := t.TempDir()
1172+ cfg := &Cfg{
1173+ ArtifactDir: artifactDir,
1174+ MonitorInterval: 100 * time.Millisecond,
1175+ }
1176+
1177+ err := runLocal(cfg, "")
1178+ if err == nil {
1179+ t.Fatal("expected error from runLocal when pico.sh exits non-zero, got nil")
1180+ }
1181+
1182+ var jobFailedErr *JobFailedError
1183+ if !errors.As(err, &jobFailedErr) {
1184+ t.Fatalf("expected error of type *JobFailedError, got %T: %v", err, err)
1185+ }
1186+ if jobFailedErr.ExitCode != 7 {
1187+ t.Errorf("expected exit code 7, got %d", jobFailedErr.ExitCode)
1188+ }
1189+}
1190+
1191+func TestRunRunner_Wait_ExitCodeFailure(t *testing.T) {
1192+ if _, err := exec.LookPath("zmx"); err != nil {
1193+ t.Skip("zmx not found, skipping test")
1194+ }
1195+
1196+ workspaceDir := t.TempDir()
1197+ picoContent := `#!/usr/bin/env bash
1198+exit 13
1199+`
1200+ if err := os.WriteFile(filepath.Join(workspaceDir, "pico.sh"), []byte(picoContent), 0755); err != nil {
1201+ t.Fatal(err)
1202+ }
1203+
1204+ eventJSON := fmt.Sprintf(`{"type":"local","name":"runner-fail-repo","workspace":%q}`, workspaceDir)
1205+
1206+ artifactDir := t.TempDir()
1207+ cfg := &Cfg{
1208+ ArtifactDir: artifactDir,
1209+ MonitorInterval: 100 * time.Millisecond,
1210+ Wait: true,
1211+ Event: eventJSON,
1212+ Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
1213+ NewWorkspace: defaultWorkspaceFactory,
1214+ }
1215+
1216+ err := RunRunner(cfg)
1217+ if err == nil {
1218+ t.Fatal("expected error from RunRunner --wait on failed job, got nil")
1219+ }
1220+
1221+ var jobFailedErr *JobFailedError
1222+ if !errors.As(err, &jobFailedErr) {
1223+ t.Fatalf("expected wrapped *JobFailedError, got %T: %v", err, err)
1224+ }
1225+ if jobFailedErr.ExitCode != 13 {
1226+ t.Errorf("expected exit code 13, got %d", jobFailedErr.ExitCode)
1227+ }
1228+}