Commit fe39804
Eric Bower
·
2026-08-11 11:10:59 -0400 EDT
parent 6702fca
fix: more sync issues with the `pici` local runner
2 files changed,
+143,
-19
M
main.go
M
main.go
+67,
-15
1@@ -448,9 +448,9 @@ func (w *WorkspaceRsync) Setup() error {
2 w.Cfg.KeyLocation,
3 w.Cfg.CertificateLocation,
4 )
5- cmd = exec.Command("rsync", "-e", sshcmd, "-rv", "--exclude=/.git", "--exclude=/.jj", w.Source+"/", w.Dest+"/")
6+ cmd = exec.Command("rsync", "-e", sshcmd, "-rv", "--exclude=/.git", "--exclude=/.jj", "--filter=:- .gitignore", w.Source+"/", w.Dest+"/")
7 } else {
8- cmd = exec.Command("rsync", "-rv", "--exclude=/.git", "--exclude=/.jj", w.Source+"/", w.Dest+"/")
9+ cmd = exec.Command("rsync", "-rv", "--exclude=/.git", "--exclude=/.jj", "--filter=:- .gitignore", w.Source+"/", w.Dest+"/")
10 }
11 if err := runCmd(cmd, log); err != nil {
12 return err
13@@ -990,8 +990,11 @@ func waitAndReport(cfg *Cfg, log *slog.Logger, name, jobID, eventType string) er
14 }
15 liveLines = lines
16
17- // Check if all sessions are done
18- if allCompleted(jobSessions) {
19+ // Stage local artifacts on tick so progress and HTML reports update live
20+ _ = stageLocalArtifacts(cfg, log, name, jobID, eventType)
21+
22+ // Check if all sessions are done and the runner session has finished
23+ if isJobComplete(jobSessions) {
24 break
25 }
26 }
27@@ -1308,7 +1311,7 @@ func monitorTick(cfg *Cfg, log *slog.Logger, output io.Writer, jobStates map[str
28 // Compute job-level timing from session timestamps
29 startedAt, endedAt, duration := computeJobTiming(group)
30
31- if allCompleted(group) {
32+ if isJobComplete(group) {
33 // Check sentinel — publish terminal status exactly once
34 sentinel := filepath.Join(cfg.ArtifactDir, name, jobID, "artifacts", "published.json")
35 log.Debug("checking completion", "all_completed", true, "sentinel", sentinel)
36@@ -1772,6 +1775,26 @@ func allCompleted(sessions []SessionInfo) bool {
37 return true
38 }
39
40+// isJobComplete returns true if all sessions for a job have completed
41+// AND a runner session exists and has completed. This ensures monitor and wait
42+// do not mark a job complete while pico.sh (the runner) is still executing and
43+// potentially spawning new tasks.
44+func isJobComplete(sessions []SessionInfo) bool {
45+ if len(sessions) == 0 {
46+ return false
47+ }
48+ hasRunner := false
49+ for _, s := range sessions {
50+ if s.Ended == "" {
51+ return false
52+ }
53+ if strings.HasSuffix(s.Name, ".runner") {
54+ hasRunner = true
55+ }
56+ }
57+ return hasRunner
58+}
59+
60 func runCmd(cmd *exec.Cmd, log *slog.Logger) error {
61 stdout, err := cmd.StdoutPipe()
62 if err != nil {
63@@ -1918,8 +1941,25 @@ func cancelJobSessions(prefix string) {
64 }
65
66 // findSessionsForGC identifies finished sessions or sessions older than 3 hours matching given prefixes.
67+// It checks whether a session's job has an active runner session (a .runner session with Ended == "");
68+// if so, finished child sessions for that job are NOT cleaned up yet to ensure no new tasks spawn
69+// in pico.sh before cleaning up tasks.
70 func findSessionsForGC(sessions []SessionInfo, prefixes []string, now time.Time) []string {
71 cutoff := now.Add(-3 * time.Hour).Unix()
72+
73+ // Track which job prefixes have ANY active (running) session.
74+ // As long as any session for a job is still running (Ended == ""),
75+ // the job is active and none of its sessions should be garbage collected.
76+ activeJobPrefixes := make(map[string]bool)
77+ for _, s := range sessions {
78+ if s.Ended == "" {
79+ prefix := extractJobPrefix(s.Name)
80+ if prefix != "" {
81+ activeJobPrefixes[prefix] = true
82+ }
83+ }
84+ }
85+
86 var toKill []string
87 for _, s := range sessions {
88 matched := false
89@@ -1933,6 +1973,22 @@ func findSessionsForGC(sessions []SessionInfo, prefixes []string, now time.Time)
90 continue
91 }
92
93+ // If the session belongs to a job with an active runner, do not kill it
94+ // unless it has expired (> 3 hours old).
95+ jobPrefix := extractJobPrefix(s.Name)
96+ if jobPrefix != "" && activeJobPrefixes[jobPrefix] {
97+ if s.Created != "" {
98+ var created int64
99+ if _, err := fmt.Sscanf(s.Created, "%d", &created); err != nil {
100+ continue
101+ }
102+ if created < cutoff {
103+ toKill = append(toKill, s.Name)
104+ }
105+ }
106+ continue
107+ }
108+
109 if s.Ended != "" {
110 toKill = append(toKill, s.Name)
111 continue
112@@ -2322,7 +2378,7 @@ func runLocal(cfg *Cfg, dest string) error {
113 }
114
115 // Generate and stage full HTML/txt artifacts and index
116- if err := stageLocalArtifacts(cfg, log, eventData.Name, jobID); err != nil {
117+ if err := stageLocalArtifacts(cfg, log, eventData.Name, jobID, eventData.Type); err != nil {
118 log.Error("stage local artifacts", "err", err)
119 }
120
121@@ -2341,22 +2397,18 @@ func runLocal(cfg *Cfg, dest string) error {
122 return nil
123 }
124
125-func stageLocalArtifacts(cfg *Cfg, log *slog.Logger, repoName, jobID string) error {
126+func stageLocalArtifacts(cfg *Cfg, log *slog.Logger, repoName, jobID, eventType string) error {
127+ domain := getDomain(eventType)
128+ prefix := fmt.Sprintf("%s.%s.%s.", domain, repoName, jobID)
129+
130 listOutput, err := exec.Command("zmx", "list").CombinedOutput()
131 if err != nil {
132 return fmt.Errorf("zmx list: %w", err)
133 }
134 sessions := parseZMXList(string(listOutput))
135- var localSessions []SessionInfo
136- for _, s := range sessions {
137- if strings.HasPrefix(s.Name, "local.") {
138- localSessions = append(localSessions, s)
139- }
140- }
141- prefix := fmt.Sprintf("local.%s.%s.", repoName, jobID)
142
143 var jobSessions []SessionInfo
144- for _, s := range localSessions {
145+ for _, s := range sessions {
146 if strings.HasPrefix(s.Name, prefix) {
147 s.Short = cleanSessionShort(s.Name, prefix, repoName, jobID)
148 jobSessions = append(jobSessions, s)
+76,
-4
1@@ -519,6 +519,60 @@ func TestAllCompleted(t *testing.T) {
2 }
3 }
4
5+func TestIsJobComplete(t *testing.T) {
6+ tests := []struct {
7+ name string
8+ sessions []SessionInfo
9+ want bool
10+ }{
11+ {
12+ name: "all completed with runner",
13+ sessions: []SessionInfo{
14+ {Name: "ci.repo.abc.runner", Ended: "100"},
15+ {Name: "ci.repo.abc.step.fmt", Ended: "90"},
16+ },
17+ want: true,
18+ },
19+ {
20+ name: "runner active, step completed",
21+ sessions: []SessionInfo{
22+ {Name: "ci.repo.abc.runner", Ended: ""},
23+ {Name: "ci.repo.abc.step.fmt", Ended: "90"},
24+ },
25+ want: false,
26+ },
27+ {
28+ name: "no runner session present",
29+ sessions: []SessionInfo{
30+ {Name: "ci.repo.abc.step.fmt", Ended: "90"},
31+ },
32+ want: false,
33+ },
34+ {
35+ name: "runner completed, step active",
36+ sessions: []SessionInfo{
37+ {Name: "ci.repo.abc.runner", Ended: "100"},
38+ {Name: "ci.repo.abc.step.fmt", Ended: ""},
39+ },
40+ want: false,
41+ },
42+ {
43+ name: "empty sessions",
44+ sessions: []SessionInfo{},
45+ want: false,
46+ },
47+ }
48+
49+ for _, tt := range tests {
50+ t.Run(tt.name, func(t *testing.T) {
51+ got := isJobComplete(tt.sessions)
52+ if got != tt.want {
53+ t.Errorf("isJobComplete() = %v, want %v", got, tt.want)
54+ }
55+ })
56+ }
57+}
58+
59 func TestParseZMXList(t *testing.T) {
60 output := `name=ci-lint pid=1064464 clients=0 created=1777519944 start_dir=/home/erock/dev/pico ended=1777519986 exit_code=0
61 name=ci-tests pid=1064472 clients=0 created=1777519944 start_dir=/home/erock/dev/pico ended=1777519958 exit_code=2
62@@ -645,17 +699,26 @@ func TestFindSessionsForGC(t *testing.T) {
63
64 sessions := []SessionInfo{
65 {Name: "ci.repo.active.runner", Created: twoHoursAgo, Ended: ""},
66+ {Name: "ci.repo.active.step.fmt", Created: twoHoursAgo, Ended: "1786289500"},
67 {Name: "ci.repo.finished.runner", Created: twoHoursAgo, Ended: "1786289600"},
68+ {Name: "ci.repo.finished.step.lint", Created: twoHoursAgo, Ended: "1786289550"},
69 {Name: "ci.repo.expired.runner", Created: fourHoursAgo, Ended: ""},
70 {Name: "other.session", Created: fourHoursAgo, Ended: ""},
71 }
72
73 toKill := findSessionsForGC(sessions, []string{"ci.", "local."}, now)
74- if len(toKill) != 2 {
75- t.Fatalf("expected 2 sessions for GC, got %d: %v", len(toKill), toKill)
76+ if len(toKill) != 3 {
77+ t.Fatalf("expected 3 sessions for GC, got %d: %v", len(toKill), toKill)
78 }
79- if toKill[0] != "ci.repo.finished.runner" || toKill[1] != "ci.repo.expired.runner" {
80- t.Errorf("unexpected sessions for GC: %v", toKill)
81+ expected := map[string]bool{
82+ "ci.repo.finished.runner": true,
83+ "ci.repo.finished.step.lint": true,
84+ "ci.repo.expired.runner": true,
85+ }
86+ for _, k := range toKill {
87+ if !expected[k] {
88+ t.Errorf("unexpected session in GC kill list: %s", k)
89+ }
90 }
91 }
92
93@@ -858,6 +921,12 @@ func TestWorkspaceRsync_ExcludesGitAndJJ(t *testing.T) {
94 if err := os.WriteFile(filepath.Join(srcDir, "hello.txt"), []byte("world"), 0644); err != nil {
95 t.Fatalf("failed to write hello.txt: %v", err)
96 }
97+ if err := os.WriteFile(filepath.Join(srcDir, ".gitignore"), []byte("ignored.txt\n"), 0644); err != nil {
98+ t.Fatalf("failed to write .gitignore: %v", err)
99+ }
100+ if err := os.WriteFile(filepath.Join(srcDir, "ignored.txt"), []byte("secret"), 0644); err != nil {
101+ t.Fatalf("failed to write ignored.txt: %v", err)
102+ }
103 if err := os.MkdirAll(filepath.Join(srcDir, ".git"), 0755); err != nil {
104 t.Fatalf("failed to mkdir .git: %v", err)
105 }
106@@ -888,6 +957,9 @@ func TestWorkspaceRsync_ExcludesGitAndJJ(t *testing.T) {
107 if _, err := os.Stat(filepath.Join(destDir, "hello.txt")); os.IsNotExist(err) {
108 t.Error("expected hello.txt to exist in dest workspace")
109 }
110+ if _, err := os.Stat(filepath.Join(destDir, "ignored.txt")); !os.IsNotExist(err) {
111+ t.Error("expected ignored.txt to be excluded by .gitignore from dest workspace")
112+ }
113 if _, err := os.Stat(filepath.Join(destDir, ".git")); !os.IsNotExist(err) {
114 t.Error("expected .git to be excluded from dest workspace")
115 }