fix(gc): wait 3 hours before killing sessions

Commit `50e9dba`

Eric Bower · 2026-08-15 14:08:35 -0400 EDT

parent eb0002c

fix(gc): wait 3 hours before killing sessions

2 files changed, +29, -26

M main.go

M main_test.go

M main.go

+21, -19

......@@ -2215,16 +2215,15 @@ func cancelJobSessions(prefix string) {
	}
}

// findSessionsForGC identifies finished sessions or sessions older than 3 hours matching given prefixes.
// It checks whether a session's job has an active runner session (a .runner session with Ended == "");
// if so, finished child sessions for that job are NOT cleaned up yet to ensure no new tasks spawn
// in pico.sh before cleaning up tasks.
// findSessionsForGC identifies finished or running sessions older than the retention cutoff (3 hours) matching given prefixes.
// It checks whether a session's job has an active session; active jobs are preserved unless their sessions have expired.
// Finished sessions are kept until they are older than the cutoff so users can inspect/debug test failures.
func findSessionsForGC(sessions []SessionInfo, prefixes []string, now time.Time) []string {
	cutoff := now.Add(-3 * time.Hour).Unix()

	// Track which job prefixes have ANY active (running) session.
	// As long as any session for a job is still running (Ended == ""),
	// the job is active and none of its sessions should be garbage collected.
	// the job is active and none of its sessions should be garbage collected unless expired.
	activeJobPrefixes := make(map[string]bool)
	for _, s := range sessions {
		if s.Ended == "" {

......@@ -2248,7 +2247,7 @@ func findSessionsForGC(sessions []SessionInfo, prefixes []string, now time.Time)
			continue
		}

		// If the session belongs to a job with an active runner, do not kill it
		// If the session belongs to a job with an active session, do not kill it
		// unless it has expired (> 3 hours old).
		jobPrefix := extractJobPrefix(s.Name)
		if jobPrefix != "" && activeJobPrefixes[jobPrefix] {

......@@ -2264,22 +2263,25 @@ func findSessionsForGC(sessions []SessionInfo, prefixes []string, now time.Time)
			continue
		}

		// For completed/inactive jobs, only kill sessions if they ended or were created before the cutoff
		if s.Ended != "" {
			toKill = append(toKill, s.Name)
			continue
		}

		if s.Created == "" {
			continue
		}

		var created int64
		if _, err := fmt.Sscanf(s.Created, "%d", &created); err != nil {
			continue
			var ended int64
			if _, err := fmt.Sscanf(s.Ended, "%d", &ended); err == nil {
				if ended < cutoff {
					toKill = append(toKill, s.Name)
				}
				continue
			}
		}

		if created < cutoff {
			toKill = append(toKill, s.Name)
		if s.Created != "" {
			var created int64
			if _, err := fmt.Sscanf(s.Created, "%d", &created); err != nil {
				continue
			}
			if created < cutoff {
				toKill = append(toKill, s.Name)
			}
		}
	}
	return toKill

M main_test.go

+8, -7

......@@ -706,9 +706,11 @@ func TestFindSessionsForGC(t *testing.T) {

	sessions := []SessionInfo{
		{Name: "ci.repo.active.runner", Created: twoHoursAgo, Ended: ""},
		{Name: "ci.repo.active.step.fmt", Created: twoHoursAgo, Ended: "1786289500"},
		{Name: "ci.repo.finished.runner", Created: twoHoursAgo, Ended: "1786289600"},
		{Name: "ci.repo.finished.step.lint", Created: twoHoursAgo, Ended: "1786289550"},
		{Name: "ci.repo.active.step.fmt", Created: twoHoursAgo, Ended: twoHoursAgo},
		{Name: "ci.repo.recent.runner", Created: twoHoursAgo, Ended: twoHoursAgo},
		{Name: "ci.repo.recent.step.lint", Created: twoHoursAgo, Ended: twoHoursAgo},
		{Name: "ci.repo.oldfinished.runner", Created: fourHoursAgo, Ended: fourHoursAgo},
		{Name: "ci.repo.oldfinished.step.lint", Created: fourHoursAgo, Ended: fourHoursAgo},
		{Name: "ci.repo.expired.runner", Created: fourHoursAgo, Ended: ""},
		{Name: "other.session", Created: fourHoursAgo, Ended: ""},
	}

......@@ -718,9 +720,9 @@ func TestFindSessionsForGC(t *testing.T) {
		t.Fatalf("expected 3 sessions for GC, got %d: %v", len(toKill), toKill)
	}
	expected := map[string]bool{
		"ci.repo.finished.runner":    true,
		"ci.repo.finished.step.lint": true,
		"ci.repo.expired.runner":     true,
		"ci.repo.oldfinished.runner":    true,
		"ci.repo.oldfinished.step.lint": true,
		"ci.repo.expired.runner":        true,
	}
	for _, k := range toKill {
		if !expected[k] {

......@@ -1495,4 +1497,3 @@ func TestFindSSHAuthMethod_ExplicitKeyAndCert(t *testing.T) {
		t.Fatal("expected error when non-existent certificate is explicitly specified")
	}
}

...	...		@@ -2215,16 +2215,15 @@ func cancelJobSessions(prefix string) {
2215	2215		}
2216	2216		}
2217	2217
2218		-	// findSessionsForGC identifies finished sessions or sessions older than 3 hours matching given prefixes.
2219		-	// It checks whether a session's job has an active runner session (a .runner session with Ended == "");
2220		-	// if so, finished child sessions for that job are NOT cleaned up yet to ensure no new tasks spawn
2221		-	// in pico.sh before cleaning up tasks.
	2218	+	// findSessionsForGC identifies finished or running sessions older than the retention cutoff (3 hours) matching given prefixes.
	2219	+	// It checks whether a session's job has an active session; active jobs are preserved unless their sessions have expired.
	2220	+	// Finished sessions are kept until they are older than the cutoff so users can inspect/debug test failures.
2222	2221		func findSessionsForGC(sessions []SessionInfo, prefixes []string, now time.Time) []string {
2223	2222		cutoff := now.Add(-3 * time.Hour).Unix()
2224	2223
2225	2224		// Track which job prefixes have ANY active (running) session.
2226	2225		// As long as any session for a job is still running (Ended == ""),
2227		-	// the job is active and none of its sessions should be garbage collected.
	2226	+	// the job is active and none of its sessions should be garbage collected unless expired.
2228	2227		activeJobPrefixes := make(map[string]bool)
2229	2228		for _, s := range sessions {
2230	2229		if s.Ended == "" {

...	...		@@ -2264,22 +2263,25 @@ func findSessionsForGC(sessions []SessionInfo, prefixes []string, now time.Time)
2264	2263		continue
2265	2264		}
2266	2265
	2266	+	// For completed/inactive jobs, only kill sessions if they ended or were created before the cutoff
2267	2267		if s.Ended != "" {
2268		-	toKill = append(toKill, s.Name)
2269		-	continue
2270		-	}
2271		-
2272		-	if s.Created == "" {
2273		-	continue
2274		-	}
2275		-
2276		-	var created int64
2277		-	if _, err := fmt.Sscanf(s.Created, "%d", &created); err != nil {
2278		-	continue
	2268	+	var ended int64
	2269	+	if _, err := fmt.Sscanf(s.Ended, "%d", &ended); err == nil {
	2270	+	if ended < cutoff {
	2271	+	toKill = append(toKill, s.Name)
	2272	+	}
	2273	+	continue
	2274	+	}
2279	2275		}
2280	2276
2281		-	if created < cutoff {
2282		-	toKill = append(toKill, s.Name)
	2277	+	if s.Created != "" {
	2278	+	var created int64
	2279	+	if _, err := fmt.Sscanf(s.Created, "%d", &created); err != nil {
	2280	+	continue
	2281	+	}
	2282	+	if created < cutoff {
	2283	+	toKill = append(toKill, s.Name)
	2284	+	}
2283	2285		}
2284	2286		}
2285	2287		return toKill

...	...		@@ -706,9 +706,11 @@ func TestFindSessionsForGC(t *testing.T) {
706	706
707	707		sessions := []SessionInfo{
708	708		{Name: "ci.repo.active.runner", Created: twoHoursAgo, Ended: ""},
709		-	{Name: "ci.repo.active.step.fmt", Created: twoHoursAgo, Ended: "1786289500"},
710		-	{Name: "ci.repo.finished.runner", Created: twoHoursAgo, Ended: "1786289600"},
711		-	{Name: "ci.repo.finished.step.lint", Created: twoHoursAgo, Ended: "1786289550"},
	709	+	{Name: "ci.repo.active.step.fmt", Created: twoHoursAgo, Ended: twoHoursAgo},
	710	+	{Name: "ci.repo.recent.runner", Created: twoHoursAgo, Ended: twoHoursAgo},
	711	+	{Name: "ci.repo.recent.step.lint", Created: twoHoursAgo, Ended: twoHoursAgo},
	712	+	{Name: "ci.repo.oldfinished.runner", Created: fourHoursAgo, Ended: fourHoursAgo},
	713	+	{Name: "ci.repo.oldfinished.step.lint", Created: fourHoursAgo, Ended: fourHoursAgo},
712	714		{Name: "ci.repo.expired.runner", Created: fourHoursAgo, Ended: ""},
713	715		{Name: "other.session", Created: fourHoursAgo, Ended: ""},
714	716		}

...	...		@@ -2248,7 +2247,7 @@ func findSessionsForGC(sessions []SessionInfo, prefixes []string, now time.Time)
2248	2247		continue
2249	2248		}
2250	2249
2251		-	// If the session belongs to a job with an active runner, do not kill it
	2250	+	// If the session belongs to a job with an active session, do not kill it
2252	2251		// unless it has expired (> 3 hours old).
2253	2252		jobPrefix := extractJobPrefix(s.Name)
2254	2253		if jobPrefix != "" && activeJobPrefixes[jobPrefix] {

...	...		@@ -718,9 +720,9 @@ func TestFindSessionsForGC(t *testing.T) {
718	720		t.Fatalf("expected 3 sessions for GC, got %d: %v", len(toKill), toKill)
719	721		}
720	722		expected := map[string]bool{
721		-	"ci.repo.finished.runner": true,
722		-	"ci.repo.finished.step.lint": true,
723		-	"ci.repo.expired.runner": true,
	723	+	"ci.repo.oldfinished.runner": true,
	724	+	"ci.repo.oldfinished.step.lint": true,
	725	+	"ci.repo.expired.runner": true,
724	726		}
725	727		for _, k := range toKill {
726	728		if !expected[k] {

...	...		@@ -1495,4 +1497,3 @@ func TestFindSSHAuthMethod_ExplicitKeyAndCert(t *testing.T) {
1495	1497		t.Fatal("expected error when non-existent certificate is explicitly specified")
1496	1498		}
1497	1499		}
1498		-

Commit 50e9dba

Commit `50e9dba`