main main_test.go
Eric Bower  ·  2026-08-15
   1package main
   2
   3import (
   4	"bufio"
   5	"bytes"
   6	"context"
   7	"encoding/json"
   8	"errors"
   9	"fmt"
  10	"io"
  11	"log/slog"
  12	"os"
  13	"os/exec"
  14	"path/filepath"
  15	"strings"
  16	"testing"
  17	"time"
  18)
  19
  20// TestE2E_RunnerWithZMXSessions is a full integration test that:
  21// 1. Creates a workspace with pico.sh that spawns zmx sessions
  22// 2. Feeds an event to RunRunner (fire-and-forget)
  23// 3. Runs the monitor to track job completion
  24// 4. Reads the status file and asserts correct status transitions.
  25func TestE2E_RunnerWithZMXSessions(t *testing.T) {
  26	if testing.Short() {
  27		t.Skip("skip integration test")
  28	}
  29	zmxPath, err := exec.LookPath("zmx")
  30	if err != nil {
  31		t.Skip("zmx not found, skipping integration test")
  32	}
  33	zmxDir := filepath.Dir(zmxPath)
  34
  35	// 1. Create workspace with pico.sh that spawns zmx sessions
  36	workspaceDir := t.TempDir()
  37	picoSh := fmt.Sprintf(`#!/usr/bin/env bash
  38set -e
  39export PATH="%s:$PATH"
  40zmx run step1 echo "hello from step1"
  41zmx run step2 echo "hello from step2"
  42`, zmxDir)
  43	if err := os.WriteFile(filepath.Join(workspaceDir, "pico.sh"), []byte(picoSh), 0755); err != nil {
  44		t.Fatalf("write pico.sh: %v", err)
  45	}
  46
  47	// 2. Create config
  48	artifactDir := t.TempDir()
  49	ctx, cancel := context.WithCancel(context.Background())
  50	testJobID := fmt.Sprintf("j-%d", time.Now().UnixNano()%100000000)
  51	t.Cleanup(func() {
  52		_ = exec.Command("zmx", "kill", "-f", fmt.Sprintf("local.test-repo.%s", testJobID)).Run()
  53	})
  54	event := Event{
  55		Type:      "local",
  56		Name:      "test-repo",
  57		JobID:     testJobID,
  58		Workspace: workspaceDir,
  59	}
  60	eventJSON, _ := json.Marshal(event)
  61	statusBuf := new(bytes.Buffer)
  62	cfg := &Cfg{
  63		Logger:          slog.New(slog.NewTextHandler(io.Discard, nil)),
  64		Ctx:             ctx,
  65		Cancel:          cancel,
  66		ArtifactDir:     artifactDir,
  67		EventSource:     io.NopCloser(bytes.NewReader(append(eventJSON, '\n'))),
  68		MonitorInterval: 200 * time.Millisecond,
  69		NewWorkspace:    defaultWorkspaceFactory,
  70		StatusOutput:    statusBuf,
  71		IncludeRunning:  true,
  72		SessionPrefix:   "local.",
  73	}
  74
  75	// 3. Run the runner (fire-and-forget, exits quickly)
  76	runnerDone := make(chan error, 1)
  77	go func() {
  78		runnerDone <- RunRunner(cfg)
  79	}()
  80
  81	select {
  82	case err := <-runnerDone:
  83		if err != nil {
  84			t.Fatalf("runner: %v", err)
  85		}
  86	case <-time.After(30 * time.Second):
  87		t.Fatal("timeout waiting for runner to complete")
  88	}
  89
  90	// Verify that RunRunner does NOT create published.json (monitor's job)
  91	sentinel := filepath.Join(artifactDir, "test-repo", testJobID, "artifacts", "published.json")
  92	if _, err := os.Stat(sentinel); err == nil {
  93		t.Fatalf("expected published.json NOT to exist after RunRunner, but it exists at %s", sentinel)
  94	}
  95
  96	// 4. Run the monitor and poll for incremental artifacts
  97	monitorDone := make(chan error, 1)
  98	go func() {
  99		monitorDone <- runMonitor(cfg)
 100	}()
 101
 102	// Track what we've seen during the run
 103	seenIndexHTML := false
 104	seenIndexTXT := false
 105	sessionArtifactShorts := make(map[string]bool)
 106	var finalPayload *StatusPayload
 107
 108	for {
 109		select {
 110		case err := <-monitorDone:
 111			t.Fatalf("monitor exited unexpectedly: %v", err)
 112		case <-time.After(30 * time.Second):
 113			t.Fatal("timeout waiting for job to complete")
 114		default:
 115		}
 116
 117		// Read all statuses and artifacts seen so far
 118		data := statusBuf.Bytes()
 119		if len(data) > 0 {
 120			lines := scanLines(data)
 121			for _, line := range lines {
 122				var p StatusPayload
 123				if err := json.Unmarshal([]byte(line), &p); err != nil {
 124					continue
 125				}
 126
 127				// Ignore statuses from unrelated jobs (e.g., leftover sessions from previous tests)
 128				if p.Name != "test-repo" || p.JobID != testJobID {
 129					continue
 130				}
 131
 132				// Check index files exist (generated at every tick)
 133				indexHTMLPath := filepath.Join(artifactDir, p.Name, p.JobID, "index.html")
 134				indexTXTPath := filepath.Join(artifactDir, p.Name, p.JobID, "index.txt")
 135				if _, err := os.Stat(indexHTMLPath); err == nil {
 136					seenIndexHTML = true
 137					t.Logf("index.html exists (status=%s)", p.Status)
 138				}
 139				if _, err := os.Stat(indexTXTPath); err == nil {
 140					seenIndexTXT = true
 141				}
 142
 143				// Track per-session artifacts
 144				for _, s := range p.Sessions {
 145					htmlPath := filepath.Join(artifactDir, p.Name, p.JobID, s.Short+".html")
 146					txtPath := filepath.Join(artifactDir, p.Name, p.JobID, s.Short+".txt")
 147					if _, err := os.Stat(htmlPath); err == nil {
 148						sessionArtifactShorts[s.Short+"_html"] = true
 149					}
 150					if _, err := os.Stat(txtPath); err == nil {
 151						sessionArtifactShorts[s.Short+"_txt"] = true
 152					}
 153				}
 154
 155				// Check if we've reached a final state
 156				if p.Status == "success" || p.Status == "failed" {
 157					cancel() // stop the monitor
 158					finalPayload = &p
 159					break
 160				}
 161			}
 162		}
 163
 164		if finalPayload != nil {
 165			break
 166		}
 167
 168		// Assert progress: if we see session artifacts, we should see index files too
 169		if len(sessionArtifactShorts) > 0 && (!seenIndexHTML || !seenIndexTXT) {
 170			t.Error("index files should exist once any session artifact exists")
 171		}
 172
 173		time.Sleep(cfg.MonitorInterval / 2) // poll at twice the interval rate
 174	}
 175
 176	// Wait for monitor to exit gracefully
 177	select {
 178	case <-monitorDone:
 179	case <-time.After(5 * time.Second):
 180		t.Log("warning: monitor did not exit gracefully")
 181	}
 182
 183	// 5. Assert we saw index files during the run
 184	if !seenIndexHTML {
 185		t.Error("never saw index.html during monitoring")
 186	}
 187	if !seenIndexTXT {
 188		t.Error("never saw index.txt during monitoring")
 189	}
 190
 191	// 6. Assert final payload has correct data
 192	if finalPayload == nil {
 193		t.Fatal("no final payload")
 194	}
 195	// Filter to only sessions with our job prefix to avoid picking up unrelated local.* sessions
 196	expectedPrefix := fmt.Sprintf("local.%s.%s.", finalPayload.Name, finalPayload.JobID)
 197	sessions := make([]SessionInfo, 0, len(finalPayload.Sessions))
 198	for _, s := range finalPayload.Sessions {
 199		if strings.HasPrefix(s.Name, expectedPrefix) {
 200			sessions = append(sessions, s)
 201		}
 202	}
 203	finalPayload.Sessions = sessions
 204
 205	if finalPayload.Name != "test-repo" {
 206		t.Errorf("expected name test-repo, got %q", finalPayload.Name)
 207	}
 208	if finalPayload.Status != "success" {
 209		t.Errorf("expected status success, got %q", finalPayload.Status)
 210	}
 211	if finalPayload.ExitCode == nil || *finalPayload.ExitCode != 0 {
 212		t.Errorf("expected exit code 0, got %v", finalPayload.ExitCode)
 213	}
 214	if len(finalPayload.Sessions) < 2 {
 215		t.Errorf("expected at least 2 sessions, got %d", len(finalPayload.Sessions))
 216	}
 217
 218	// 7. Assert sessions have correct names
 219	sessionNames := make(map[string]bool)
 220	for _, s := range finalPayload.Sessions {
 221		sessionNames[s.Short] = true
 222		t.Logf("session: name=%s short=%s exit_code=%s ended=%s", s.Name, s.Short, s.ExitCode, s.Ended)
 223	}
 224	// Sessions from the test's pico.sh: step1 and step2
 225	if !sessionNames["step1"] {
 226		t.Error("expected session 'step1'")
 227	}
 228	if !sessionNames["step2"] {
 229		t.Error("expected session 'step2'")
 230	}
 231
 232	// 8. Assert HTML artifacts were staged for each session
 233	for _, s := range finalPayload.Sessions {
 234		artifactPath := filepath.Join(cfg.ArtifactDir, finalPayload.Name, finalPayload.JobID, s.Short+".html")
 235		data, err := os.ReadFile(artifactPath)
 236		if err != nil {
 237			t.Errorf("read artifact %s: %v", artifactPath, err)
 238			continue
 239		}
 240		if len(data) == 0 {
 241			t.Errorf("artifact %s is empty", artifactPath)
 242		}
 243		if !bytes.Contains(data, []byte("<div")) {
 244			t.Errorf("artifact %s does not contain HTML content", artifactPath)
 245		}
 246	}
 247
 248	// Cleanup any leftover zmx sessions for this job
 249	if finalPayload != nil {
 250		_ = exec.Command("zmx", "kill", "-f", fmt.Sprintf("local.test-repo.%s", finalPayload.JobID)).Run()
 251	}
 252}
 253
 254func scanLines(data []byte) []string {
 255	var lines []string
 256	scanner := bufio.NewScanner(bytes.NewReader(data))
 257	for scanner.Scan() {
 258		lines = append(lines, scanner.Text())
 259	}
 260	return lines
 261}
 262
 263// TestE2E_DuplicateCancellation verifies that starting a new job for the same
 264// repo cancels any existing running job for that repo.
 265func TestE2E_DuplicateCancellation(t *testing.T) {
 266	if testing.Short() {
 267		t.Skip("skip integration test")
 268	}
 269	if _, err := exec.LookPath("zmx"); err != nil {
 270		t.Skip("zmx not found, skipping integration test")
 271	}
 272
 273	// 1. Create workspace with a slow pico.sh so the first job stays running
 274	workspaceDir := t.TempDir()
 275	picoSh := `#!/usr/bin/env bash
 276set -e
 277zmx run slow-step sleep 30
 278`
 279	if err := os.WriteFile(filepath.Join(workspaceDir, "pico.sh"), []byte(picoSh), 0755); err != nil {
 280		t.Fatalf("write pico.sh: %v", err)
 281	}
 282
 283	// 2. Create a fast pico.sh for the second job
 284	workspaceDir2 := t.TempDir()
 285	picoSh2 := `#!/usr/bin/env bash
 286set -e
 287zmx run fast-step echo "done"
 288`
 289	if err := os.WriteFile(filepath.Join(workspaceDir2, "pico.sh"), []byte(picoSh2), 0755); err != nil {
 290		t.Fatalf("write pico.sh: %v", err)
 291	}
 292
 293	artifactDir := t.TempDir()
 294	ctx, cancel := context.WithCancel(context.Background())
 295	defer cancel()
 296	t.Cleanup(func() {
 297		_ = exec.Command("zmx", "kill", "-f", "local.dup-repo").Run()
 298	})
 299
 300	makeCfg := func(eventSource io.ReadCloser) *Cfg {
 301		return &Cfg{
 302			Logger:          slog.New(slog.NewTextHandler(io.Discard, nil)),
 303			Ctx:             ctx,
 304			Cancel:          cancel,
 305			ArtifactDir:     artifactDir,
 306			EventSource:     eventSource,
 307			MonitorInterval: 200 * time.Millisecond,
 308			NewWorkspace:    defaultWorkspaceFactory,
 309		}
 310	}
 311
 312	// 3. Start first job (slow)
 313	event1 := Event{Type: "local", Name: "dup-repo", Workspace: workspaceDir}
 314	event1JSON, _ := json.Marshal(event1)
 315	cfg1 := makeCfg(io.NopCloser(bytes.NewReader(append(event1JSON, '\n'))))
 316
 317	go func() {
 318		_ = RunRunner(cfg1)
 319	}()
 320
 321	// Wait for the first job's runner session to appear
 322	if !waitForSessionPrefix(t, "local.dup-repo.", 10*time.Second) {
 323		t.Fatal("first job's runner session never appeared")
 324	}
 325
 326	// Record the first job's runner session name
 327	firstRunner := findRunnerSession(t, "dup-repo")
 328	if firstRunner == "" {
 329		t.Fatal("could not find first job's runner session")
 330	}
 331	t.Logf("first job runner: %s", firstRunner)
 332
 333	// 4. Start second job (same repo name, should cancel the first)
 334	event2 := Event{Type: "local", Name: "dup-repo", Workspace: workspaceDir2}
 335	event2JSON, _ := json.Marshal(event2)
 336	cfg2 := makeCfg(io.NopCloser(bytes.NewReader(append(event2JSON, '\n'))))
 337
 338	runner2Done := make(chan error, 1)
 339	go func() {
 340		runner2Done <- RunRunner(cfg2)
 341	}()
 342
 343	// Wait for second runner to complete
 344	select {
 345	case err := <-runner2Done:
 346		if err != nil {
 347			t.Fatalf("second runner: %v", err)
 348		}
 349	case <-time.After(30 * time.Second):
 350		t.Fatal("timeout waiting for second runner")
 351	}
 352
 353	// 5. Verify the first job's runner session was killed
 354	// Give zmx kill time to propagate
 355	time.Sleep(1 * time.Second)
 356
 357	listOutput, _ := exec.Command("zmx", "list").CombinedOutput()
 358	sessions := parseZMXList(string(listOutput))
 359	for _, s := range sessions {
 360		if s.Name == firstRunner {
 361			if s.Ended == "" {
 362				t.Errorf("first job's runner session %s should have been killed (ended is empty)", firstRunner)
 363			} else {
 364				t.Logf("first job's runner session %s was killed (ended=%s)", firstRunner, s.Ended)
 365			}
 366		}
 367	}
 368
 369	// Cleanup
 370	_ = exec.Command("zmx", "kill", "-f", "local.dup-repo").Run()
 371}
 372
 373// waitForSessionPrefix returns true if a session with the given prefix appears within the timeout.
 374func waitForSessionPrefix(t *testing.T, prefix string, timeout time.Duration) bool {
 375	deadline := time.Now().Add(timeout)
 376	for time.Now().Before(deadline) {
 377		listOutput, err := exec.Command("zmx", "list").CombinedOutput()
 378		if err == nil {
 379			sessions := parseZMXList(string(listOutput))
 380			for _, s := range sessions {
 381				if strings.HasPrefix(s.Name, prefix) {
 382					return true
 383				}
 384			}
 385		}
 386		time.Sleep(200 * time.Millisecond)
 387	}
 388	return false
 389}
 390
 391// findRunnerSession finds the runner session for a given repo name.
 392func findRunnerSession(t *testing.T, name string) string {
 393	listOutput, err := exec.Command("zmx", "list").CombinedOutput()
 394	if err != nil {
 395		t.Fatalf("zmx list: %v", err)
 396	}
 397	sessions := parseZMXList(string(listOutput))
 398	for _, s := range sessions {
 399		if (strings.HasPrefix(s.Name, "local."+name+".") || strings.HasPrefix(s.Name, "ci."+name+".")) && strings.HasSuffix(s.Name, ".runner") {
 400			return s.Name
 401		}
 402	}
 403	return ""
 404}
 405
 406func TestGenerateJobID(t *testing.T) {
 407	// Same inputs should produce same hash
 408	id1 := jobIDFor("myrepo", "/workspace", 1000)
 409	id2 := jobIDFor("myrepo", "/workspace", 1000)
 410	if id1 != id2 {
 411		t.Errorf("expected same ID for same inputs, got %q and %q", id1, id2)
 412	}
 413
 414	// Different name should produce different hash
 415	id3 := jobIDFor("otherrepo", "/workspace", 1000)
 416	if id1 == id3 {
 417		t.Errorf("expected different IDs for different names, got %q", id1)
 418	}
 419
 420	// Different timestamp should produce different hash
 421	id4 := jobIDFor("myrepo", "/workspace", 2000)
 422	if id1 == id4 {
 423		t.Errorf("expected different IDs for different timestamps, got %q", id1)
 424	}
 425
 426	// ID should be 8 hex chars
 427	if len(id1) != 8 {
 428		t.Errorf("expected 8 char ID, got %d chars: %q", len(id1), id1)
 429	}
 430
 431	// generateJobID (with real time) should also produce valid IDs
 432	id := generateJobID("myrepo", "/workspace")
 433	if len(id) != 8 {
 434		t.Errorf("generateJobID expected 8 char ID, got %d chars: %q", len(id), id)
 435	}
 436}
 437
 438func TestValidateJobID(t *testing.T) {
 439	tests := []struct {
 440		id   string
 441		want bool
 442	}{
 443		{"a3f2b8c1", true},           // 8-char hex (commit short SHA)
 444		{"abc-123", true},            // alphanumeric + hyphen
 445		{"PR42", true},               // uppercase + digits
 446		{"a", true},                  // single char
 447		{"abcdefghij123456", true},   // 16 chars (max)
 448		{"", false},                  // empty
 449		{"abcdefghij1234567", false}, // 17 chars (too long)
 450		{"abc.def", false},           // contains dot (breaks parseJobPrefix)
 451		{"abc/def", false},           // contains slash
 452		{"abc def", false},           // contains space
 453		{"abc_def", false},           // contains underscore
 454	}
 455
 456	for _, tt := range tests {
 457		got := validateJobID(tt.id)
 458		if got != tt.want {
 459			t.Errorf("validateJobID(%q) = %v, want %v", tt.id, got, tt.want)
 460		}
 461	}
 462}
 463
 464func TestResolveJobID(t *testing.T) {
 465	// Valid producer ID is used as-is
 466	id := resolveJobID("myrepo", "/workspace", "a3f2b8c1")
 467	if id != "a3f2b8c1" {
 468		t.Errorf("resolveJobID with valid ID = %q, want %q", id, "a3f2b8c1")
 469	}
 470
 471	// Empty producer ID falls back to generated
 472	id = resolveJobID("myrepo", "/workspace", "")
 473	if id == "" || len(id) != 8 {
 474		t.Errorf("resolveJobID with empty ID = %q, want 8-char generated ID", id)
 475	}
 476
 477	// Invalid producer ID (contains dot) falls back to generated
 478	id = resolveJobID("myrepo", "/workspace", "1.0.0")
 479	if id == "1.0.0" || len(id) != 8 {
 480		t.Errorf("resolveJobID with invalid ID = %q, want 8-char generated ID", id)
 481	}
 482
 483	// Invalid producer ID (too long) falls back to generated
 484	id = resolveJobID("myrepo", "/workspace", "this_is_way_too_long_for_a_job_id")
 485	if len(id) != 8 {
 486		t.Errorf("resolveJobID with too-long ID = %q, want 8-char generated ID", id)
 487	}
 488}
 489
 490func TestAllCompleted(t *testing.T) {
 491	tests := []struct {
 492		name     string
 493		sessions []SessionInfo
 494		want     bool
 495	}{
 496		{
 497			name:     "empty sessions",
 498			sessions: []SessionInfo{},
 499			want:     true,
 500		},
 501		{
 502			name: "all completed",
 503			sessions: []SessionInfo{
 504				{Name: "a", Ended: "123"},
 505				{Name: "b", Ended: "456"},
 506			},
 507			want: true,
 508		},
 509		{
 510			name: "one not completed",
 511			sessions: []SessionInfo{
 512				{Name: "a", Ended: "123"},
 513				{Name: "b", Ended: ""},
 514			},
 515			want: false,
 516		},
 517	}
 518
 519	for _, tt := range tests {
 520		t.Run(tt.name, func(t *testing.T) {
 521			got := allCompleted(tt.sessions)
 522			if got != tt.want {
 523				t.Errorf("allCompleted() = %v, want %v", got, tt.want)
 524			}
 525		})
 526	}
 527}
 528
 529func TestIsJobComplete(t *testing.T) {
 530	tests := []struct {
 531		name     string
 532		sessions []SessionInfo
 533		want     bool
 534	}{
 535		{
 536			name: "all completed with runner",
 537			sessions: []SessionInfo{
 538				{Name: "ci.repo.abc.runner", Ended: "100"},
 539				{Name: "ci.repo.abc.step.fmt", Ended: "90"},
 540			},
 541			want: true,
 542		},
 543		{
 544			name: "runner active, step completed",
 545			sessions: []SessionInfo{
 546				{Name: "ci.repo.abc.runner", Ended: ""},
 547				{Name: "ci.repo.abc.step.fmt", Ended: "90"},
 548			},
 549			want: false,
 550		},
 551		{
 552			name: "no runner session present",
 553			sessions: []SessionInfo{
 554				{Name: "ci.repo.abc.step.fmt", Ended: "90"},
 555			},
 556			want: false,
 557		},
 558		{
 559			name: "runner completed, step active",
 560			sessions: []SessionInfo{
 561				{Name: "ci.repo.abc.runner", Ended: "100"},
 562				{Name: "ci.repo.abc.step.fmt", Ended: ""},
 563			},
 564			want: false,
 565		},
 566		{
 567			name:     "empty sessions",
 568			sessions: []SessionInfo{},
 569			want:     false,
 570		},
 571	}
 572
 573	for _, tt := range tests {
 574		t.Run(tt.name, func(t *testing.T) {
 575			got := isJobComplete(tt.sessions)
 576			if got != tt.want {
 577				t.Errorf("isJobComplete() = %v, want %v", got, tt.want)
 578			}
 579		})
 580	}
 581}
 582
 583func TestParseZMXList(t *testing.T) {
 584	output := `name=ci-lint	pid=1064464	clients=0	created=1777519944	start_dir=/home/erock/dev/pico	ended=1777519986	exit_code=0
 585  name=ci-tests	pid=1064472	clients=0	created=1777519944	start_dir=/home/erock/dev/pico	ended=1777519958	exit_code=2
 586→ name=d.build.1	pid=549652	clients=0	created=1777513430	start_dir=/home/erock`
 587
 588	sessions := parseZMXList(output)
 589	if len(sessions) != 3 {
 590		t.Fatalf("expected 3 sessions, got %d", len(sessions))
 591	}
 592
 593	if sessions[0].Name != "ci-lint" {
 594		t.Errorf("expected first session name ci-lint, got %q", sessions[0].Name)
 595	}
 596	if sessions[0].Ended != "1777519986" {
 597		t.Errorf("expected ended 1777519986, got %q", sessions[0].Ended)
 598	}
 599	if sessions[0].ExitCode != "0" {
 600		t.Errorf("expected exit_code 0, got %q", sessions[0].ExitCode)
 601	}
 602
 603	if sessions[2].Name != "d.build.1" {
 604		t.Errorf("expected third session name d.build.1, got %q", sessions[2].Name)
 605	}
 606	if sessions[2].Ended != "" {
 607		t.Errorf("expected empty ended for active session, got %q", sessions[2].Ended)
 608	}
 609}
 610
 611func TestGetDomain(t *testing.T) {
 612	tests := []struct {
 613		eventType string
 614		want      string
 615	}{
 616		{"git.push", "ci"},
 617		{"git.tag", "ci"},
 618		{"push", "ci"},
 619		{"manual", "ci"},
 620		{"ci", "ci"},
 621		{"", "ci"},
 622		{"local", "local"},
 623	}
 624
 625	for _, tt := range tests {
 626		got := getDomain(tt.eventType)
 627		if got != tt.want {
 628			t.Errorf("getDomain(%q) = %q, want %q", tt.eventType, got, tt.want)
 629		}
 630	}
 631}
 632
 633func TestExtractJobID(t *testing.T) {
 634	tests := []struct {
 635		runnerName string
 636		want       string
 637	}{
 638		{"ci.myrepo.abc123.runner", "abc123"},
 639		{"ci.test-repo.006d0847.runner", "006d0847"},
 640		{"ci.my_org.project.abc123.runner", "project.abc123"}, // name with underscore
 641	}
 642
 643	for _, tt := range tests {
 644		got := extractJobID(tt.runnerName)
 645		if got != tt.want {
 646			t.Errorf("extractJobID(%q) = %q, want %q", tt.runnerName, got, tt.want)
 647		}
 648	}
 649}
 650
 651func TestExtractJobPrefix(t *testing.T) {
 652	tests := []struct {
 653		sessionName string
 654		want        string
 655	}{
 656		{"ci.myrepo.abc123.lint", "ci.myrepo.abc123."},
 657		{"ci.myrepo.abc123.runner", "ci.myrepo.abc123."},
 658		{"ci.myrepo.abc123.tests", "ci.myrepo.abc123."},
 659		{"ci.name.jobID.step.substep", "ci.name.jobID."},
 660		{"ci.a.b", ""}, // too few parts
 661	}
 662
 663	for _, tt := range tests {
 664		got := extractJobPrefix(tt.sessionName)
 665		if got != tt.want {
 666			t.Errorf("extractJobPrefix(%q) = %q, want %q", tt.sessionName, got, tt.want)
 667		}
 668	}
 669}
 670
 671func TestFindRunningJobs(t *testing.T) {
 672	output := `name=ci.myrepo.abc123.runner	pid=100	clients=0	created=1777519944	start_dir=/home/erock
 673  name=ci.myrepo.abc123.lint	pid=101	clients=0	created=1777519944	start_dir=/home/erock
 674  name=ci.myrepo.abc123.tests	pid=102	clients=0	created=1777519944	start_dir=/home/erock	ended=1777519986	exit_code=0
 675  name=ci.myrepo.def456.runner	pid=103	clients=0	created=1777519944	start_dir=/home/erock	ended=1777519986	exit_code=0
 676  name=ci.other.abc123.runner	pid=104	clients=0	created=1777519944	start_dir=/home/erock
 677→ name=d.build.1	pid=549652	clients=0	created=1777513430	start_dir=/home/erock`
 678
 679	runners, sessions := findRunningJobsFromOutput(output, "myrepo")
 680	if len(runners) != 1 {
 681		t.Fatalf("expected 1 running job, got %d: %v", len(runners), runners)
 682	}
 683	if runners[0] != "ci.myrepo.abc123.runner" {
 684		t.Errorf("expected runner ci.myrepo.abc123.runner, got %q", runners[0])
 685	}
 686	if len(sessions) != 6 {
 687		t.Errorf("expected 6 total sessions, got %d", len(sessions))
 688	}
 689}
 690
 691func findRunningJobsFromOutput(output, name string) ([]string, []SessionInfo) {
 692	sessions := parseZMXList(output)
 693	var runners []string
 694	for _, s := range sessions {
 695		if strings.HasPrefix(s.Name, "ci."+name+".") && strings.HasSuffix(s.Name, ".runner") && s.Ended == "" {
 696			runners = append(runners, s.Name)
 697		}
 698	}
 699	return runners, sessions
 700}
 701
 702func TestFindSessionsForGC(t *testing.T) {
 703	now := time.Unix(1786289700, 0)
 704	twoHoursAgo := fmt.Sprintf("%d", now.Add(-2*time.Hour).Unix())
 705	fourHoursAgo := fmt.Sprintf("%d", now.Add(-4*time.Hour).Unix())
 706
 707	sessions := []SessionInfo{
 708		{Name: "ci.repo.active.runner", Created: twoHoursAgo, Ended: ""},
 709		{Name: "ci.repo.active.step.fmt", Created: twoHoursAgo, Ended: twoHoursAgo},
 710		{Name: "ci.repo.recent.runner", Created: twoHoursAgo, Ended: twoHoursAgo},
 711		{Name: "ci.repo.recent.step.lint", Created: twoHoursAgo, Ended: twoHoursAgo},
 712		{Name: "ci.repo.oldfinished.runner", Created: fourHoursAgo, Ended: fourHoursAgo},
 713		{Name: "ci.repo.oldfinished.step.lint", Created: fourHoursAgo, Ended: fourHoursAgo},
 714		{Name: "ci.repo.expired.runner", Created: fourHoursAgo, Ended: ""},
 715		{Name: "other.session", Created: fourHoursAgo, Ended: ""},
 716	}
 717
 718	toKill := findSessionsForGC(sessions, []string{"ci.", "local."}, now)
 719	if len(toKill) != 3 {
 720		t.Fatalf("expected 3 sessions for GC, got %d: %v", len(toKill), toKill)
 721	}
 722	expected := map[string]bool{
 723		"ci.repo.oldfinished.runner":    true,
 724		"ci.repo.oldfinished.step.lint": true,
 725		"ci.repo.expired.runner":        true,
 726	}
 727	for _, k := range toKill {
 728		if !expected[k] {
 729			t.Errorf("unexpected session in GC kill list: %s", k)
 730		}
 731	}
 732}
 733
 734func TestKillSessionsEmpty(t *testing.T) {
 735	// Should not error with empty list
 736	if err := killSessions(nil); err != nil {
 737		t.Errorf("killSessions(nil) = %v, want nil", err)
 738	}
 739	if err := killSessions([]string{}); err != nil {
 740		t.Errorf("killSessions([]) = %v, want nil", err)
 741	}
 742}
 743
 744func TestResolveJobExitCode(t *testing.T) {
 745	tests := []struct {
 746		name       string
 747		sessions   []SessionInfo
 748		wantCode   int
 749		wantStatus string
 750	}{
 751		{
 752			name: "all success",
 753			sessions: []SessionInfo{
 754				{Name: "ci.repo.abc.runner", ExitCode: "0", Ended: "1"},
 755				{Name: "ci.repo.abc.step1", ExitCode: "0", Ended: "1"},
 756			},
 757			wantCode:   0,
 758			wantStatus: "success",
 759		},
 760		{
 761			name: "runner failed",
 762			sessions: []SessionInfo{
 763				{Name: "ci.repo.abc.runner", ExitCode: "1", Ended: "1"},
 764				{Name: "ci.repo.abc.step1", ExitCode: "0", Ended: "1"},
 765			},
 766			wantCode:   1,
 767			wantStatus: "failed",
 768		},
 769		{
 770			name: "child failed, runner says 0 (defensive)",
 771			sessions: []SessionInfo{
 772				{Name: "ci.repo.abc.runner", ExitCode: "0", Ended: "1"},
 773				{Name: "ci.repo.abc.step1", ExitCode: "0", Ended: "1"},
 774				{Name: "ci.repo.abc.step2", ExitCode: "2", Ended: "1"},
 775			},
 776			wantCode:   2,
 777			wantStatus: "failed",
 778		},
 779		{
 780			name: "worst child exit code wins",
 781			sessions: []SessionInfo{
 782				{Name: "ci.repo.abc.runner", ExitCode: "0", Ended: "1"},
 783				{Name: "ci.repo.abc.step1", ExitCode: "1", Ended: "1"},
 784				{Name: "ci.repo.abc.step2", ExitCode: "3", Ended: "1"},
 785			},
 786			wantCode:   3,
 787			wantStatus: "failed",
 788		},
 789		{
 790			name: "no runner session",
 791			sessions: []SessionInfo{
 792				{Name: "ci.repo.abc.step1", ExitCode: "0", Ended: "1"},
 793			},
 794			wantCode:   0,
 795			wantStatus: "success",
 796		},
 797		{
 798			name: "sessions not yet ended (no exit code)",
 799			sessions: []SessionInfo{
 800				{Name: "ci.repo.abc.runner", ExitCode: "", Ended: ""},
 801				{Name: "ci.repo.abc.step1", ExitCode: "", Ended: ""},
 802			},
 803			wantCode:   0,
 804			wantStatus: "success",
 805		},
 806	}
 807
 808	for _, tt := range tests {
 809		t.Run(tt.name, func(t *testing.T) {
 810			code, status := resolveJobExitCode(tt.sessions)
 811			if code != tt.wantCode {
 812				t.Errorf("exit code = %d, want %d", code, tt.wantCode)
 813			}
 814			if status != tt.wantStatus {
 815				t.Errorf("status = %q, want %q", status, tt.wantStatus)
 816			}
 817		})
 818	}
 819}
 820
 821func TestRunLocal_MissingPico(t *testing.T) {
 822	tempDir := t.TempDir()
 823	origWd, _ := os.Getwd()
 824	defer func() { _ = os.Chdir(origWd) }()
 825	_ = os.Chdir(tempDir)
 826
 827	cfg := &Cfg{
 828		ArtifactDir: t.TempDir(),
 829	}
 830	err := runLocal(cfg, "")
 831	if err == nil {
 832		t.Fatal("expected error when pico.sh is missing")
 833	}
 834}
 835
 836func TestRunLocal_EnvOverrides(t *testing.T) {
 837	tempDir := t.TempDir()
 838	origWd, _ := os.Getwd()
 839	defer func() { _ = os.Chdir(origWd) }()
 840	_ = os.Chdir(tempDir)
 841
 842	picoContent := `#!/usr/bin/env bash
 843echo "hello from pico"
 844`
 845	if err := os.WriteFile("pico.sh", []byte(picoContent), 0755); err != nil {
 846		t.Fatal(err)
 847	}
 848
 849	cfg := &Cfg{
 850		ArtifactDir: t.TempDir(),
 851		EnvVars:     envList{"PICI_REPO=custom-repo", "CUSTOM_VAR=hello"},
 852	}
 853
 854	_ = runLocal(cfg, "")
 855
 856	if os.Getenv("CUSTOM_VAR") != "hello" {
 857		t.Errorf("expected CUSTOM_VAR to be hello, got %q", os.Getenv("CUSTOM_VAR"))
 858	}
 859}
 860
 861func TestDetectGitCommit(t *testing.T) {
 862	cwd, err := os.Getwd()
 863	if err != nil {
 864		t.Fatal(err)
 865	}
 866	commit := detectGitCommit(cwd)
 867	if commit == "" {
 868		t.Log("git commit sha not detected (not a git repository or git unavailable)")
 869	} else {
 870		t.Logf("detected git commit sha: %s", commit)
 871	}
 872}
 873
 874func TestWaitAndReport_Cancellation(t *testing.T) {
 875	if testing.Short() {
 876		t.Skip("skip integration test")
 877	}
 878	if _, err := exec.LookPath("zmx"); err != nil {
 879		t.Skip("zmx not found, skipping integration test")
 880	}
 881
 882	ctx, cancel := context.WithCancel(context.Background())
 883	cfg := &Cfg{
 884		Ctx:             ctx,
 885		Cancel:          cancel,
 886		MonitorInterval: 100 * time.Millisecond,
 887	}
 888
 889	jobID := fmt.Sprintf("canceltest-%d", time.Now().UnixNano())
 890	prefix := "local.testrepo." + jobID + "."
 891	runnerSession := prefix + "runner"
 892
 893	t.Cleanup(func() {
 894		_ = exec.Command("zmx", "kill", "-f", runnerSession).Run()
 895	})
 896
 897	cmd := exec.Command("zmx", "run", runnerSession, "-d", "sleep", "30")
 898	if err := cmd.Run(); err != nil {
 899		t.Fatalf("failed to start zmx session: %v", err)
 900	}
 901
 902	go func() {
 903		time.Sleep(200 * time.Millisecond)
 904		cancel()
 905	}()
 906
 907	err := waitAndReport(cfg, nil, "testrepo", jobID, "local")
 908	if err == nil {
 909		t.Error("expected error when waitAndReport is cancelled, got nil")
 910	}
 911
 912	time.Sleep(200 * time.Millisecond)
 913	listOutput, _ := exec.Command("zmx", "list").CombinedOutput()
 914	sessions := parseZMXList(string(listOutput))
 915	for _, s := range sessions {
 916		if s.Name == runnerSession {
 917			if s.Ended == "" {
 918				t.Errorf("session %s should have been killed on cancellation", runnerSession)
 919			}
 920		}
 921	}
 922}
 923
 924func TestWorkspaceRsync_ExcludesGitAndJJ(t *testing.T) {
 925	if _, err := exec.LookPath("rsync"); err != nil {
 926		t.Skip("rsync not found, skipping test")
 927	}
 928
 929	srcDir := t.TempDir()
 930	if err := os.WriteFile(filepath.Join(srcDir, "hello.txt"), []byte("world"), 0644); err != nil {
 931		t.Fatalf("failed to write hello.txt: %v", err)
 932	}
 933	if err := os.WriteFile(filepath.Join(srcDir, ".gitignore"), []byte("ignored.txt\n"), 0644); err != nil {
 934		t.Fatalf("failed to write .gitignore: %v", err)
 935	}
 936	if err := os.WriteFile(filepath.Join(srcDir, "ignored.txt"), []byte("secret"), 0644); err != nil {
 937		t.Fatalf("failed to write ignored.txt: %v", err)
 938	}
 939	if err := os.MkdirAll(filepath.Join(srcDir, ".git"), 0755); err != nil {
 940		t.Fatalf("failed to mkdir .git: %v", err)
 941	}
 942	if err := os.WriteFile(filepath.Join(srcDir, ".git", "HEAD"), []byte("ref: refs/heads/main"), 0644); err != nil {
 943		t.Fatalf("failed to write .git/HEAD: %v", err)
 944	}
 945	if err := os.MkdirAll(filepath.Join(srcDir, ".jj"), 0755); err != nil {
 946		t.Fatalf("failed to mkdir .jj: %v", err)
 947	}
 948	if err := os.WriteFile(filepath.Join(srcDir, ".jj", "repo"), []byte("jj data"), 0644); err != nil {
 949		t.Fatalf("failed to write .jj/repo: %v", err)
 950	}
 951
 952	wk := &WorkspaceRsync{
 953		Cfg:    &Cfg{},
 954		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
 955		Source: srcDir,
 956	}
 957	t.Cleanup(func() {
 958		_ = wk.Cleanup()
 959	})
 960
 961	if err := wk.Setup(); err != nil {
 962		t.Fatalf("WorkspaceRsync.Setup failed: %v", err)
 963	}
 964
 965	destDir := wk.GetDir()
 966	if _, err := os.Stat(filepath.Join(destDir, "hello.txt")); os.IsNotExist(err) {
 967		t.Error("expected hello.txt to exist in dest workspace")
 968	}
 969	if _, err := os.Stat(filepath.Join(destDir, "ignored.txt")); !os.IsNotExist(err) {
 970		t.Error("expected ignored.txt to be excluded by .gitignore from dest workspace")
 971	}
 972	if _, err := os.Stat(filepath.Join(destDir, ".git")); !os.IsNotExist(err) {
 973		t.Error("expected .git to be excluded from dest workspace")
 974	}
 975	if _, err := os.Stat(filepath.Join(destDir, ".jj")); !os.IsNotExist(err) {
 976		t.Error("expected .jj to be excluded from dest workspace")
 977	}
 978
 979	checksum := wk.Checksum()
 980	if !strings.HasPrefix(checksum, "sha256:") {
 981		t.Errorf("expected checksum to start with 'sha256:', got %q", checksum)
 982	}
 983}
 984
 985func TestRunnerDoesNotCreatePublishedSentinel(t *testing.T) {
 986	if _, err := exec.LookPath("zmx"); err != nil {
 987		t.Skip("zmx not found, skipping test")
 988	}
 989
 990	tmpDir := t.TempDir()
 991	artifactDir := filepath.Join(tmpDir, "artifacts")
 992	wkDir := filepath.Join(tmpDir, "workspace")
 993
 994	if err := os.MkdirAll(wkDir, 0755); err != nil {
 995		t.Fatalf("failed to create workspace dir: %v", err)
 996	}
 997
 998	picoScript := `#!/usr/bin/env bash
 999set -euo pipefail
1000zmx run step1 -d echo "step1"
1001zmx wait "*"
1002`
1003	if err := os.WriteFile(filepath.Join(wkDir, "pico.sh"), []byte(picoScript), 0755); err != nil {
1004		t.Fatalf("failed to write pico.sh: %v", err)
1005	}
1006
1007	jobID := "j-no-published-test"
1008	event := Event{
1009		Type:      "local",
1010		Name:      "no-published-repo",
1011		JobID:     jobID,
1012		Workspace: wkDir,
1013	}
1014	eventJSON, _ := json.Marshal(event)
1015
1016	cfg := &Cfg{
1017		Logger:        slog.New(slog.NewTextHandler(io.Discard, nil)),
1018		ArtifactDir:   artifactDir,
1019		EventSource:   io.NopCloser(bytes.NewReader(append(eventJSON, '\n'))),
1020		NewWorkspace:  func(cfg *Cfg, logger *slog.Logger, source string) Workspace { return &mockLocalWorkspace{dir: source} },
1021		SessionPrefix: "test.",
1022	}
1023
1024	if err := RunRunner(cfg); err != nil {
1025		t.Fatalf("RunRunner failed: %v", err)
1026	}
1027
1028	sentinel := filepath.Join(artifactDir, "no-published-repo", jobID, "artifacts", "published.json")
1029	if _, err := os.Stat(sentinel); err == nil {
1030		t.Errorf("expected published.json NOT to exist after RunRunner, but it exists at %s", sentinel)
1031	}
1032}
1033
1034type mockLocalWorkspace struct {
1035	dir string
1036}
1037
1038func (w *mockLocalWorkspace) Setup() error     { return nil }
1039func (w *mockLocalWorkspace) Cleanup() error   { return nil }
1040func (w *mockLocalWorkspace) GetDir() string   { return w.dir }
1041func (w *mockLocalWorkspace) Checksum() string { return "" }
1042
1043func TestJobEngine_ExportsArtifactsDir(t *testing.T) {
1044	tmpDir := t.TempDir()
1045	eng := &JobEngine{
1046		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
1047		Cfg: &Cfg{
1048			ArtifactDir: tmpDir,
1049		},
1050		Ev: &Event{
1051			Name: "myrepo",
1052			Type: "local",
1053		},
1054		JobID: "job123",
1055	}
1056
1057	expectedArtifactsDir := filepath.Join(tmpDir, "myrepo", "job123", "artifacts")
1058
1059	// Verify that artifacts directory path matches expected layout
1060	artifactsDir := filepath.Join(eng.Cfg.ArtifactDir, eng.Ev.Name, eng.JobID, "artifacts")
1061	if artifactsDir != expectedArtifactsDir {
1062		t.Errorf("expected artifactsDir %s, got %s", expectedArtifactsDir, artifactsDir)
1063	}
1064}
1065
1066func TestGenerateJobIndex_WalksSubdirectories(t *testing.T) {
1067	tmpDir := t.TempDir()
1068	repoName := "testrepo"
1069	jobID := "j999"
1070	artifactsDir := filepath.Join(tmpDir, repoName, jobID, "artifacts")
1071
1072	if err := os.MkdirAll(filepath.Join(artifactsDir, "build"), 0755); err != nil {
1073		t.Fatalf("mkdir failed: %v", err)
1074	}
1075	if err := os.WriteFile(filepath.Join(artifactsDir, "build", "output.json"), []byte("{}"), 0644); err != nil {
1076		t.Fatalf("write file failed: %v", err)
1077	}
1078
1079	htmlContent, txtContent := generateJobIndex(tmpDir, repoName, jobID, nil)
1080	if !strings.Contains(htmlContent, "build/output.json") {
1081		t.Errorf("html index missing nested artifact build/output.json: %s", htmlContent)
1082	}
1083	if !strings.Contains(txtContent, "build/output.json") {
1084		t.Errorf("txt index missing nested artifact build/output.json: %s", txtContent)
1085	}
1086}
1087
1088func TestWaitAndReport_ExitCodeFailure(t *testing.T) {
1089	if _, err := exec.LookPath("zmx"); err != nil {
1090		t.Skip("zmx not found, skipping test")
1091	}
1092
1093	cfg := &Cfg{
1094		MonitorInterval: 100 * time.Millisecond,
1095		ArtifactDir:     t.TempDir(),
1096	}
1097
1098	jobID := fmt.Sprintf("failtest-%d", time.Now().UnixNano())
1099	prefix := "local.testrepo." + jobID + "."
1100	runnerSession := prefix + "runner"
1101
1102	t.Cleanup(func() {
1103		_ = exec.Command("zmx", "kill", "-f", runnerSession).Run()
1104	})
1105
1106	// Start a zmx session that exits with code 42
1107	cmd := exec.Command("zmx", "run", runnerSession, "-d", "bash", "-c", "exit 42")
1108	if err := cmd.Run(); err != nil {
1109		t.Fatalf("failed to start zmx session: %v", err)
1110	}
1111
1112	err := waitAndReport(cfg, nil, "testrepo", jobID, "local")
1113	if err == nil {
1114		t.Fatal("expected error from waitAndReport for failed job, got nil")
1115	}
1116
1117	var jobFailedErr *JobFailedError
1118	if !errors.As(err, &jobFailedErr) {
1119		t.Fatalf("expected error of type *JobFailedError, got %T: %v", err, err)
1120	}
1121	if jobFailedErr.ExitCode != 42 {
1122		t.Errorf("expected exit code 42, got %d", jobFailedErr.ExitCode)
1123	}
1124}
1125
1126func TestWaitAndReport_Success(t *testing.T) {
1127	if _, err := exec.LookPath("zmx"); err != nil {
1128		t.Skip("zmx not found, skipping test")
1129	}
1130
1131	cfg := &Cfg{
1132		MonitorInterval: 100 * time.Millisecond,
1133		ArtifactDir:     t.TempDir(),
1134	}
1135
1136	jobID := fmt.Sprintf("successtest-%d", time.Now().UnixNano())
1137	prefix := "local.testrepo." + jobID + "."
1138	runnerSession := prefix + "runner"
1139
1140	t.Cleanup(func() {
1141		_ = exec.Command("zmx", "kill", "-f", runnerSession).Run()
1142	})
1143
1144	// Start a zmx session that exits with code 0
1145	cmd := exec.Command("zmx", "run", runnerSession, "-d", "bash", "-c", "exit 0")
1146	if err := cmd.Run(); err != nil {
1147		t.Fatalf("failed to start zmx session: %v", err)
1148	}
1149
1150	err := waitAndReport(cfg, nil, "testrepo", jobID, "local")
1151	if err != nil {
1152		t.Fatalf("expected nil error for successful job, got %v", err)
1153	}
1154}
1155
1156func TestRunLocal_ExitCodeFailure(t *testing.T) {
1157	if _, err := exec.LookPath("zmx"); err != nil {
1158		t.Skip("zmx not found, skipping test")
1159	}
1160
1161	tempDir := t.TempDir()
1162	origWd, _ := os.Getwd()
1163	defer func() { _ = os.Chdir(origWd) }()
1164	_ = os.Chdir(tempDir)
1165
1166	picoContent := `#!/usr/bin/env bash
1167exit 7
1168`
1169	if err := os.WriteFile("pico.sh", []byte(picoContent), 0755); err != nil {
1170		t.Fatal(err)
1171	}
1172
1173	artifactDir := t.TempDir()
1174	cfg := &Cfg{
1175		ArtifactDir:     artifactDir,
1176		MonitorInterval: 100 * time.Millisecond,
1177	}
1178
1179	err := runLocal(cfg, "")
1180	if err == nil {
1181		t.Fatal("expected error from runLocal when pico.sh exits non-zero, got nil")
1182	}
1183
1184	var jobFailedErr *JobFailedError
1185	if !errors.As(err, &jobFailedErr) {
1186		t.Fatalf("expected error of type *JobFailedError, got %T: %v", err, err)
1187	}
1188	if jobFailedErr.ExitCode != 7 {
1189		t.Errorf("expected exit code 7, got %d", jobFailedErr.ExitCode)
1190	}
1191}
1192
1193func TestRunRunner_Wait_ExitCodeFailure(t *testing.T) {
1194	if _, err := exec.LookPath("zmx"); err != nil {
1195		t.Skip("zmx not found, skipping test")
1196	}
1197
1198	workspaceDir := t.TempDir()
1199	picoContent := `#!/usr/bin/env bash
1200exit 13
1201`
1202	if err := os.WriteFile(filepath.Join(workspaceDir, "pico.sh"), []byte(picoContent), 0755); err != nil {
1203		t.Fatal(err)
1204	}
1205
1206	eventJSON := fmt.Sprintf(`{"type":"local","name":"runner-fail-repo","workspace":%q}`, workspaceDir)
1207
1208	artifactDir := t.TempDir()
1209	cfg := &Cfg{
1210		ArtifactDir:     artifactDir,
1211		MonitorInterval: 100 * time.Millisecond,
1212		Wait:            true,
1213		Event:           eventJSON,
1214		Logger:          slog.New(slog.NewTextHandler(io.Discard, nil)),
1215		NewWorkspace:    defaultWorkspaceFactory,
1216	}
1217
1218	err := RunRunner(cfg)
1219	if err == nil {
1220		t.Fatal("expected error from RunRunner --wait on failed job, got nil")
1221	}
1222
1223	var jobFailedErr *JobFailedError
1224	if !errors.As(err, &jobFailedErr) {
1225		t.Fatalf("expected wrapped *JobFailedError, got %T: %v", err, err)
1226	}
1227	if jobFailedErr.ExitCode != 13 {
1228		t.Errorf("expected exit code 13, got %d", jobFailedErr.ExitCode)
1229	}
1230}
1231
1232func TestWorkspaceInPlace(t *testing.T) {
1233	tempDir := t.TempDir()
1234	testFile := filepath.Join(tempDir, "test.txt")
1235	if err := os.WriteFile(testFile, []byte("hello world"), 0644); err != nil {
1236		t.Fatal(err)
1237	}
1238
1239	cfg := &Cfg{
1240		InPlace: true,
1241	}
1242	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
1243	wk := defaultWorkspaceFactory(cfg, logger, tempDir)
1244
1245	if _, ok := wk.(*WorkspaceInPlace); !ok {
1246		t.Fatalf("expected WorkspaceInPlace, got %T", wk)
1247	}
1248
1249	if err := wk.Setup(); err != nil {
1250		t.Fatalf("Setup failed: %v", err)
1251	}
1252
1253	if wk.GetDir() != tempDir {
1254		t.Errorf("expected GetDir() == %q, got %q", tempDir, wk.GetDir())
1255	}
1256
1257	if wk.Checksum() == "" {
1258		t.Errorf("expected non-empty checksum")
1259	}
1260
1261	if err := wk.Cleanup(); err != nil {
1262		t.Fatalf("Cleanup failed: %v", err)
1263	}
1264
1265	if _, err := os.Stat(testFile); os.IsNotExist(err) {
1266		t.Fatalf("expected test file to still exist after Cleanup() on in-place workspace")
1267	}
1268}
1269
1270func TestRunLocal_InPlace(t *testing.T) {
1271	if _, err := exec.LookPath("zmx"); err != nil {
1272		t.Skip("zmx not found, skipping test")
1273	}
1274
1275	tempDir := t.TempDir()
1276	workDir := filepath.Join(tempDir, "inplace-repo")
1277	if err := os.MkdirAll(workDir, 0755); err != nil {
1278		t.Fatal(err)
1279	}
1280	origWd, _ := os.Getwd()
1281	defer func() { _ = os.Chdir(origWd) }()
1282	_ = os.Chdir(workDir)
1283
1284	picoContent := `#!/usr/bin/env bash
1285set -e
1286echo "inplace marker" > marker.txt
1287`
1288	if err := os.WriteFile("pico.sh", []byte(picoContent), 0755); err != nil {
1289		t.Fatal(err)
1290	}
1291
1292	artifactDir := t.TempDir()
1293	cfg := &Cfg{
1294		ArtifactDir:     artifactDir,
1295		MonitorInterval: 100 * time.Millisecond,
1296		InPlace:         true,
1297	}
1298
1299	err := runLocal(cfg, "")
1300	if err != nil {
1301		t.Fatalf("expected runLocal in-place to succeed, got %v", err)
1302	}
1303
1304	markerPath := filepath.Join(workDir, "marker.txt")
1305	data, err := os.ReadFile(markerPath)
1306	if err != nil {
1307		t.Fatalf("expected marker.txt to be created in working directory: %v", err)
1308	}
1309	if !strings.Contains(string(data), "inplace marker") {
1310		t.Errorf("unexpected marker.txt content: %q", string(data))
1311	}
1312}
1313
1314func TestRunLocal_SummaryFile(t *testing.T) {
1315	if _, err := exec.LookPath("zmx"); err != nil {
1316		t.Skip("zmx not found, skipping test")
1317	}
1318
1319	tempDir := t.TempDir()
1320	workDir := filepath.Join(tempDir, "summary-repo")
1321	if err := os.MkdirAll(workDir, 0755); err != nil {
1322		t.Fatal(err)
1323	}
1324	origWd, _ := os.Getwd()
1325	defer func() { _ = os.Chdir(origWd) }()
1326	_ = os.Chdir(workDir)
1327
1328	picoContent := `#!/usr/bin/env bash
1329echo "hello from step"
1330`
1331	if err := os.WriteFile("pico.sh", []byte(picoContent), 0755); err != nil {
1332		t.Fatal(err)
1333	}
1334
1335	artifactDir := t.TempDir()
1336	summaryFilePath := filepath.Join(workDir, "summary.md")
1337	cfg := &Cfg{
1338		ArtifactDir:     artifactDir,
1339		MonitorInterval: 100 * time.Millisecond,
1340		InPlace:         true,
1341		SummaryFile:     summaryFilePath,
1342	}
1343
1344	err := runLocal(cfg, "")
1345	if err != nil {
1346		t.Fatalf("expected runLocal to succeed, got %v", err)
1347	}
1348
1349	summaryBytes, err := os.ReadFile(summaryFilePath)
1350	if err != nil {
1351		t.Fatalf("expected summary file %s to exist: %v", summaryFilePath, err)
1352	}
1353
1354	summary := string(summaryBytes)
1355	if !strings.Contains(summary, "### Pici CI Summary:") {
1356		t.Errorf("expected summary to contain header, got:\n%s", summary)
1357	}
1358	if !strings.Contains(summary, "| Session | Status | Exit Code | Duration |") {
1359		t.Errorf("expected summary to contain table header, got:\n%s", summary)
1360	}
1361}
1362
1363func TestWriteSummaryMarkdown_WithFailure(t *testing.T) {
1364	tempFile := filepath.Join(t.TempDir(), "summary.md")
1365	sessions := []SessionInfo{
1366		{Short: "setup", Name: "local.repo.job.setup", ExitCode: "0"},
1367		{Short: "failing-step", Name: "local.repo.job.failing-step", ExitCode: "42"},
1368	}
1369	known := map[string]*sessionState{
1370		"setup":        {status: "success", exitCode: "0", duration: "1.2s"},
1371		"failing-step": {status: "failed", exitCode: "42", duration: "3.4s"},
1372	}
1373
1374	err := writeSummaryMarkdown(tempFile, "test-repo", "job-123", sessions, known, false, "")
1375	if err != nil {
1376		t.Fatalf("writeSummaryMarkdown failed: %v", err)
1377	}
1378
1379	data, err := os.ReadFile(tempFile)
1380	if err != nil {
1381		t.Fatalf("read summary file: %v", err)
1382	}
1383	content := string(data)
1384
1385	if !strings.Contains(content, "### Pici CI Summary: `test-repo` (Job `job-123`)") {
1386		t.Errorf("missing header in content:\n%s", content)
1387	}
1388	if !strings.Contains(content, "| `failing-step` | ❌ failed | `42` | 3.4s |") {
1389		t.Errorf("missing table row in content:\n%s", content)
1390	}
1391	if !strings.Contains(content, "### Failed Session Details") {
1392		t.Errorf("missing failed details in content:\n%s", content)
1393	}
1394}
1395
1396func TestWriteSummaryMarkdown_WithDebugActive(t *testing.T) {
1397	tempFile := filepath.Join(t.TempDir(), "summary.md")
1398	sessions := []SessionInfo{
1399		{Short: "failing-step", Name: "local.repo.job.failing-step", ExitCode: "42"},
1400	}
1401	known := map[string]*sessionState{
1402		"failing-step": {status: "failed", exitCode: "42", duration: "1.0s"},
1403	}
1404
1405	err := writeSummaryMarkdown(tempFile, "my-repo", "job-456", sessions, known, true, "ci.my-repo.job-456")
1406	if err != nil {
1407		t.Fatalf("writeSummaryMarkdown failed: %v", err)
1408	}
1409
1410	data, err := os.ReadFile(tempFile)
1411	if err != nil {
1412		t.Fatalf("read summary file: %v", err)
1413	}
1414	content := string(data)
1415
1416	if !strings.Contains(content, "### 🔧 Remote Debug Active") {
1417		t.Errorf("expected debug header in summary markdown, got:\n%s", content)
1418	}
1419	if !strings.Contains(content, "ssh -t pipe.pico.sh pipe ci.my-repo.job-456") {
1420		t.Errorf("expected ssh command in summary markdown, got:\n%s", content)
1421	}
1422}
1423
1424func TestScrubEnvironment(t *testing.T) {
1425	envIn := []string{
1426		"PATH=/usr/bin:/bin",
1427		"USER=runner",
1428		"HOME=/home/runner",
1429		"SHELL=/bin/bash",
1430		"TERM=xterm-256color",
1431		"ZMX_SESSION_PREFIX=ci.repo.123.",
1432		"PICI_JOB=123",
1433		"GITHUB_TOKEN=ghp_secret_token_12345",
1434		"ACTIONS_RUNTIME_TOKEN=actions_token_67890",
1435		"MY_APP_SECRET=super_secret_val",
1436		"API_KEY=abc123xyz",
1437		"DATABASE_PASSWORD=secret_db_pass",
1438		"PICO_SSH_KEY=-----BEGIN OPENSSH PRIVATE KEY-----",
1439		"CUSTOM_SAFE_FLAG=1",
1440	}
1441
1442	cleaned := scrubEnvironment(envIn)
1443	joined := strings.Join(cleaned, "\n")
1444
1445	// Verify safe vars remain
1446	for _, expected := range []string{
1447		"PATH=/usr/bin:/bin",
1448		"USER=runner",
1449		"HOME=/home/runner",
1450		"SHELL=/bin/bash",
1451		"ZMX_SESSION_PREFIX=ci.repo.123.",
1452		"PICI_JOB=123",
1453		"CUSTOM_SAFE_FLAG=1",
1454	} {
1455		if !strings.Contains(joined, expected) {
1456			t.Errorf("expected %q to be preserved in environment, got:\n%s", expected, joined)
1457		}
1458	}
1459
1460	// Verify secrets are scrubbed
1461	for _, forbidden := range []string{
1462		"GITHUB_TOKEN",
1463		"ACTIONS_RUNTIME_TOKEN",
1464		"MY_APP_SECRET",
1465		"API_KEY",
1466		"DATABASE_PASSWORD",
1467		"PICO_SSH_KEY",
1468	} {
1469		if strings.Contains(joined, forbidden) {
1470			t.Errorf("expected %q to be SCRUBBED from environment, but found it in:\n%s", forbidden, joined)
1471		}
1472	}
1473}
1474
1475func TestFindSSHAuthMethod_ExplicitKeyAndCert(t *testing.T) {
1476	tempDir := t.TempDir()
1477	keyPath := filepath.Join(tempDir, "id_ed25519")
1478
1479	// Generate a temporary ED25519 key
1480	cmd := exec.Command("ssh-keygen", "-t", "ed25519", "-N", "", "-f", keyPath)
1481	if err := cmd.Run(); err != nil {
1482		t.Skipf("ssh-keygen not available or failed: %v", err)
1483	}
1484
1485	// Test 1: Explicit key path
1486	auth, err := findSSHAuthMethod(keyPath, "")
1487	if err != nil {
1488		t.Fatalf("findSSHAuthMethod failed: %v", err)
1489	}
1490	if auth == nil {
1491		t.Fatal("expected non-nil auth method")
1492	}
1493
1494	// Test 2: Non-existent cert path explicitly specified returns error
1495	_, err = findSSHAuthMethod(keyPath, filepath.Join(tempDir, "nonexistent-cert.pub"))
1496	if err == nil {
1497		t.Fatal("expected error when non-existent certificate is explicitly specified")
1498	}
1499}