feat(runner): support tar files for workspaces

Commit `8fc0a2e`

Eric Bower · 2026-05-07 16:34:40 -0400 EDT

parent 1ac7c95

feat(runner): support tar files for workspaces

4 files changed, +136, -15

M README.md

M event.json

M hooks/post-receive

M main.go

M README.md

+3, -0

...	...		@@ -1 +1,4 @@
1	1		# pici - a simple ci system
	2	+
	3	+	testing
	4	+	hook test 3 1778209807

M event.json

+1, -1

......@@ -1 +1 @@
{"type":"git.push","name":"pici","workspace":"/home/erock/dev/pici/"}
{"type":"git.push","name":"pici","workspace":"pgs.sh:/private-ci/workspaces/pici_8f0a260d.tar","branch":"main","commit":"8f0a260d584027f657e38b44f397b97f59099e85","artifact_dest":"","artifact_url":""}

M hooks/post-receive

+34, -14

...	...		@@ -1,5 +1,6 @@
1	1		#!/usr/bin/env bash
2		-	# post-receive hook — publishes build events to pici via ssh pipe.
	2	+	# post-receive hook — archives the pushed commit and uploads to pgs.sh,
	3	+	# then publishes a build event to pici via ssh pipe.
3	4		#
4	5		# Install: ./install-hooks.sh /path/to/bare-repos
5	6		#

...	...		@@ -8,24 +9,43 @@
8	9		set -euo pipefail
9	10
10	11		PIPE_HOST="${PIPE_HOST:-pipe}"
	12	+	PGS_HOST="${PGS_HOST:-pgs.sh}"
	13	+
	14	+	log() {
	15	+	echo "[post-receive] $*" >&2
	16	+	}
11	17
12	18		while read -r old_sha new_sha ref; do
13		-	# Skip delete refs
14		-	if [ "$new_sha" = "0000000000000000000000000000000000000000" ]; then
15		-	continue
16		-	fi
	19	+	# Skip delete refs
	20	+	if [ "$new_sha" = "0000000000000000000000000000000000000000" ]; then
	21	+	log "skip delete ref: $ref"
	22	+	continue
	23	+	fi
	24	+
	25	+	# Extract branch name from ref (e.g. refs/heads/main → main)
	26	+	branch="${ref#refs/heads/}"
	27	+
	28	+	# Repo name and short SHA from bare repo
	29	+	repo="$(basename "$(pwd)" .git)"
	30	+	short_sha="${new_sha:0:8}"
	31	+
	32	+	log "repo=$repo branch=$branch commit=$new_sha"
17	33
18		-	# Extract branch name from ref (e.g. refs/heads/main → main)
19		-	branch="${ref#refs/heads/}"
	34	+	tar_path="/private-ci/workspaces/${repo}_${short_sha}.tar"
	35	+	log "upload: git archive $new_sha → $PGS_HOST:$tar_path"
20	36
21		-	# Repo name from the bare repo directory
22		-	repo="$(basename "$(pwd)" .git)"
	37	+	if ! git archive "$new_sha" \| ssh -T "$PGS_HOST" "$tar_path"; then
	38	+	log "ERROR: upload failed" >&2
	39	+	continue
	40	+	fi
23	41
24		-	# Workspace is the bare repo itself (runner rsyncs from here)
25		-	workspace="$(pwd)"
	42	+	# Workspace points to the tar on pgs.sh
	43	+	workspace="${PGS_HOST}:${tar_path}"
26	44
27		-	event=$(printf '{"type":"git.push","name":"%s","workspace":"%s","branch":"%s","commit":"%s"}' \
28		-	"$repo" "$workspace" "$branch" "$new_sha")
	45	+	event=$(printf '{"type":"git.push","name":"%s","workspace":"%s","branch":"%s","commit":"%s"}' \
	46	+	"$repo" "$workspace" "$branch" "$new_sha")
29	47
30		-	echo "$event" \| ssh "$PIPE_HOST" pub -b=false build.event
	48	+	log "publish: $event"
	49	+	echo "$event" \| ssh -T "$PIPE_HOST" pub -b=false build.event
	50	+	log "done"
31	51		done

M main.go

+98, -0

......@@ -1,6 +1,7 @@
package main

import (
	"archive/tar"
	"bufio"
	"context"
	"crypto/sha256"

......@@ -23,6 +24,13 @@ import (
type WorkspaceFactory func(cfg *Cfg, logger *slog.Logger, source string) Workspace

func defaultWorkspaceFactory(cfg *Cfg, logger *slog.Logger, source string) Workspace {
	if strings.HasSuffix(source, ".tar") {
		return &WorkspaceTar{
			Cfg:    cfg,
			Logger: logger,
			Source: source,
		}
	}
	return &WorkspaceRsync{
		Cfg:    cfg,
		Logger: logger,

......@@ -340,6 +348,96 @@ func (w *WorkspaceRsync) GetDir() string {
	return w.Dest
}

type WorkspaceTar struct {
	Cfg    *Cfg
	Logger *slog.Logger
	Source string // e.g. "pgs.sh:/private-ci/workspaces/repo_abc123.tar"
	Dest   string
}

func (w *WorkspaceTar) Setup() error {
	tempDir, err := os.MkdirTemp("", "pici-*")
	if err != nil {
		return err
	}
	w.Dest = tempDir

	log := w.Logger.With("source", w.Source, "dest", w.Dest)
	log.Debug("downloading and extracting workspace tar")

	// Parse host:path from source
	host, path := splitSSHSource(w.Source)

	// Use rsync to download the tar file
	tarPath := filepath.Join(tempDir, "workspace.tar")
	rsyncCmd := exec.Command("rsync", "-e", "ssh", host+":"+path, tarPath)
	rsyncCmd.Stderr = os.Stderr
	if err := rsyncCmd.Run(); err != nil {
		return fmt.Errorf("rsync download: %w", err)
	}

	// Open the tar file for extraction
	f, err := os.Open(tarPath)
	if err != nil {
		return fmt.Errorf("open tar: %w", err)
	}
	defer f.Close()

	// Extract tar to temp dir
	tr := tar.NewReader(f)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("tar read: %w", err)
		}

		target := filepath.Join(tempDir, hdr.Name)
		switch hdr.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(target, 0755); err != nil {
				return fmt.Errorf("mkdir %s: %w", target, err)
			}
		case tar.TypeReg:
			// Ensure parent dir exists
			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
				return fmt.Errorf("mkdir parent %s: %w", filepath.Dir(target), err)
			}
			tf, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
			if err != nil {
				return fmt.Errorf("open %s: %w", target, err)
			}
			if _, err := io.Copy(tf, tr); err != nil {
				tf.Close()
				return fmt.Errorf("write %s: %w", target, err)
			}
			tf.Close()
		}
	}

	log.Debug("workspace extracted")
	return nil
}

func (w *WorkspaceTar) Cleanup() error {
	return nil
}

func (w *WorkspaceTar) GetDir() string {
	return w.Dest
}

// splitSSHSource splits "host:path" into host and path.
func splitSSHSource(source string) (string, string) {
	idx := strings.Index(source, ":")
	if idx == -1 {
		return source, ""
	}
	return source[:idx], source[idx+1:]
}

type JobEngine struct {
	Wk     Workspace
	Logger *slog.Logger

...	...		@@ -340,6 +348,96 @@ func (w *WorkspaceRsync) GetDir() string {
340	348		return w.Dest
341	349		}
342	350
	351	+	type WorkspaceTar struct {
	352	+	Cfg *Cfg
	353	+	Logger *slog.Logger
	354	+	Source string // e.g. "pgs.sh:/private-ci/workspaces/repo_abc123.tar"
	355	+	Dest string
	356	+	}
	357	+
	358	+	func (w *WorkspaceTar) Setup() error {
	359	+	tempDir, err := os.MkdirTemp("", "pici-*")
	360	+	if err != nil {
	361	+	return err
	362	+	}
	363	+	w.Dest = tempDir
	364	+
	365	+	log := w.Logger.With("source", w.Source, "dest", w.Dest)
	366	+	log.Debug("downloading and extracting workspace tar")
	367	+
	368	+	// Parse host:path from source
	369	+	host, path := splitSSHSource(w.Source)
	370	+
	371	+	// Use rsync to download the tar file
	372	+	tarPath := filepath.Join(tempDir, "workspace.tar")
	373	+	rsyncCmd := exec.Command("rsync", "-e", "ssh", host+":"+path, tarPath)
	374	+	rsyncCmd.Stderr = os.Stderr
	375	+	if err := rsyncCmd.Run(); err != nil {
	376	+	return fmt.Errorf("rsync download: %w", err)
	377	+	}
	378	+
	379	+	// Open the tar file for extraction
	380	+	f, err := os.Open(tarPath)
	381	+	if err != nil {
	382	+	return fmt.Errorf("open tar: %w", err)
	383	+	}
	384	+	defer f.Close()
	385	+
	386	+	// Extract tar to temp dir
	387	+	tr := tar.NewReader(f)
	388	+	for {
	389	+	hdr, err := tr.Next()
	390	+	if err == io.EOF {
	391	+	break
	392	+	}
	393	+	if err != nil {
	394	+	return fmt.Errorf("tar read: %w", err)
	395	+	}
	396	+
	397	+	target := filepath.Join(tempDir, hdr.Name)
	398	+	switch hdr.Typeflag {
	399	+	case tar.TypeDir:
	400	+	if err := os.MkdirAll(target, 0755); err != nil {
	401	+	return fmt.Errorf("mkdir %s: %w", target, err)
	402	+	}
	403	+	case tar.TypeReg:
	404	+	// Ensure parent dir exists
	405	+	if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
	406	+	return fmt.Errorf("mkdir parent %s: %w", filepath.Dir(target), err)
	407	+	}
	408	+	tf, err := os.OpenFile(target, os.O_CREATE\|os.O_WRONLY, os.FileMode(hdr.Mode))
	409	+	if err != nil {
	410	+	return fmt.Errorf("open %s: %w", target, err)
	411	+	}
	412	+	if _, err := io.Copy(tf, tr); err != nil {
	413	+	tf.Close()
	414	+	return fmt.Errorf("write %s: %w", target, err)
	415	+	}
	416	+	tf.Close()
	417	+	}
	418	+	}
	419	+
	420	+	log.Debug("workspace extracted")
	421	+	return nil
	422	+	}
	423	+
	424	+	func (w *WorkspaceTar) Cleanup() error {
	425	+	return nil
	426	+	}
	427	+
	428	+	func (w *WorkspaceTar) GetDir() string {
	429	+	return w.Dest
	430	+	}
	431	+
	432	+	// splitSSHSource splits "host:path" into host and path.
	433	+	func splitSSHSource(source string) (string, string) {
	434	+	idx := strings.Index(source, ":")
	435	+	if idx == -1 {
	436	+	return source, ""
	437	+	}
	438	+	return source[:idx], source[idx+1:]
	439	+	}
	440	+
343	441		type JobEngine struct {
344	442		Wk Workspace
345	443		Logger *slog.Logger

...	...		@@ -1 +1 @@
1		-	{"type":"git.push","name":"pici","workspace":"/home/erock/dev/pici/"}
	1	+	{"type":"git.push","name":"pici","workspace":"pgs.sh:/private-ci/workspaces/pici_8f0a260d.tar","branch":"main","commit":"8f0a260d584027f657e38b44f397b97f59099e85","artifact_dest":"","artifact_url":""}

...	...		@@ -1,6 +1,7 @@
1	1		package main
2	2
3	3		import (
	4	+	"archive/tar"
4	5		"bufio"
5	6		"context"
6	7		"crypto/sha256"

...	...		@@ -23,6 +24,13 @@ import (
23	24		type WorkspaceFactory func(cfg Cfg, logger slog.Logger, source string) Workspace
24	25
25	26		func defaultWorkspaceFactory(cfg Cfg, logger slog.Logger, source string) Workspace {
	27	+	if strings.HasSuffix(source, ".tar") {
	28	+	return &WorkspaceTar{
	29	+	Cfg: cfg,
	30	+	Logger: logger,
	31	+	Source: source,
	32	+	}
	33	+	}
26	34		return &WorkspaceRsync{
27	35		Cfg: cfg,
28	36		Logger: logger,

Commit 8fc0a2e

Commit `8fc0a2e`