manager.go

  1// Package services spawns, health-checks, and tears down the two self-hosted
  2// model services (llama.cpp router and audio.cpp) as subprocesses. It owns the
  3// process lifecycle only; it does not serve game or LLM logic.
  4package services
  5
  6import (
  7	"context"
  8	"fmt"
  9	"os/exec"
 10	"syscall"
 11	"time"
 12)
 13
 14const (
 15	LLMListen   = "127.0.0.1:9931"
 16	AudioListen = "127.0.0.1:9932"
 17
 18	minFreeVRAMBytes  = 16 << 30         // 16 GiB free required to start
 19	modelReadyTimeout = 5 * time.Minute  // total budget for both services to become healthy
 20	killGrace         = 10 * time.Second // SIGTERM -> wait -> SIGKILL
 21)
 22
 23const (
 24	llamaBinary = "llama-server"
 25	audioBinary = "audiocpp_server"
 26	audioConfig = "audio.cpp.json"
 27)
 28
 29// llamaServerArgs mirrors the Makefile `llama` target but bound to loopback.
 30// Keep in sync with the Makefile.
 31var llamaServerArgs = []string{
 32	"--host", "127.0.0.1",
 33	"--port", "9931",
 34	"--hf-repo", "unsloth/gemma-4-12B-it-qat-GGUF:UD-Q4_K_XL",
 35	"--n-gpu-layers", "99",
 36	"--n-gpu-layers-draft", "99",
 37	"--fit", "off",
 38	"--flash-attn", "on",
 39	"--parallel", "5",
 40	"--kv-unified",
 41	"--ctx-size", "32768",
 42	"--batch-size", "1024",
 43	"--ubatch-size", "1024",
 44	"--temp", "1.0",
 45	"--top-p", "0.95",
 46	"--top-k", "64",
 47	"--spec-type", "draft-mtp",
 48	"--spec-draft-n-max", "4",
 49	"--reasoning", "off",
 50}
 51
 52type child struct {
 53	cmd  *exec.Cmd
 54	pgid int
 55	done chan struct{}
 56}
 57
 58// Manager owns the spawned model-service subprocesses. Create one with
 59// NewManager, Start it, and Stop it on exit. Stop is safe to call multiple times.
 60type Manager struct {
 61	llama   *child
 62	audio   *child
 63	stopped bool
 64}
 65
 66func NewManager() *Manager { return &Manager{} }
 67
 68// Start runs the VRAM gate, spawns both services (all-or-nothing), and blocks
 69// until all three probes are healthy or ctx / the readiness deadline expires.
 70// On any failure it tears down whatever was started and returns an error.
 71func (m *Manager) Start(ctx context.Context) error {
 72	if err := checkVRAM(); err != nil {
 73		return err
 74	}
 75
 76	llama, err := startChild("llama-server", llamaBinary, llamaServerArgs)
 77	if err != nil {
 78		return err
 79	}
 80	m.llama = llama
 81
 82	audio, err := startChild("audiocpp_server", audioBinary, []string{"--config", audioConfig})
 83	if err != nil {
 84		m.Stop()
 85		return err
 86	}
 87	m.audio = audio
 88
 89	if err := m.waitHealthy(ctx); err != nil {
 90		m.Stop()
 91		return err
 92	}
 93	return nil
 94}
 95
 96// Stop SIGTERMs each child process group, waits up to killGrace, SIGKILLs any
 97// survivor, and reaps both. Safe to call multiple times; a no-op if nothing was
 98// started.
 99func (m *Manager) Stop() {
100	if m.stopped {
101		return
102	}
103	m.stopped = true
104
105	living := m.livingChildren()
106	if len(living) == 0 {
107		return
108	}
109
110	for _, c := range living {
111		go reap(c)
112	}
113	for _, c := range living {
114		_ = syscall.Kill(-c.pgid, syscall.SIGTERM)
115	}
116	deadline := time.Now().Add(killGrace)
117	for _, c := range living {
118		if !waitDone(c, deadline) {
119			_ = syscall.Kill(-c.pgid, syscall.SIGKILL)
120		}
121	}
122	for _, c := range living {
123		<-c.done
124	}
125}
126
127func (m *Manager) livingChildren() []*child {
128	var out []*child
129	if m.llama != nil {
130		out = append(out, m.llama)
131	}
132	if m.audio != nil {
133		out = append(out, m.audio)
134	}
135	return out
136}
137
138func startChild(name, binary string, args []string) (*child, error) {
139	cmd := exec.Command(binary, args...)
140	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
141	if err := cmd.Start(); err != nil {
142		return nil, fmt.Errorf("services: start %s: %w", name, err)
143	}
144	return &child{cmd: cmd, pgid: cmd.Process.Pid, done: make(chan struct{})}, nil
145}
146
147func reap(c *child) {
148	_ = c.cmd.Wait()
149	close(c.done)
150}
151
152// waitDone reports whether the child has exited by the deadline. It never blocks
153// past the deadline; the final drain in Stop guarantees reaping either way.
154func waitDone(c *child, deadline time.Time) bool {
155	wait := time.Until(deadline)
156	if wait <= 0 {
157		select {
158		case <-c.done:
159			return true
160		default:
161			return false
162		}
163	}
164	t := time.NewTimer(wait)
165	defer t.Stop()
166	select {
167	case <-c.done:
168		return true
169	case <-t.C:
170		return false
171	}
172}