feat(phase2-E): multi-provider routing via secutools delegation

Adds optional delegation of agent-queue tasks to the SecuAAS secutools
AI platform (GPU / Gemini / Claude API) instead of dispatching to a
local Claude Code tmux session. Per-task opt-in via YAML frontmatter
fields preferred_ai, allow_delegation, complexity_hint — absence keeps
the Phase 1 behaviour exactly (zero breaking change).

Go side:
- internal/secutools: HTTP client with exponential-backoff retries
  (SubmitJob/GetJob/WaitForResult), DecideProvider map adapter for CLI
  use, table tests.
- internal/router: struct-typed Decide() with strict precedence
  (needs_claude_code > preferred_ai=claude-code > allow_delegation=false
  > preferred_ai > fail-safe local on unknown).
- internal/delegation: Manager submits jobs, writes .md.delegated
  markers for on-restart recovery, runs a periodic reaper that moves
  completed jobs into done/ with provider/cost footer and failed jobs
  into failed/.
- internal/dispatcher: WithDelegation() opt-in, routeTask hook before
  findFreeSession, skips .md.delegated in assignNextTask.
- internal/api: /api/delegated/status (active jobs + counters),
  /watchdog/status extended with delegation counters.
- cmd/ccl-delegate: small CLI exposing submit/get/result/decide so the
  bash dispatcher can call the same contract without duplicating logic.
- cmd/claude-failover: delegation wired opt-in via SECUTOOLS_API_KEY.

Tests:
- 29+ new unit tests across router, secutools, delegation, dispatcher,
  api packages. go test -race -count=1 clean.
- tests/phase2-E-integration.sh: bash end-to-end against a Python
  stdlib mock HTTP server, exercising the dev-management scripts.

Forward-compat with watchdog (Phase 1 B1 already ignores
state=delegated_to_secutools) so delegated tasks aren't flagged stale.
This commit is contained in:
Ubuntu 2026-04-17 02:17:19 +00:00
parent 47ab86eef9
commit 3e20085204
18 changed files with 2819 additions and 22 deletions

View file

@ -16,16 +16,26 @@ import (
"gopkg.in/yaml.v3"
"forge.secuaas.ovh/olivier/claude-failover/internal/config"
"forge.secuaas.ovh/olivier/claude-failover/internal/delegation"
"forge.secuaas.ovh/olivier/claude-failover/internal/router"
"forge.secuaas.ovh/olivier/claude-failover/internal/secutools"
"forge.secuaas.ovh/olivier/claude-failover/internal/state"
"forge.secuaas.ovh/olivier/claude-failover/internal/tmux"
)
// TaskFrontmatter is the YAML header parsed from task .md files.
//
// Phase 2 chantier E added three new fields (PreferredAI, AllowDelegation,
// ComplexityHint). They are all optional; absence preserves Phase 1
// behaviour (Claude Code on a local ccl-auto session).
type TaskFrontmatter struct {
Title string `yaml:"title"`
Priority string `yaml:"priority"` // critical, high, default, low
Tags []string `yaml:"tags"`
NeedsClaude bool `yaml:"needs_claude_code"`
Title string `yaml:"title"`
Priority string `yaml:"priority"` // critical, high, default, low
Tags []string `yaml:"tags"`
NeedsClaude bool `yaml:"needs_claude_code"`
PreferredAI string `yaml:"preferred_ai"` // auto | claude-code | gpu | gemini | claude-api
AllowDelegation bool `yaml:"allow_delegation"` // default false → backward-compatible
ComplexityHint string `yaml:"complexity_hint"` // low | medium | high
}
// Dispatcher watches project inbox directories and assigns tasks to idle sessions.
@ -36,6 +46,12 @@ type Dispatcher struct {
doneChan <-chan string
projectsDir string
logger *log.Logger
// Phase 2 chantier E — delegation. When non-nil, tasks whose
// frontmatter routes them away from Claude Code (allow_delegation=true
// + preferred_ai!=claude-code) are submitted to secutools instead of
// being assigned to a tmux session.
delegation *delegation.Manager
}
// New creates a Dispatcher.
@ -55,6 +71,13 @@ func New(tc tmux.Client, s *state.State, cfg *config.Config, doneChan <-chan str
}
}
// WithDelegation enables delegation routing. Pass nil to disable (the
// Phase 1 default). Returns d for chaining.
func (d *Dispatcher) WithDelegation(m *delegation.Manager) *Dispatcher {
d.delegation = m
return d
}
// Run starts the dispatcher event loop until ctx is cancelled.
func (d *Dispatcher) Run(ctx context.Context) {
ticker := time.NewTicker(60 * time.Second)
@ -113,6 +136,11 @@ func (d *Dispatcher) fullScan() {
}
// dispatchProject assigns undispatched tasks in inboxDir to idle sessions.
//
// For each task we first ask the router whether the task should run on a
// local Claude Code session (the Phase 1 path) or be delegated to
// secutools (Phase 2 chantier E). Delegated tasks bypass the
// findFreeSession() check entirely — they don't need a tmux slot.
func (d *Dispatcher) dispatchProject(inboxDir string) {
entries, err := os.ReadDir(inboxDir)
if err != nil {
@ -121,15 +149,42 @@ func (d *Dispatcher) dispatchProject(inboxDir string) {
projectDir := filepath.Dir(filepath.Dir(inboxDir)) // inboxDir/.agent-queue/inbox → project
for _, e := range entries {
name := e.Name()
if !strings.HasSuffix(name, ".md") || strings.Contains(name, ".dispatched") {
if !strings.HasSuffix(name, ".md") ||
strings.Contains(name, ".dispatched") ||
strings.Contains(name, ".delegated") {
continue
}
taskPath := filepath.Join(inboxDir, name)
// Decide route based on frontmatter.
decision, body, err := d.routeTask(taskPath)
if err != nil {
d.logger.Printf("[dispatcher] routeTask %s: %v", taskPath, err)
continue
}
// Delegated path: submit to secutools, mark the task with .delegated.
if decision.Provider.IsDelegated() && d.delegation != nil {
if _, err := d.delegation.Submit(context.Background(), projectDir, taskPath,
body, decision.Provider, mapPriority(d.taskPriority(taskPath))); err != nil {
d.logger.Printf("[dispatcher] delegate %s: %v — falling back to Claude Code",
filepath.Base(taskPath), err)
// Fall through to Claude Code path on submit failure.
} else {
d.logger.Printf("[dispatcher] DELEGATED task=%s provider=%s reason=%s",
filepath.Base(taskPath), decision.Provider, decision.Reason)
// Original .md is left in place under inbox/ until the reaper
// finalises it. The .delegated marker prevents re-dispatch.
continue
}
}
// Claude Code path (Phase 1 behaviour).
session := d.findFreeSession()
if session == "" {
d.logger.Printf("[dispatcher] no free session for task in %s", inboxDir)
return
}
taskPath := filepath.Join(inboxDir, name)
if err := d.launchAgent(session, projectDir, taskPath); err != nil {
d.logger.Printf("[dispatcher] launchAgent error: %v", err)
continue
@ -140,6 +195,50 @@ func (d *Dispatcher) dispatchProject(inboxDir string) {
}
}
// routeTask reads taskPath, parses its frontmatter, and asks the router
// for a decision. Returns the decision plus the parsed body so callers
// don't have to read the file twice.
func (d *Dispatcher) routeTask(taskPath string) (router.Decision, string, error) {
content, err := os.ReadFile(taskPath)
if err != nil {
return router.Decision{}, "", fmt.Errorf("read task: %w", err)
}
fm, body := parseFrontmatter(content)
dec := router.Decide(router.Task{
PreferredAI: fm.PreferredAI,
AllowDelegation: fm.AllowDelegation,
NeedsClaudeCode: fm.NeedsClaude,
ComplexityHint: fm.ComplexityHint,
})
return dec, body, nil
}
// taskPriority returns the Priority field of a task's frontmatter without
// re-parsing the body. Defensive — used only for mapping into a secutools
// priority when delegating.
func (d *Dispatcher) taskPriority(taskPath string) string {
content, err := os.ReadFile(taskPath)
if err != nil {
return ""
}
fm, _ := parseFrontmatter(content)
return fm.Priority
}
// mapPriority converts a task priority string into a secutools Priority.
func mapPriority(p string) secutools.Priority {
switch strings.ToLower(strings.TrimSpace(p)) {
case "critical":
return secutools.PriorityCritical
case "high":
return secutools.PriorityHigh
case "low":
return secutools.PriorityLow
default:
return secutools.PriorityDefault
}
}
// findFreeSession returns the name of an idle, live, cooldown-free session
// from the autonomous pool. Dedicated sessions are intentionally NOT
// considered: those host the operator's manual interactive work. Routing a
@ -175,7 +274,13 @@ func (d *Dispatcher) isSessionFree(name string) bool {
return true
}
// assignNextTask scans all inboxes for work to give to a freshly-idled session.
// assignNextTask scans all inboxes for work to give to a freshly-idled
// session. Skips tasks that are already delegated to secutools (a
// .delegated marker exists alongside the .md) or already dispatched.
//
// Tasks whose router decision is "delegate" are also skipped here — they
// will be picked up by the next dispatchProject scan (which knows how to
// submit to secutools).
func (d *Dispatcher) assignNextTask(session string) {
for _, ds := range d.config.Pool.Dedicated {
inbox := filepath.Join(ds.Project, ".agent-queue", "inbox")
@ -183,11 +288,30 @@ func (d *Dispatcher) assignNextTask(session string) {
if err != nil {
continue
}
// Build a set of base names already delegated, so we can skip the
// associated .md without re-dispatching it locally.
delegated := make(map[string]bool)
for _, e := range entries {
if !strings.HasSuffix(e.Name(), ".md") || strings.Contains(e.Name(), ".dispatched") {
if strings.HasSuffix(e.Name(), ".md.delegated") {
delegated[strings.TrimSuffix(e.Name(), ".delegated")] = true
}
}
for _, e := range entries {
if !strings.HasSuffix(e.Name(), ".md") ||
strings.Contains(e.Name(), ".dispatched") ||
strings.Contains(e.Name(), ".delegated") {
continue
}
if delegated[e.Name()] {
continue
}
taskPath := filepath.Join(inbox, e.Name())
// Respect routing decisions: don't take a delegated task here.
if d.delegation != nil {
if dec, _, err := d.routeTask(taskPath); err == nil && dec.Provider.IsDelegated() {
continue
}
}
if err := d.launchAgent(session, ds.Project, taskPath); err == nil {
os.Rename(taskPath, taskPath+".dispatched")
return

View file

@ -0,0 +1,318 @@
package dispatcher
import (
"context"
"errors"
"log"
"os"
"path/filepath"
"sync"
"testing"
"time"
"forge.secuaas.ovh/olivier/claude-failover/internal/config"
"forge.secuaas.ovh/olivier/claude-failover/internal/delegation"
"forge.secuaas.ovh/olivier/claude-failover/internal/router"
"forge.secuaas.ovh/olivier/claude-failover/internal/secutools"
"forge.secuaas.ovh/olivier/claude-failover/internal/state"
)
// fakeSecutools is a minimal stub used for routing tests. It records every
// SubmitJob call so tests can assert on the number/contents of submissions.
type fakeSecutools struct {
mu sync.Mutex
submits []*secutools.JobRequest
statuses map[string]string
results map[string]*secutools.JobResult
nextID int
}
func newFakeSecutools() *fakeSecutools {
return &fakeSecutools{
statuses: make(map[string]string),
results: make(map[string]*secutools.JobResult),
}
}
func (f *fakeSecutools) SubmitJob(_ context.Context, req *secutools.JobRequest) (*secutools.JobResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.submits = append(f.submits, req)
f.nextID++
id := "job-" + itoa(f.nextID)
f.statuses[id] = "pending"
return &secutools.JobResponse{JobID: id, Status: "pending"}, nil
}
func (f *fakeSecutools) GetJob(_ context.Context, id string) (*secutools.JobStatus, error) {
f.mu.Lock()
defer f.mu.Unlock()
st, ok := f.statuses[id]
if !ok {
return nil, errors.New("unknown")
}
return &secutools.JobStatus{JobID: id, Status: st}, nil
}
func (f *fakeSecutools) WaitForResult(_ context.Context, id string, _ time.Duration) (*secutools.JobResult, error) {
f.mu.Lock()
defer f.mu.Unlock()
if r, ok := f.results[id]; ok {
return r, nil
}
return nil, errors.New("no result")
}
// (uses itoa from dispatcher.go)
func setupTaskFile(t *testing.T, frontmatter, body string) (projectDir, inbox, taskPath string) {
t.Helper()
projectDir = t.TempDir()
inbox = filepath.Join(projectDir, ".agent-queue", "inbox")
if err := os.MkdirAll(inbox, 0755); err != nil {
t.Fatal(err)
}
content := "---\n" + frontmatter + "\n---\n" + body
taskPath = filepath.Join(inbox, "task-routing.md")
if err := os.WriteFile(taskPath, []byte(content), 0644); err != nil {
t.Fatal(err)
}
return projectDir, inbox, taskPath
}
// TestRouteTask_ParsesNewFrontmatterFields ensures the dispatcher actually
// reads the new preferred_ai/allow_delegation/needs_claude_code fields.
func TestRouteTask_ParsesNewFrontmatterFields(t *testing.T) {
_, _, taskPath := setupTaskFile(t,
"title: Demo\npreferred_ai: gpu\nallow_delegation: true\ncomplexity_hint: low",
"Body of task.")
d := &Dispatcher{logger: log.Default()}
dec, body, err := d.routeTask(taskPath)
if err != nil {
t.Fatal(err)
}
if dec.Provider != router.ProviderGPU {
t.Errorf("expected ProviderGPU, got %v (%s)", dec.Provider, dec.Reason)
}
if body != "Body of task." {
t.Errorf("body parsed wrong: %q", body)
}
}
// TestDispatchProject_DelegatesGPUTask: a task with allow_delegation=true
// and preferred_ai=gpu must be sent to secutools and never reach a tmux
// session, even when one is available.
func TestDispatchProject_DelegatesGPUTask(t *testing.T) {
projectDir, inbox, taskPath := setupTaskFile(t,
"title: Delegated\npreferred_ai: gpu\nallow_delegation: true",
"Analyze something.")
tc := newMockTmux()
tc.sessions["pool-0"] = true
tc.paneOutput["pool-0"] = " "
s := state.New("")
s.SetIdle("pool-0")
fc := newFakeSecutools()
mgr := delegation.New(fc, time.Millisecond)
d := &Dispatcher{
tmux: tc,
state: s,
config: &config.Config{
Pool: config.PoolConfig{
Autonomous: config.AutonomousConfig{Prefix: "pool-", Max: 1},
},
},
logger: log.Default(),
delegation: mgr,
}
d.dispatchProject(inbox)
if len(fc.submits) != 1 {
t.Fatalf("expected 1 SubmitJob call, got %d", len(fc.submits))
}
if fc.submits[0].PreferredAI != "gpu" {
t.Errorf("expected preferred_ai=gpu, got %q", fc.submits[0].PreferredAI)
}
// .delegated marker is present, original .md kept (the reaper finalises later).
if _, err := os.Stat(taskPath + ".delegated"); err != nil {
t.Errorf("expected .delegated marker, got %v", err)
}
if _, err := os.Stat(taskPath); err != nil {
t.Errorf("expected original .md to remain until reaped, got %v", err)
}
// pool-0 must remain idle (we did NOT launch a Claude Code agent).
if st := s.GetSession("pool-0"); st == nil || st.State != "idle" {
t.Errorf("expected pool-0 to stay idle, got %v", st)
}
_ = projectDir
}
// TestDispatchProject_LegacyTaskKeepsClaudeCode: backward-compat. A task
// with no new fields (or allow_delegation=false implicit) MUST go to a
// local Claude Code session.
func TestDispatchProject_LegacyTaskKeepsClaudeCode(t *testing.T) {
_, inbox, taskPath := setupTaskFile(t,
"title: Legacy\npriority: default",
"Do classic work.")
tc := newMockTmux()
tc.sessions["pool-0"] = true
tc.paneOutput["pool-0"] = " "
s := state.New("")
s.SetIdle("pool-0")
fc := newFakeSecutools()
mgr := delegation.New(fc, time.Millisecond)
d := &Dispatcher{
tmux: tc,
state: s,
config: &config.Config{
Pool: config.PoolConfig{
Autonomous: config.AutonomousConfig{Prefix: "pool-", Max: 1},
},
},
logger: log.Default(),
delegation: mgr,
}
d.dispatchProject(inbox)
if len(fc.submits) != 0 {
t.Errorf("legacy task must NOT delegate, got %d submits", len(fc.submits))
}
// .dispatched marker present, session is working.
if _, err := os.Stat(taskPath + ".dispatched"); err != nil {
t.Errorf("expected .dispatched marker for Claude Code path, got %v", err)
}
if st := s.GetSession("pool-0"); st == nil || st.State != "working" {
t.Errorf("expected pool-0 working, got %v", st)
}
}
// TestDispatchProject_NeedsClaudeCodeBypassesDelegation: even with
// allow_delegation=true, needs_claude_code: true forces local execution.
func TestDispatchProject_NeedsClaudeCodeBypassesDelegation(t *testing.T) {
_, inbox, _ := setupTaskFile(t,
"title: Bypass\npreferred_ai: gpu\nallow_delegation: true\nneeds_claude_code: true",
"This needs a real Claude session.")
tc := newMockTmux()
tc.sessions["pool-0"] = true
tc.paneOutput["pool-0"] = " "
s := state.New("")
s.SetIdle("pool-0")
fc := newFakeSecutools()
mgr := delegation.New(fc, time.Millisecond)
d := &Dispatcher{
tmux: tc,
state: s,
config: &config.Config{
Pool: config.PoolConfig{
Autonomous: config.AutonomousConfig{Prefix: "pool-", Max: 1},
},
},
logger: log.Default(),
delegation: mgr,
}
d.dispatchProject(inbox)
if len(fc.submits) != 0 {
t.Errorf("needs_claude_code must skip delegation, got %d submits", len(fc.submits))
}
}
// TestEndToEnd_DelegationFlow: full happy path from inbox → submit →
// reaper → done/, with the dispatcher and delegation manager wired up.
func TestEndToEnd_DelegationFlow(t *testing.T) {
projectDir, inbox, taskPath := setupTaskFile(t,
"title: E2E\npreferred_ai: auto\nallow_delegation: true",
"Summarize this.")
tc := newMockTmux()
s := state.New("")
fc := newFakeSecutools()
mgr := delegation.New(fc, time.Millisecond)
d := &Dispatcher{
tmux: tc,
state: s,
config: &config.Config{Pool: config.PoolConfig{Autonomous: config.AutonomousConfig{Max: 0}}},
logger: log.Default(),
delegation: mgr,
}
// 1. Dispatch — submits to fakeSecutools.
d.dispatchProject(inbox)
if len(fc.submits) != 1 {
t.Fatalf("expected 1 submit, got %d", len(fc.submits))
}
jobID := "job-1"
// 2. Backend completes the job.
fc.mu.Lock()
fc.statuses[jobID] = "completed"
fc.results[jobID] = &secutools.JobResult{
JobID: jobID, Response: "Summary", Provider: "gpu", Model: "qwen", CostCAD: 0.001,
}
fc.mu.Unlock()
// 3. Reaper picks it up. mgr.Run drives reapOnce on each ticker tick;
// interval was set to 1ms in newDelegationManager so a 200ms
// context yields many cycles.
mgr.SetLogger(log.Default())
delegationReapNow(t, mgr, jobID)
donePath := filepath.Join(projectDir, ".agent-queue", "done", "task-routing.md")
body, err := os.ReadFile(donePath)
if err != nil {
t.Fatalf("missing done/ file: %v", err)
}
if !contains(string(body), "Summary") || !contains(string(body), "provider: gpu") {
t.Errorf("done body missing expected fields:\n%s", body)
}
// Marker and original .md gone.
if _, err := os.Stat(taskPath + ".delegated"); !os.IsNotExist(err) {
t.Error("delegated marker should be removed")
}
if _, err := os.Stat(taskPath); !os.IsNotExist(err) {
t.Error("original .md should be removed")
}
}
// delegationReapNow drives one reap cycle of mgr by calling its public
// API. We need this because reapOnce is unexported in the delegation
// package. The Run loop ticker is too slow for tests, so we expose a
// trivial ticker-equivalent: temporarily run with an immediate context.
func delegationReapNow(t *testing.T, mgr *delegation.Manager, _ string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()
// Run the manager briefly; the ticker fires after `interval` (1ms in
// these tests) so within 200ms we get many reap cycles.
mgr.Run(ctx)
}
func contains(s, sub string) bool {
if sub == "" {
return true
}
for i := 0; i+len(sub) <= len(s); i++ {
if s[i:i+len(sub)] == sub {
return true
}
}
return false
}