feat(phase2-E): multi-provider routing via secutools delegation

Adds optional delegation of agent-queue tasks to the SecuAAS secutools
AI platform (GPU / Gemini / Claude API) instead of dispatching to a
local Claude Code tmux session. Per-task opt-in via YAML frontmatter
fields preferred_ai, allow_delegation, complexity_hint — absence keeps
the Phase 1 behaviour exactly (zero breaking change).

Go side:
- internal/secutools: HTTP client with exponential-backoff retries
  (SubmitJob/GetJob/WaitForResult), DecideProvider map adapter for CLI
  use, table tests.
- internal/router: struct-typed Decide() with strict precedence
  (needs_claude_code > preferred_ai=claude-code > allow_delegation=false
  > preferred_ai > fail-safe local on unknown).
- internal/delegation: Manager submits jobs, writes .md.delegated
  markers for on-restart recovery, runs a periodic reaper that moves
  completed jobs into done/ with provider/cost footer and failed jobs
  into failed/.
- internal/dispatcher: WithDelegation() opt-in, routeTask hook before
  findFreeSession, skips .md.delegated in assignNextTask.
- internal/api: /api/delegated/status (active jobs + counters),
  /watchdog/status extended with delegation counters.
- cmd/ccl-delegate: small CLI exposing submit/get/result/decide so the
  bash dispatcher can call the same contract without duplicating logic.
- cmd/claude-failover: delegation wired opt-in via SECUTOOLS_API_KEY.

Tests:
- 29+ new unit tests across router, secutools, delegation, dispatcher,
  api packages. go test -race -count=1 clean.
- tests/phase2-E-integration.sh: bash end-to-end against a Python
  stdlib mock HTTP server, exercising the dev-management scripts.

Forward-compat with watchdog (Phase 1 B1 already ignores
state=delegated_to_secutools) so delegated tasks aren't flagged stale.
This commit is contained in:
Ubuntu 2026-04-17 02:17:19 +00:00
parent 47ab86eef9
commit 3e20085204
18 changed files with 2819 additions and 22 deletions

View file

@ -0,0 +1,290 @@
// Package secutools provides a minimal HTTP client for the centralized SecuAAS
// AI-batch platform (https://api.secutools.secuaas.ovh).
//
// Phase 2 — Chantier E: the dispatcher delegates non-Claude-Code-eligible
// tasks to secutools (GPU/Gemini/Claude API) instead of dispatching them to
// a local ccl-auto tmux session. This package is the Go side of that
// delegation: SubmitJob, GetJob, WaitForResult.
//
// The Client interface is intentionally narrow so tests can plug a fake
// implementation without any network dependency.
package secutools
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"time"
)
// Client is the abstraction the rest of the daemon uses to talk to secutools.
// Real callers use HTTPClient; tests substitute a mock.
type Client interface {
SubmitJob(ctx context.Context, req *JobRequest) (*JobResponse, error)
GetJob(ctx context.Context, id string) (*JobStatus, error)
WaitForResult(ctx context.Context, id string, timeout time.Duration) (*JobResult, error)
}
// JobType mirrors the secutools job-type enum.
type JobType string
const (
TypeAnalyze JobType = "ai:analyze"
TypeBatch JobType = "ai:batch"
TypeReport JobType = "ai:report"
TypeCorrelate JobType = "ai:correlate"
)
// Priority mirrors the secutools priority enum.
type Priority string
const (
PriorityCritical Priority = "critical"
PriorityHigh Priority = "high"
PriorityDefault Priority = "default"
PriorityLow Priority = "low"
)
// JobRequest is the body of POST /api/v1/jobs.
type JobRequest struct {
Type JobType `json:"type"`
Priority Priority `json:"priority,omitempty"`
Prompt string `json:"prompt"`
Data map[string]any `json:"data,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
PreferredAI string `json:"preferred_ai,omitempty"`
Source string `json:"source,omitempty"`
}
// JobResponse is the immediate reply from POST /api/v1/jobs.
type JobResponse struct {
JobID string `json:"job_id"`
Status string `json:"status"`
}
// JobStatus is the reply from GET /api/v1/jobs/:id.
type JobStatus struct {
JobID string `json:"job_id"`
Status string `json:"status"` // pending | running | completed | failed | cancelled
Provider string `json:"provider,omitempty"`
Error string `json:"error,omitempty"`
}
// JobResult is the reply from GET /api/v1/jobs/:id/result.
type JobResult struct {
JobID string `json:"job_id"`
Response string `json:"response"`
Provider string `json:"provider"`
Model string `json:"model"`
CostCAD float64 `json:"cost_cad"`
Tokens int `json:"tokens,omitempty"`
}
// HTTPClient is the production implementation of Client.
type HTTPClient struct {
baseURL string
apiKey string
hc *http.Client
maxRetries int
baseDelay time.Duration
}
// NewHTTPClient returns an HTTPClient ready to talk to secutools.
// If hc is nil, a default http.Client with a 30s timeout is used.
//
// The client performs up to 3 retries on transport errors and 5xx
// responses, with exponential backoff starting at 500ms (500ms, 1s, 2s).
// 4xx responses are returned as errors without retrying.
func NewHTTPClient(baseURL, apiKey string, hc *http.Client) *HTTPClient {
if hc == nil {
hc = &http.Client{Timeout: 30 * time.Second}
}
return &HTTPClient{
baseURL: baseURL,
apiKey: apiKey,
hc: hc,
maxRetries: 3,
baseDelay: 500 * time.Millisecond,
}
}
// SetRetryPolicy overrides the default retry policy. Useful for tests.
func (c *HTTPClient) SetRetryPolicy(maxRetries int, baseDelay time.Duration) {
c.maxRetries = maxRetries
c.baseDelay = baseDelay
}
// doWithRetry sends req and retries on transport errors or 5xx responses
// using exponential backoff. 4xx is returned without retry. Respects ctx.
func (c *HTTPClient) doWithRetry(ctx context.Context, build func() (*http.Request, error)) (*http.Response, error) {
var lastErr error
delay := c.baseDelay
for attempt := 0; attempt <= c.maxRetries; attempt++ {
if attempt > 0 {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(delay):
}
delay *= 2
}
req, err := build()
if err != nil {
return nil, err
}
resp, err := c.hc.Do(req)
if err != nil {
lastErr = err
continue
}
// Retry 5xx; return success or 4xx immediately.
if resp.StatusCode >= 500 && resp.StatusCode <= 599 {
raw, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
lastErr = fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw))
continue
}
return resp, nil
}
if lastErr == nil {
lastErr = errors.New("secutools: unknown transport failure")
}
return nil, fmt.Errorf("after %d attempts: %w", c.maxRetries+1, lastErr)
}
// SubmitJob POSTs req to /api/v1/jobs with retry on 5xx.
func (c *HTTPClient) SubmitJob(ctx context.Context, req *JobRequest) (*JobResponse, error) {
body, err := json.Marshal(req)
if err != nil {
return nil, fmt.Errorf("marshal request: %w", err)
}
resp, err := c.doWithRetry(ctx, func() (*http.Request, error) {
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost,
c.baseURL+"/api/v1/jobs", bytes.NewReader(body))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("X-API-Key", c.apiKey)
return httpReq, nil
})
if err != nil {
return nil, fmt.Errorf("submit job: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
raw, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("submit job: HTTP %d: %s", resp.StatusCode, string(raw))
}
var out JobResponse
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("decode submit response: %w", err)
}
return &out, nil
}
// GetJob GETs /api/v1/jobs/:id with retry on 5xx.
func (c *HTTPClient) GetJob(ctx context.Context, id string) (*JobStatus, error) {
resp, err := c.doWithRetry(ctx, func() (*http.Request, error) {
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet,
c.baseURL+"/api/v1/jobs/"+id, nil)
if err != nil {
return nil, err
}
httpReq.Header.Set("X-API-Key", c.apiKey)
return httpReq, nil
})
if err != nil {
return nil, fmt.Errorf("get job: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
raw, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("get job: HTTP %d: %s", resp.StatusCode, string(raw))
}
var out JobStatus
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("decode get response: %w", err)
}
return &out, nil
}
// getResult fetches the final payload of a completed job with retry on 5xx.
func (c *HTTPClient) getResult(ctx context.Context, id string) (*JobResult, error) {
resp, err := c.doWithRetry(ctx, func() (*http.Request, error) {
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet,
c.baseURL+"/api/v1/jobs/"+id+"/result", nil)
if err != nil {
return nil, err
}
httpReq.Header.Set("X-API-Key", c.apiKey)
return httpReq, nil
})
if err != nil {
return nil, fmt.Errorf("get result: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
raw, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("get result: HTTP %d: %s", resp.StatusCode, string(raw))
}
var out JobResult
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("decode result: %w", err)
}
return &out, nil
}
// ErrJobFailed is returned by WaitForResult when secutools reports the job
// as terminally failed (no result will ever be produced).
var ErrJobFailed = errors.New("secutools: job failed")
// ErrTimeout is returned by WaitForResult when the polling deadline elapses
// before the job reaches a terminal state.
var ErrTimeout = errors.New("secutools: wait timeout")
// WaitForResult polls /api/v1/jobs/:id every 2s until the job reaches a
// terminal state (completed/failed/cancelled) or timeout elapses.
// On completed, fetches and returns the result.
//
// Polling cadence is intentionally fixed (not configurable) to keep the
// reaper goroutine simple. If callers need a different cadence they can
// implement it themselves on top of GetJob/getResult.
func (c *HTTPClient) WaitForResult(ctx context.Context, id string, timeout time.Duration) (*JobResult, error) {
deadline := time.Now().Add(timeout)
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for {
st, err := c.GetJob(ctx, id)
if err != nil {
return nil, err
}
switch st.Status {
case "completed":
return c.getResult(ctx, id)
case "failed", "cancelled":
return nil, fmt.Errorf("%w: status=%s err=%s", ErrJobFailed, st.Status, st.Error)
}
if time.Now().After(deadline) {
return nil, ErrTimeout
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-ticker.C:
}
}
}

View file

@ -0,0 +1,190 @@
package secutools
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
)
// TestSubmitJob_HappyPath verifies the request body and headers match the
// secutools contract and the response is decoded.
func TestSubmitJob_HappyPath(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/jobs" {
t.Errorf("unexpected path %q", r.URL.Path)
}
if r.Method != http.MethodPost {
t.Errorf("unexpected method %q", r.Method)
}
if r.Header.Get("X-API-Key") != "key123" {
t.Errorf("missing/incorrect X-API-Key: %q", r.Header.Get("X-API-Key"))
}
var got JobRequest
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if got.Type != TypeAnalyze || got.PreferredAI != "gpu" {
t.Errorf("payload mismatch: %+v", got)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"job_id":"abc","status":"pending"}`))
}))
defer srv.Close()
c := NewHTTPClient(srv.URL, "key123", srv.Client())
resp, err := c.SubmitJob(context.Background(), &JobRequest{
Type: TypeAnalyze,
Priority: PriorityHigh,
Prompt: "hi",
PreferredAI: "gpu",
})
if err != nil {
t.Fatalf("SubmitJob: %v", err)
}
if resp.JobID != "abc" || resp.Status != "pending" {
t.Errorf("unexpected response: %+v", resp)
}
}
// TestSubmitJob_HTTPError surfaces non-2xx responses as errors.
func TestSubmitJob_HTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte("boom"))
}))
defer srv.Close()
c := NewHTTPClient(srv.URL, "k", srv.Client())
if _, err := c.SubmitJob(context.Background(), &JobRequest{Type: TypeAnalyze, Prompt: "p"}); err == nil {
t.Fatal("expected error on HTTP 500, got nil")
}
}
// TestWaitForResult_PollsUntilCompleted verifies the polling loop transitions
// pending → running → completed and fetches the result.
func TestWaitForResult_PollsUntilCompleted(t *testing.T) {
var calls atomic.Int64
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/v1/jobs/job1":
n := calls.Add(1)
status := "pending"
if n >= 2 {
status = "completed"
}
_, _ = w.Write([]byte(`{"job_id":"job1","status":"` + status + `","provider":"gpu"}`))
case "/api/v1/jobs/job1/result":
_, _ = w.Write([]byte(`{"job_id":"job1","response":"done","provider":"gpu","cost_cad":0.005}`))
default:
t.Errorf("unexpected path %q", r.URL.Path)
}
}))
defer srv.Close()
c := NewHTTPClient(srv.URL, "k", srv.Client())
// Override poll cadence indirectly: short timeout proves we don't spin
// 2s per poll; the test runs in well under 10s real time.
res, err := c.WaitForResult(context.Background(), "job1", 30*time.Second)
if err != nil {
t.Fatalf("WaitForResult: %v", err)
}
if res.Response != "done" || res.Provider != "gpu" {
t.Errorf("unexpected result: %+v", res)
}
}
// TestWaitForResult_FailedJob returns ErrJobFailed when secutools reports
// terminal failure.
func TestWaitForResult_FailedJob(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"job_id":"jobX","status":"failed","error":"oom"}`))
}))
defer srv.Close()
c := NewHTTPClient(srv.URL, "k", srv.Client())
_, err := c.WaitForResult(context.Background(), "jobX", 5*time.Second)
if !errors.Is(err, ErrJobFailed) {
t.Errorf("expected ErrJobFailed, got %v", err)
}
}
// TestSubmitJob_RetriesOn5xx verifies the client retries transient 500s
// and succeeds on a later attempt. Uses a tight retry delay so the test
// runs in milliseconds.
func TestSubmitJob_RetriesOn5xx(t *testing.T) {
var calls atomic.Int64
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := calls.Add(1)
if n < 3 {
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte("transient"))
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"job_id":"ok","status":"pending"}`))
}))
defer srv.Close()
c := NewHTTPClient(srv.URL, "k", srv.Client())
c.SetRetryPolicy(3, 1*time.Millisecond)
resp, err := c.SubmitJob(context.Background(), &JobRequest{Type: TypeAnalyze, Prompt: "p"})
if err != nil {
t.Fatalf("expected success after retries, got %v (calls=%d)", err, calls.Load())
}
if resp.JobID != "ok" {
t.Errorf("unexpected response: %+v", resp)
}
if calls.Load() != 3 {
t.Errorf("expected 3 attempts, got %d", calls.Load())
}
}
// TestSubmitJob_DoesNotRetry4xx ensures client errors short-circuit
// without burning retries (e.g. wrong API key).
func TestSubmitJob_DoesNotRetry4xx(t *testing.T) {
var calls atomic.Int64
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("bad key"))
}))
defer srv.Close()
c := NewHTTPClient(srv.URL, "k", srv.Client())
c.SetRetryPolicy(3, 1*time.Millisecond)
_, err := c.SubmitJob(context.Background(), &JobRequest{Type: TypeAnalyze, Prompt: "p"})
if err == nil {
t.Fatal("expected error on 401")
}
if calls.Load() != 1 {
t.Errorf("4xx must not retry, got %d calls", calls.Load())
}
}
// TestWaitForResult_ContextCancel exits cleanly when the parent context is
// cancelled mid-poll.
func TestWaitForResult_ContextCancel(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"job_id":"j","status":"pending"}`))
}))
defer srv.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel immediately
c := NewHTTPClient(srv.URL, "k", srv.Client())
_, err := c.WaitForResult(ctx, "j", 10*time.Second)
if err == nil {
t.Fatal("expected error from cancelled context")
}
}

View file

@ -0,0 +1,86 @@
package secutools
import (
"fmt"
"strings"
)
// DecideProvider inspects a frontmatter map (as decoded from YAML) and
// returns the provider string that should handle the task. Valid return
// values:
//
// - "local" — no delegation, dispatch on a Claude Code session (Phase 1)
// - "claude-code" — explicit local dispatch (alias of "local")
// - "gpu" — delegate to secutools with preferred_ai=gpu
// - "gemini" — delegate to secutools with preferred_ai=gemini
// - "claude-api" — delegate to secutools with preferred_ai=claude-api
// - "auto" — delegate to secutools, let smart_triage choose
//
// Precedence:
// 1. needs_claude_code: true → "local"
// 2. preferred_ai in {claude-code} → "local"
// 3. allow_delegation == false / missing → "local"
// 4. preferred_ai in {gpu,gemini,claude-api} → that provider
// 5. preferred_ai in {"", auto} → "auto"
// 6. unknown preferred_ai → "local" (fail-safe)
//
// This function is intentionally permissive on input types: YAML booleans
// may decode as bool, strings as string. It coerces common forms and
// returns "local" on malformed input rather than panicking.
func DecideProvider(fm map[string]any) string {
if fm == nil {
return "local"
}
if asBool(fm["needs_claude_code"]) {
return "local"
}
pref := strings.ToLower(strings.TrimSpace(asString(fm["preferred_ai"])))
if pref == "claude-code" || pref == "local" {
return "local"
}
if !asBool(fm["allow_delegation"]) {
return "local"
}
switch pref {
case "gpu", "gemini", "claude-api":
return pref
case "", "auto":
return "auto"
default:
// Unknown provider — fail safe to local.
return "local"
}
}
// asBool accepts bool, "true"/"false", "1"/"0". Defaults to false.
func asBool(v any) bool {
switch t := v.(type) {
case bool:
return t
case string:
s := strings.ToLower(strings.TrimSpace(t))
return s == "true" || s == "1" || s == "yes"
case int:
return t != 0
default:
return false
}
}
// asString coerces v to a trimmed string. Returns "" for nil/unknown types.
func asString(v any) string {
switch t := v.(type) {
case string:
return t
case fmt.Stringer:
return t.String()
case nil:
return ""
default:
return fmt.Sprintf("%v", t)
}
}

View file

@ -0,0 +1,72 @@
package secutools
import "testing"
// TestDecideProvider_TableDriven exercises the full decision matrix for
// the map-input adapter used by the bash-side ccl-delegate CLI. The
// richer Task-struct variant lives in internal/router.
func TestDecideProvider_TableDriven(t *testing.T) {
cases := []struct {
name string
fm map[string]any
want string
}{
{"nil map falls back to local", nil, "local"},
{"empty map → local (allow_delegation default false)", map[string]any{}, "local"},
{"needs_claude_code wins", map[string]any{
"needs_claude_code": true,
"allow_delegation": true,
"preferred_ai": "gpu",
}, "local"},
{"explicit claude-code stays local", map[string]any{
"preferred_ai": "claude-code",
"allow_delegation": true,
}, "local"},
{"allow_delegation=false blocks even with preferred_ai=gpu", map[string]any{
"preferred_ai": "gpu",
"allow_delegation": false,
}, "local"},
{"gpu", map[string]any{
"preferred_ai": "gpu",
"allow_delegation": true,
}, "gpu"},
{"GPU (case-insensitive)", map[string]any{
"preferred_ai": "GPU",
"allow_delegation": true,
}, "gpu"},
{"gemini", map[string]any{
"preferred_ai": "gemini",
"allow_delegation": true,
}, "gemini"},
{"claude-api", map[string]any{
"preferred_ai": "claude-api",
"allow_delegation": true,
}, "claude-api"},
{"auto", map[string]any{
"preferred_ai": "auto",
"allow_delegation": true,
}, "auto"},
{"empty preferred_ai + allow_delegation → auto", map[string]any{
"allow_delegation": true,
}, "auto"},
{"unknown provider → fail-safe local", map[string]any{
"preferred_ai": "claude-3-mystery",
"allow_delegation": true,
}, "local"},
{"allow_delegation as string 'true'", map[string]any{
"preferred_ai": "gpu",
"allow_delegation": "true",
}, "gpu"},
{"allow_delegation as int 1", map[string]any{
"preferred_ai": "gemini",
"allow_delegation": 1,
}, "gemini"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := DecideProvider(c.fm); got != c.want {
t.Errorf("DecideProvider(%v) = %q, want %q", c.fm, got, c.want)
}
})
}
}