feat(quota): Phase 2.3 — QuotaMonitor (scraping pane tmux)
- internal/quota: SwitchRequest, poll() toutes les 30s - isQuotaExhausted: 5 patterns (hit limit, rate limit, quota exceeded, etc.) - extractResetTime: regex pour "resets 8pm / resets at 11:30pm / resets in N min" - Seuils: >=2 sessions pool OU >=1 session dedicated → SwitchRequest channel(1) - 5 tests: patterns, reset time, trigger 2 pool, trigger 1 dedicated, no-trigger Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Ubuntu
parent
0a7e5efcfd
commit
133165b432
2 changed files with 342 additions and 0 deletions
190
internal/quota/monitor.go
Normal file
190
internal/quota/monitor.go
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
// Package quota monitors Claude Code sessions for quota exhaustion and triggers
|
||||
// account switches when thresholds are crossed.
|
||||
package quota
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"forge.secuaas.ovh/olivier/claude-failover/internal/config"
|
||||
"forge.secuaas.ovh/olivier/claude-failover/internal/state"
|
||||
"forge.secuaas.ovh/olivier/claude-failover/internal/tmux"
|
||||
)
|
||||
|
||||
// SwitchRequest is emitted when quota exhaustion is detected, requesting
|
||||
// the AccountSwitcher to activate a different account.
|
||||
type SwitchRequest struct {
|
||||
From string // current active account name
|
||||
To string // desired account name (empty = auto-select)
|
||||
ResetTime string // human-readable reset time extracted from the pane
|
||||
}
|
||||
|
||||
// quotaPatterns are substrings that indicate quota exhaustion in a pane.
|
||||
var quotaPatterns = []string{
|
||||
"you've hit your limit",
|
||||
"rate limit",
|
||||
"quota exceeded",
|
||||
"usage limit reached",
|
||||
"claude pro usage",
|
||||
"too many requests",
|
||||
}
|
||||
|
||||
// resetTimeRe extracts reset times like "resets 8pm", "resets at 11:30pm",
|
||||
// "resets in 45 minutes".
|
||||
var resetTimeRe = regexp.MustCompile(
|
||||
`(?i)resets?\s+(?:at\s+)?([0-9]+(?::[0-9]+)?\s*[ap]m|in\s+[0-9]+\s+(?:minute|hour)s?)`,
|
||||
)
|
||||
|
||||
// Monitor polls tmux panes for quota exhaustion messages.
|
||||
type Monitor struct {
|
||||
tmux tmux.Client
|
||||
state *state.State
|
||||
config *config.Config
|
||||
switchCh chan SwitchRequest
|
||||
interval time.Duration
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// New creates a Monitor with defaults from cfg.
|
||||
func New(tc tmux.Client, s *state.State, cfg *config.Config) *Monitor {
|
||||
interval := cfg.Quota.PollInterval.Duration
|
||||
if interval == 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
return &Monitor{
|
||||
tmux: tc,
|
||||
state: s,
|
||||
config: cfg,
|
||||
switchCh: make(chan SwitchRequest, 1),
|
||||
interval: interval,
|
||||
logger: log.Default(),
|
||||
}
|
||||
}
|
||||
|
||||
// SwitchChan returns the channel on which SwitchRequests are sent.
|
||||
func (m *Monitor) SwitchChan() <-chan SwitchRequest {
|
||||
return m.switchCh
|
||||
}
|
||||
|
||||
// Run starts the quota monitor loop until ctx is cancelled.
|
||||
func (m *Monitor) Run(ctx context.Context) {
|
||||
ticker := time.NewTicker(m.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.poll()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// poll checks all sessions for quota exhaustion once.
|
||||
func (m *Monitor) poll() {
|
||||
if m.state.ActiveAccount() != "" && m.isQuotaPaused() {
|
||||
return
|
||||
}
|
||||
|
||||
blockedPool := 0
|
||||
blockedInteractive := 0
|
||||
var resetTime string
|
||||
|
||||
prefix := m.config.Pool.Autonomous.Prefix
|
||||
if prefix == "" {
|
||||
prefix = "ccl-auto-"
|
||||
}
|
||||
for i := 0; i < m.config.Pool.Autonomous.Max; i++ {
|
||||
name := sessionName(prefix, i)
|
||||
if !m.tmux.HasSession(name) {
|
||||
continue
|
||||
}
|
||||
// Only capture 3 lines — avoids false positives on stale history.
|
||||
tail, err := m.tmux.CapturePaneTail(name, 3)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isQuotaExhausted(tail) {
|
||||
blockedPool++
|
||||
if rt := extractResetTime(tail); rt != "" {
|
||||
resetTime = rt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, ds := range m.config.Pool.Dedicated {
|
||||
if !m.tmux.HasSession(ds.Name) {
|
||||
continue
|
||||
}
|
||||
tail, err := m.tmux.CapturePaneTail(ds.Name, 3)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isQuotaExhausted(tail) {
|
||||
blockedInteractive++
|
||||
if rt := extractResetTime(tail); rt != "" {
|
||||
resetTime = rt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if blockedPool >= 2 || blockedInteractive >= 1 {
|
||||
req := SwitchRequest{
|
||||
From: m.state.ActiveAccount(),
|
||||
ResetTime: resetTime,
|
||||
}
|
||||
select {
|
||||
case m.switchCh <- req:
|
||||
m.logger.Printf("[quota] SwapRequested: from=%s pool=%d interactive=%d reset=%q",
|
||||
req.From, blockedPool, blockedInteractive, resetTime)
|
||||
default:
|
||||
// Swap already pending — do not queue another.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// isQuotaPaused checks whether a swap is already in progress.
|
||||
func (m *Monitor) isQuotaPaused() bool {
|
||||
// Lightweight proxy: if switch channel is full, a swap is pending.
|
||||
return len(m.switchCh) > 0
|
||||
}
|
||||
|
||||
// isQuotaExhausted returns true if the pane content indicates quota exhaustion.
|
||||
func isQuotaExhausted(paneContent string) bool {
|
||||
lower := strings.ToLower(paneContent)
|
||||
for _, p := range quotaPatterns {
|
||||
if strings.Contains(lower, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// extractResetTime parses a reset time string from pane content.
|
||||
// Returns "" if none found.
|
||||
func extractResetTime(content string) string {
|
||||
m := resetTimeRe.FindStringSubmatch(content)
|
||||
if len(m) >= 2 {
|
||||
return strings.TrimSpace(m[1])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sessionName(prefix string, i int) string {
|
||||
return prefix + itoa(i)
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
b := make([]byte, 0, 10)
|
||||
for n > 0 {
|
||||
b = append([]byte{byte('0' + n%10)}, b...)
|
||||
n /= 10
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
152
internal/quota/monitor_test.go
Normal file
152
internal/quota/monitor_test.go
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
package quota
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"forge.secuaas.ovh/olivier/claude-failover/internal/config"
|
||||
"forge.secuaas.ovh/olivier/claude-failover/internal/state"
|
||||
)
|
||||
|
||||
// mockTmux for quota tests.
|
||||
type mockTmux struct {
|
||||
sessions map[string]bool
|
||||
paneOutput map[string]string
|
||||
}
|
||||
|
||||
func newMockTmux() *mockTmux {
|
||||
return &mockTmux{
|
||||
sessions: make(map[string]bool),
|
||||
paneOutput: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockTmux) HasSession(name string) bool { return m.sessions[name] }
|
||||
func (m *mockTmux) CreateSession(name, _ string) error { m.sessions[name] = true; return nil }
|
||||
func (m *mockTmux) KillSession(_ string) error { return nil }
|
||||
func (m *mockTmux) SendKeys(_, _ string) error { return nil }
|
||||
func (m *mockTmux) CapturePaneTail(session string, _ int) (string, error) {
|
||||
return m.paneOutput[session], nil
|
||||
}
|
||||
|
||||
// TestIsQuotaExhausted verifies pattern matching on pane output.
|
||||
func TestIsQuotaExhausted(t *testing.T) {
|
||||
cases := []struct {
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{"You've hit your limit for Claude Pro.", true},
|
||||
{"rate limit exceeded", true},
|
||||
{"quota exceeded for this period", true},
|
||||
{"Usage limit reached", true},
|
||||
{"Too many requests", true},
|
||||
{"Some normal output ❯", false},
|
||||
{"❯ ", false},
|
||||
{"still running 5s · ", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := isQuotaExhausted(c.input); got != c.want {
|
||||
t.Errorf("isQuotaExhausted(%q) = %v, want %v", c.input, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractResetTime parses various reset time formats.
|
||||
func TestExtractResetTime(t *testing.T) {
|
||||
cases := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"Usage resets 8pm", "8pm"},
|
||||
{"Your quota resets at 11:30pm", "11:30pm"},
|
||||
{"resets in 45 minutes", "in 45 minutes"},
|
||||
{"resets in 2 hours", "in 2 hours"},
|
||||
{"no reset info here", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := extractResetTime(c.input); got != c.want {
|
||||
t.Errorf("extractResetTime(%q) = %q, want %q", c.input, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPollTriggersSwitchOnTwoBlockedPool verifies swap trigger for >=2 blocked pool sessions.
|
||||
func TestPollTriggersSwitchOnTwoBlockedPool(t *testing.T) {
|
||||
tc := newMockTmux()
|
||||
tc.sessions["ccl-auto-0"] = true
|
||||
tc.sessions["ccl-auto-1"] = true
|
||||
tc.paneOutput["ccl-auto-0"] = "You've hit your limit for Claude Pro."
|
||||
tc.paneOutput["ccl-auto-1"] = "rate limit exceeded"
|
||||
|
||||
s := state.New("")
|
||||
s.SetActiveAccount("compte1")
|
||||
|
||||
cfg := &config.Config{
|
||||
Pool: config.PoolConfig{
|
||||
Autonomous: config.AutonomousConfig{Prefix: "ccl-auto-", Max: 2},
|
||||
},
|
||||
}
|
||||
m := New(tc, s, cfg)
|
||||
m.poll()
|
||||
|
||||
select {
|
||||
case req := <-m.switchCh:
|
||||
if req.From != "compte1" {
|
||||
t.Errorf("expected From=compte1, got %q", req.From)
|
||||
}
|
||||
default:
|
||||
t.Fatal("expected SwitchRequest on channel")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPollTriggersSwitchOnOneBlockedInteractive verifies swap trigger for >=1 dedicated session.
|
||||
func TestPollTriggersSwitchOnOneBlockedInteractive(t *testing.T) {
|
||||
tc := newMockTmux()
|
||||
tc.sessions["my-session"] = true
|
||||
tc.paneOutput["my-session"] = "quota exceeded"
|
||||
|
||||
s := state.New("")
|
||||
s.SetActiveAccount("compte1")
|
||||
|
||||
cfg := &config.Config{
|
||||
Pool: config.PoolConfig{
|
||||
Dedicated: []config.DedicatedSession{{Name: "my-session"}},
|
||||
Autonomous: config.AutonomousConfig{Max: 0},
|
||||
},
|
||||
}
|
||||
m := New(tc, s, cfg)
|
||||
m.poll()
|
||||
|
||||
select {
|
||||
case req := <-m.switchCh:
|
||||
if req.From != "compte1" {
|
||||
t.Errorf("expected From=compte1, got %q", req.From)
|
||||
}
|
||||
default:
|
||||
t.Fatal("expected SwitchRequest on channel")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPollNoTriggerWhenBelowThreshold verifies no swap for a single blocked pool session.
|
||||
func TestPollNoTriggerWhenBelowThreshold(t *testing.T) {
|
||||
tc := newMockTmux()
|
||||
tc.sessions["ccl-auto-0"] = true
|
||||
tc.sessions["ccl-auto-1"] = true
|
||||
tc.paneOutput["ccl-auto-0"] = "rate limit exceeded"
|
||||
tc.paneOutput["ccl-auto-1"] = "❯ " // fine
|
||||
|
||||
s := state.New("")
|
||||
cfg := &config.Config{
|
||||
Pool: config.PoolConfig{
|
||||
Autonomous: config.AutonomousConfig{Prefix: "ccl-auto-", Max: 2},
|
||||
},
|
||||
}
|
||||
m := New(tc, s, cfg)
|
||||
m.poll()
|
||||
|
||||
select {
|
||||
case req := <-m.switchCh:
|
||||
t.Errorf("unexpected SwitchRequest: %+v", req)
|
||||
default:
|
||||
// Correct: only 1 blocked pool session, threshold is 2.
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue