feat(quota): Phase 2.3 — QuotaMonitor (scraping pane tmux)
- internal/quota: SwitchRequest, poll() toutes les 30s - isQuotaExhausted: 5 patterns (hit limit, rate limit, quota exceeded, etc.) - extractResetTime: regex pour "resets 8pm / resets at 11:30pm / resets in N min" - Seuils: >=2 sessions pool OU >=1 session dedicated → SwitchRequest channel(1) - 5 tests: patterns, reset time, trigger 2 pool, trigger 1 dedicated, no-trigger Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
0a7e5efcfd
commit
133165b432
2 changed files with 342 additions and 0 deletions
190
internal/quota/monitor.go
Normal file
190
internal/quota/monitor.go
Normal file
|
|
@ -0,0 +1,190 @@
|
||||||
|
// Package quota monitors Claude Code sessions for quota exhaustion and triggers
|
||||||
|
// account switches when thresholds are crossed.
|
||||||
|
package quota
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"forge.secuaas.ovh/olivier/claude-failover/internal/config"
|
||||||
|
"forge.secuaas.ovh/olivier/claude-failover/internal/state"
|
||||||
|
"forge.secuaas.ovh/olivier/claude-failover/internal/tmux"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SwitchRequest is emitted when quota exhaustion is detected, requesting
|
||||||
|
// the AccountSwitcher to activate a different account.
|
||||||
|
type SwitchRequest struct {
|
||||||
|
From string // current active account name
|
||||||
|
To string // desired account name (empty = auto-select)
|
||||||
|
ResetTime string // human-readable reset time extracted from the pane
|
||||||
|
}
|
||||||
|
|
||||||
|
// quotaPatterns are substrings that indicate quota exhaustion in a pane.
|
||||||
|
var quotaPatterns = []string{
|
||||||
|
"you've hit your limit",
|
||||||
|
"rate limit",
|
||||||
|
"quota exceeded",
|
||||||
|
"usage limit reached",
|
||||||
|
"claude pro usage",
|
||||||
|
"too many requests",
|
||||||
|
}
|
||||||
|
|
||||||
|
// resetTimeRe extracts reset times like "resets 8pm", "resets at 11:30pm",
|
||||||
|
// "resets in 45 minutes".
|
||||||
|
var resetTimeRe = regexp.MustCompile(
|
||||||
|
`(?i)resets?\s+(?:at\s+)?([0-9]+(?::[0-9]+)?\s*[ap]m|in\s+[0-9]+\s+(?:minute|hour)s?)`,
|
||||||
|
)
|
||||||
|
|
||||||
|
// Monitor polls tmux panes for quota exhaustion messages.
|
||||||
|
type Monitor struct {
|
||||||
|
tmux tmux.Client
|
||||||
|
state *state.State
|
||||||
|
config *config.Config
|
||||||
|
switchCh chan SwitchRequest
|
||||||
|
interval time.Duration
|
||||||
|
logger *log.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// New creates a Monitor with defaults from cfg.
|
||||||
|
func New(tc tmux.Client, s *state.State, cfg *config.Config) *Monitor {
|
||||||
|
interval := cfg.Quota.PollInterval.Duration
|
||||||
|
if interval == 0 {
|
||||||
|
interval = 30 * time.Second
|
||||||
|
}
|
||||||
|
return &Monitor{
|
||||||
|
tmux: tc,
|
||||||
|
state: s,
|
||||||
|
config: cfg,
|
||||||
|
switchCh: make(chan SwitchRequest, 1),
|
||||||
|
interval: interval,
|
||||||
|
logger: log.Default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SwitchChan returns the channel on which SwitchRequests are sent.
|
||||||
|
func (m *Monitor) SwitchChan() <-chan SwitchRequest {
|
||||||
|
return m.switchCh
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run starts the quota monitor loop until ctx is cancelled.
|
||||||
|
func (m *Monitor) Run(ctx context.Context) {
|
||||||
|
ticker := time.NewTicker(m.interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
m.poll()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// poll checks all sessions for quota exhaustion once.
|
||||||
|
func (m *Monitor) poll() {
|
||||||
|
if m.state.ActiveAccount() != "" && m.isQuotaPaused() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
blockedPool := 0
|
||||||
|
blockedInteractive := 0
|
||||||
|
var resetTime string
|
||||||
|
|
||||||
|
prefix := m.config.Pool.Autonomous.Prefix
|
||||||
|
if prefix == "" {
|
||||||
|
prefix = "ccl-auto-"
|
||||||
|
}
|
||||||
|
for i := 0; i < m.config.Pool.Autonomous.Max; i++ {
|
||||||
|
name := sessionName(prefix, i)
|
||||||
|
if !m.tmux.HasSession(name) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Only capture 3 lines — avoids false positives on stale history.
|
||||||
|
tail, err := m.tmux.CapturePaneTail(name, 3)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if isQuotaExhausted(tail) {
|
||||||
|
blockedPool++
|
||||||
|
if rt := extractResetTime(tail); rt != "" {
|
||||||
|
resetTime = rt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ds := range m.config.Pool.Dedicated {
|
||||||
|
if !m.tmux.HasSession(ds.Name) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tail, err := m.tmux.CapturePaneTail(ds.Name, 3)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if isQuotaExhausted(tail) {
|
||||||
|
blockedInteractive++
|
||||||
|
if rt := extractResetTime(tail); rt != "" {
|
||||||
|
resetTime = rt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if blockedPool >= 2 || blockedInteractive >= 1 {
|
||||||
|
req := SwitchRequest{
|
||||||
|
From: m.state.ActiveAccount(),
|
||||||
|
ResetTime: resetTime,
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case m.switchCh <- req:
|
||||||
|
m.logger.Printf("[quota] SwapRequested: from=%s pool=%d interactive=%d reset=%q",
|
||||||
|
req.From, blockedPool, blockedInteractive, resetTime)
|
||||||
|
default:
|
||||||
|
// Swap already pending — do not queue another.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// isQuotaPaused checks whether a swap is already in progress.
|
||||||
|
func (m *Monitor) isQuotaPaused() bool {
|
||||||
|
// Lightweight proxy: if switch channel is full, a swap is pending.
|
||||||
|
return len(m.switchCh) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// isQuotaExhausted returns true if the pane content indicates quota exhaustion.
|
||||||
|
func isQuotaExhausted(paneContent string) bool {
|
||||||
|
lower := strings.ToLower(paneContent)
|
||||||
|
for _, p := range quotaPatterns {
|
||||||
|
if strings.Contains(lower, p) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractResetTime parses a reset time string from pane content.
|
||||||
|
// Returns "" if none found.
|
||||||
|
func extractResetTime(content string) string {
|
||||||
|
m := resetTimeRe.FindStringSubmatch(content)
|
||||||
|
if len(m) >= 2 {
|
||||||
|
return strings.TrimSpace(m[1])
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func sessionName(prefix string, i int) string {
|
||||||
|
return prefix + itoa(i)
|
||||||
|
}
|
||||||
|
|
||||||
|
func itoa(n int) string {
|
||||||
|
if n == 0 {
|
||||||
|
return "0"
|
||||||
|
}
|
||||||
|
b := make([]byte, 0, 10)
|
||||||
|
for n > 0 {
|
||||||
|
b = append([]byte{byte('0' + n%10)}, b...)
|
||||||
|
n /= 10
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
152
internal/quota/monitor_test.go
Normal file
152
internal/quota/monitor_test.go
Normal file
|
|
@ -0,0 +1,152 @@
|
||||||
|
package quota
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"forge.secuaas.ovh/olivier/claude-failover/internal/config"
|
||||||
|
"forge.secuaas.ovh/olivier/claude-failover/internal/state"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mockTmux for quota tests.
|
||||||
|
type mockTmux struct {
|
||||||
|
sessions map[string]bool
|
||||||
|
paneOutput map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMockTmux() *mockTmux {
|
||||||
|
return &mockTmux{
|
||||||
|
sessions: make(map[string]bool),
|
||||||
|
paneOutput: make(map[string]string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockTmux) HasSession(name string) bool { return m.sessions[name] }
|
||||||
|
func (m *mockTmux) CreateSession(name, _ string) error { m.sessions[name] = true; return nil }
|
||||||
|
func (m *mockTmux) KillSession(_ string) error { return nil }
|
||||||
|
func (m *mockTmux) SendKeys(_, _ string) error { return nil }
|
||||||
|
func (m *mockTmux) CapturePaneTail(session string, _ int) (string, error) {
|
||||||
|
return m.paneOutput[session], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestIsQuotaExhausted verifies pattern matching on pane output.
|
||||||
|
func TestIsQuotaExhausted(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
input string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"You've hit your limit for Claude Pro.", true},
|
||||||
|
{"rate limit exceeded", true},
|
||||||
|
{"quota exceeded for this period", true},
|
||||||
|
{"Usage limit reached", true},
|
||||||
|
{"Too many requests", true},
|
||||||
|
{"Some normal output ❯", false},
|
||||||
|
{"❯ ", false},
|
||||||
|
{"still running 5s · ", false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := isQuotaExhausted(c.input); got != c.want {
|
||||||
|
t.Errorf("isQuotaExhausted(%q) = %v, want %v", c.input, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestExtractResetTime parses various reset time formats.
|
||||||
|
func TestExtractResetTime(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
input string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"Usage resets 8pm", "8pm"},
|
||||||
|
{"Your quota resets at 11:30pm", "11:30pm"},
|
||||||
|
{"resets in 45 minutes", "in 45 minutes"},
|
||||||
|
{"resets in 2 hours", "in 2 hours"},
|
||||||
|
{"no reset info here", ""},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := extractResetTime(c.input); got != c.want {
|
||||||
|
t.Errorf("extractResetTime(%q) = %q, want %q", c.input, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPollTriggersSwitchOnTwoBlockedPool verifies swap trigger for >=2 blocked pool sessions.
|
||||||
|
func TestPollTriggersSwitchOnTwoBlockedPool(t *testing.T) {
|
||||||
|
tc := newMockTmux()
|
||||||
|
tc.sessions["ccl-auto-0"] = true
|
||||||
|
tc.sessions["ccl-auto-1"] = true
|
||||||
|
tc.paneOutput["ccl-auto-0"] = "You've hit your limit for Claude Pro."
|
||||||
|
tc.paneOutput["ccl-auto-1"] = "rate limit exceeded"
|
||||||
|
|
||||||
|
s := state.New("")
|
||||||
|
s.SetActiveAccount("compte1")
|
||||||
|
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pool: config.PoolConfig{
|
||||||
|
Autonomous: config.AutonomousConfig{Prefix: "ccl-auto-", Max: 2},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
m := New(tc, s, cfg)
|
||||||
|
m.poll()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case req := <-m.switchCh:
|
||||||
|
if req.From != "compte1" {
|
||||||
|
t.Errorf("expected From=compte1, got %q", req.From)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Fatal("expected SwitchRequest on channel")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPollTriggersSwitchOnOneBlockedInteractive verifies swap trigger for >=1 dedicated session.
|
||||||
|
func TestPollTriggersSwitchOnOneBlockedInteractive(t *testing.T) {
|
||||||
|
tc := newMockTmux()
|
||||||
|
tc.sessions["my-session"] = true
|
||||||
|
tc.paneOutput["my-session"] = "quota exceeded"
|
||||||
|
|
||||||
|
s := state.New("")
|
||||||
|
s.SetActiveAccount("compte1")
|
||||||
|
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pool: config.PoolConfig{
|
||||||
|
Dedicated: []config.DedicatedSession{{Name: "my-session"}},
|
||||||
|
Autonomous: config.AutonomousConfig{Max: 0},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
m := New(tc, s, cfg)
|
||||||
|
m.poll()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case req := <-m.switchCh:
|
||||||
|
if req.From != "compte1" {
|
||||||
|
t.Errorf("expected From=compte1, got %q", req.From)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Fatal("expected SwitchRequest on channel")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPollNoTriggerWhenBelowThreshold verifies no swap for a single blocked pool session.
|
||||||
|
func TestPollNoTriggerWhenBelowThreshold(t *testing.T) {
|
||||||
|
tc := newMockTmux()
|
||||||
|
tc.sessions["ccl-auto-0"] = true
|
||||||
|
tc.sessions["ccl-auto-1"] = true
|
||||||
|
tc.paneOutput["ccl-auto-0"] = "rate limit exceeded"
|
||||||
|
tc.paneOutput["ccl-auto-1"] = "❯ " // fine
|
||||||
|
|
||||||
|
s := state.New("")
|
||||||
|
cfg := &config.Config{
|
||||||
|
Pool: config.PoolConfig{
|
||||||
|
Autonomous: config.AutonomousConfig{Prefix: "ccl-auto-", Max: 2},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
m := New(tc, s, cfg)
|
||||||
|
m.poll()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case req := <-m.switchCh:
|
||||||
|
t.Errorf("unexpected SwitchRequest: %+v", req)
|
||||||
|
default:
|
||||||
|
// Correct: only 1 blocked pool session, threshold is 2.
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue