5a85f0cfcc
This commit addresses "Max retries exceeded" errors during stress testing where all accounts would become exhausted simultaneously due to short per-second rate limits triggering cascading failures. ## Rate Limit Parser (`rate-limit-parser.js`) - Remove 2s buffer enforcement that caused cascading failures when API returned short reset times (200-600ms). Now adds 200ms buffer for sub-500ms resets - Add `parseRateLimitReason()` for smart backoff based on error type: QUOTA_EXHAUSTED, RATE_LIMIT_EXCEEDED, MODEL_CAPACITY_EXHAUSTED, SERVER_ERROR ## Message/Streaming Handlers - Add per-account+model rate limit state tracking with exponential backoff - For short rate limits (< 1 second), wait and retry on same account instead of switching - prevents thundering herd when all accounts hit per-second limits - Add throttle wait support for fallback modes (emergency/lastResort) - Add `calculateSmartBackoff()` with progressive tiers by error type ## HybridStrategy (`hybrid-strategy.js`) - Refactor `#getCandidates()` to return 4 fallback levels: - `normal`: All filters pass (health, tokens, quota) - `quota`: Bypass critical quota check - `emergency`: Bypass health check when ALL accounts unhealthy - `lastResort`: Bypass BOTH health AND token bucket checks - Add throttle wait times: 500ms for lastResort, 250ms for emergency - Fix LRU calculation to use seconds (matches opencode-antigravity-auth) ## Health Tracker - Increase `recoveryPerHour` from 2 to 10 for faster recovery (1 hour vs 5 hours) ## Account Manager - Add consecutive failure tracking: `getConsecutiveFailures()`, `incrementConsecutiveFailures()`, `resetConsecutiveFailures()` - Add cooldown mechanism separate from rate limits with `CooldownReason` - Reset consecutive failures on successful request ## Base Strategy - Add `isAccountCoolingDown()` check in `isAccountUsable()` ## Constants - Replace fixed `CAPACITY_RETRY_DELAY_MS` with progressive `CAPACITY_BACKOFF_TIERS_MS` - Add `BACKOFF_BY_ERROR_TYPE` for smart backoff - Add `QUOTA_EXHAUSTED_BACKOFF_TIERS_MS` for progressive quota backoff - Add `MIN_BACKOFF_MS` floor to prevent "Available in 0s" loops - Increase `MAX_CAPACITY_RETRIES` from 3 to 5 - Reduce `RATE_LIMIT_DEDUP_WINDOW_MS` from 5s to 2s ## Frontend - Remove `capacityRetryDelayMs` config (replaced by progressive tiers) - Update default `maxCapacityRetries` display from 3 to 5 ## Testing - Add `tests/stress-test.cjs` for concurrent request stress testing Co-Authored-By: Claude <noreply@anthropic.com>
103 lines
2.1 KiB
JavaScript
103 lines
2.1 KiB
JavaScript
/**
|
|
* Application Constants
|
|
* Centralized configuration values and magic numbers
|
|
*/
|
|
window.AppConstants = window.AppConstants || {};
|
|
|
|
/**
|
|
* Time intervals (in milliseconds)
|
|
*/
|
|
window.AppConstants.INTERVALS = {
|
|
// Dashboard refresh interval (5 minutes)
|
|
DASHBOARD_REFRESH: 300000,
|
|
|
|
// OAuth message handler timeout (5 minutes)
|
|
OAUTH_MESSAGE_TIMEOUT: 300000,
|
|
|
|
// Server config debounce delay
|
|
CONFIG_DEBOUNCE: 500,
|
|
|
|
// General short delay (for UI transitions)
|
|
SHORT_DELAY: 2000
|
|
};
|
|
|
|
/**
|
|
* Data limits and quotas
|
|
*/
|
|
window.AppConstants.LIMITS = {
|
|
// Default log limit
|
|
DEFAULT_LOG_LIMIT: 2000,
|
|
|
|
// Minimum quota value
|
|
MIN_QUOTA: 100,
|
|
|
|
// Percentage base (for calculations)
|
|
PERCENTAGE_BASE: 100
|
|
};
|
|
|
|
/**
|
|
* Validation ranges
|
|
*/
|
|
window.AppConstants.VALIDATION = {
|
|
// Port range
|
|
PORT_MIN: 1,
|
|
PORT_MAX: 65535,
|
|
|
|
// Timeout range (0 - 5 minutes)
|
|
TIMEOUT_MIN: 0,
|
|
TIMEOUT_MAX: 300000,
|
|
|
|
// Log limit range
|
|
LOG_LIMIT_MIN: 100,
|
|
LOG_LIMIT_MAX: 10000,
|
|
|
|
// Retry configuration ranges
|
|
MAX_RETRIES_MIN: 0,
|
|
MAX_RETRIES_MAX: 20,
|
|
|
|
RETRY_BASE_MS_MIN: 100,
|
|
RETRY_BASE_MS_MAX: 10000,
|
|
|
|
RETRY_MAX_MS_MIN: 1000,
|
|
RETRY_MAX_MS_MAX: 60000,
|
|
|
|
// Cooldown range (0 - 10 minutes)
|
|
DEFAULT_COOLDOWN_MIN: 0,
|
|
DEFAULT_COOLDOWN_MAX: 600000,
|
|
|
|
// Max wait threshold (1 - 30 minutes)
|
|
MAX_WAIT_MIN: 60000,
|
|
MAX_WAIT_MAX: 1800000,
|
|
|
|
// Max accounts range (1 - 100)
|
|
MAX_ACCOUNTS_MIN: 1,
|
|
MAX_ACCOUNTS_MAX: 100,
|
|
|
|
// Rate limit dedup window (1 - 30 seconds)
|
|
RATE_LIMIT_DEDUP_MIN: 1000,
|
|
RATE_LIMIT_DEDUP_MAX: 30000,
|
|
|
|
// Consecutive failures (1 - 10)
|
|
MAX_CONSECUTIVE_FAILURES_MIN: 1,
|
|
MAX_CONSECUTIVE_FAILURES_MAX: 10,
|
|
|
|
// Extended cooldown (10 seconds - 5 minutes)
|
|
EXTENDED_COOLDOWN_MIN: 10000,
|
|
EXTENDED_COOLDOWN_MAX: 300000,
|
|
|
|
// Capacity retries (1 - 10)
|
|
MAX_CAPACITY_RETRIES_MIN: 1,
|
|
MAX_CAPACITY_RETRIES_MAX: 10
|
|
};
|
|
|
|
/**
|
|
* UI Constants
|
|
*/
|
|
window.AppConstants.UI = {
|
|
// Toast auto-dismiss duration
|
|
TOAST_DURATION: 3000,
|
|
|
|
// Loading spinner delay
|
|
LOADING_DELAY: 200
|
|
};
|