feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling

- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips
- Error capture: every failed/backoff attempt logged to queue_errors with the
  full stack trace; admin queue page expands a failed job to stream it
- Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs)
- Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop
  (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes
- Startup recovery: interrupted in_flight jobs reset to pending on boot
- fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the
  company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty
- Migration: add queue_errors, queue_schedules, queue_state tables plus error/
  scheduled_for columns on adapter_queue (idempotent ALTER on startup)

Verified: full sec-fetch backfill now succeeds for all watched symbols
(NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.).
503 backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Investor Flow Build
2026-07-12 12:55:59 -04:00
co-authored by Claude
parent 55f07e6b42
commit ca385c1960
10 changed files with 1345 additions and 141 deletions
+50 -2
View File
@@ -210,12 +210,12 @@ export const api = {
removeHolding: (symbol: string) => trpcMutate<{ removed: boolean }>("portfolio.removeHolding", { symbol }),
},
dashboard: {
rollup: () => trpcQuery<Record<string, unknown>[]>("dashboard.rollup"),
rollup: () => trpcQuery<Record<string, unknown>[]>("dashboard.rollup", {}),
},
institutional: {
flow: (symbol: string) =>
trpcQuery<{ symbol: string; flow: Array<Record<string, unknown>>; quarters: string[] }>("institutional.flow", { symbol }),
insiderStream: (symbol: string, limit = 50) =>
insiderStream: (symbol: string, limit = 200) =>
trpcQuery<{ symbol: string; events: Array<Record<string, unknown>>; netShares: number; count: number }>("institutional.insiderStream", { symbol, limit }),
},
emotionLogger: {
@@ -230,6 +230,26 @@ export const api = {
unackedCount: () => trpcQuery<{ count: number }>("alerts.unackedCount"),
acknowledgeAll: () => trpcMutate<{ ok: boolean }>("alerts.acknowledgeAll"),
},
admin: {
usersList: () => trpcQuery<Array<{ id: string; email: string; complexity: string; created_at: string; is_admin: number }>>("admin.usersList"),
userSessions: (userId: string) => trpcQuery<Array<{ id: string; user_id: string; expires_at: string; created_at: string }>>("admin.userSessions", { userId }),
resetPassword: (email: string, tempPassword: string) => trpcMutate<{ userId: string }>("admin.resetPassword", { email, tempPassword }),
queueHealth: () => trpcQuery<Array<{ source: string; status: string; count: number; last_attempt: string | null; retry_count: number; backoff_until: string | null }>>("admin.queueHealth"),
resetBackoff: (sourceKind: string) => trpcMutate<{ cleared: number }>("admin.resetBackoff", { sourceKind }),
queueSecFetch: (symbol: string) => trpcMutate<{ ok: boolean }>("admin.queueSecFetch", { symbol }),
queueStatus: () => trpcQuery<QueueStatus>("admin.queueStatus"),
queuePause: () => trpcMutate<{ paused: boolean }>("admin.queuePause", undefined),
queueResume: () => trpcMutate<{ paused: boolean }>("admin.queueResume", undefined),
queueClearDone: (olderThanHours: number) => trpcMutate<{ cleared: number }>("admin.queueClearDone", { olderThanHours }),
queueSchedules: () => trpcQuery<ScheduleEntry[]>("admin.queueSchedules"),
queueSetSchedule: (sourceKind: string, intervalMs: number) => trpcMutate<{ ok: boolean }>("admin.queueSetSchedule", { sourceKind, intervalMs }),
queueDeleteSchedule: (sourceKind: string) => trpcMutate<{ ok: boolean }>("admin.queueDeleteSchedule", { sourceKind }),
queueLogs: (key: string, limit: number) => trpcQuery<ErrorLogEntry[]>("admin.queueLogs", { key, limit }),
queueRetryJob: (key: string) => trpcMutate<{ ok: boolean }>("admin.queueRetryJob", { key }),
queueRetrySource: (sourceKind: string) => trpcMutate<{ cleared: number }>("admin.queueRetrySource", { sourceKind }),
auditLog: (limit = 50, offset = 0, actor?: string, action?: string) =>
trpcQuery<Array<{ id: string; actor: string; action: string; target: string | null; ts: string; meta: string | null }>>("admin.auditLog", { limit, offset, ...(actor ? { actor } : {}), ...(action ? { action } : {}) }),
},
};
export interface WatchlistEntry {
@@ -244,3 +264,31 @@ export interface PortfolioHolding {
avg_cost: number;
added_at: string;
}
export interface QueueStatus {
paused: boolean;
queued: number;
in_flight: number;
failed: number;
backoff: number;
done: number;
backoff_until: string | null;
last_error: string | undefined;
counts: Record<string, number>;
lastErrors: Array<{ key: string; error: string | null }>;
}
export interface ScheduleEntry {
sourceKind: string;
intervalMs: number;
lastEnqueued: string | null;
nextEnqueue: string | null;
}
export interface ErrorLogEntry {
attempt: number;
errorMessage: string | null;
errorStack: string | null;
ts: string;
}