feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling
- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips - Error capture: every failed/backoff attempt logged to queue_errors with the full stack trace; admin queue page expands a failed job to stream it - Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs) - Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes - Startup recovery: interrupted in_flight jobs reset to pending on boot - fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty - Migration: add queue_errors, queue_schedules, queue_state tables plus error/ scheduled_for columns on adapter_queue (idempotent ALTER on startup) Verified: full sec-fetch backfill now succeeds for all watched symbols (NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.). 503 backend tests pass. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
55f07e6b42
commit
ca385c1960
+50
-2
@@ -210,12 +210,12 @@ export const api = {
|
||||
removeHolding: (symbol: string) => trpcMutate<{ removed: boolean }>("portfolio.removeHolding", { symbol }),
|
||||
},
|
||||
dashboard: {
|
||||
rollup: () => trpcQuery<Record<string, unknown>[]>("dashboard.rollup"),
|
||||
rollup: () => trpcQuery<Record<string, unknown>[]>("dashboard.rollup", {}),
|
||||
},
|
||||
institutional: {
|
||||
flow: (symbol: string) =>
|
||||
trpcQuery<{ symbol: string; flow: Array<Record<string, unknown>>; quarters: string[] }>("institutional.flow", { symbol }),
|
||||
insiderStream: (symbol: string, limit = 50) =>
|
||||
insiderStream: (symbol: string, limit = 200) =>
|
||||
trpcQuery<{ symbol: string; events: Array<Record<string, unknown>>; netShares: number; count: number }>("institutional.insiderStream", { symbol, limit }),
|
||||
},
|
||||
emotionLogger: {
|
||||
@@ -230,6 +230,26 @@ export const api = {
|
||||
unackedCount: () => trpcQuery<{ count: number }>("alerts.unackedCount"),
|
||||
acknowledgeAll: () => trpcMutate<{ ok: boolean }>("alerts.acknowledgeAll"),
|
||||
},
|
||||
admin: {
|
||||
usersList: () => trpcQuery<Array<{ id: string; email: string; complexity: string; created_at: string; is_admin: number }>>("admin.usersList"),
|
||||
userSessions: (userId: string) => trpcQuery<Array<{ id: string; user_id: string; expires_at: string; created_at: string }>>("admin.userSessions", { userId }),
|
||||
resetPassword: (email: string, tempPassword: string) => trpcMutate<{ userId: string }>("admin.resetPassword", { email, tempPassword }),
|
||||
queueHealth: () => trpcQuery<Array<{ source: string; status: string; count: number; last_attempt: string | null; retry_count: number; backoff_until: string | null }>>("admin.queueHealth"),
|
||||
resetBackoff: (sourceKind: string) => trpcMutate<{ cleared: number }>("admin.resetBackoff", { sourceKind }),
|
||||
queueSecFetch: (symbol: string) => trpcMutate<{ ok: boolean }>("admin.queueSecFetch", { symbol }),
|
||||
queueStatus: () => trpcQuery<QueueStatus>("admin.queueStatus"),
|
||||
queuePause: () => trpcMutate<{ paused: boolean }>("admin.queuePause", undefined),
|
||||
queueResume: () => trpcMutate<{ paused: boolean }>("admin.queueResume", undefined),
|
||||
queueClearDone: (olderThanHours: number) => trpcMutate<{ cleared: number }>("admin.queueClearDone", { olderThanHours }),
|
||||
queueSchedules: () => trpcQuery<ScheduleEntry[]>("admin.queueSchedules"),
|
||||
queueSetSchedule: (sourceKind: string, intervalMs: number) => trpcMutate<{ ok: boolean }>("admin.queueSetSchedule", { sourceKind, intervalMs }),
|
||||
queueDeleteSchedule: (sourceKind: string) => trpcMutate<{ ok: boolean }>("admin.queueDeleteSchedule", { sourceKind }),
|
||||
queueLogs: (key: string, limit: number) => trpcQuery<ErrorLogEntry[]>("admin.queueLogs", { key, limit }),
|
||||
queueRetryJob: (key: string) => trpcMutate<{ ok: boolean }>("admin.queueRetryJob", { key }),
|
||||
queueRetrySource: (sourceKind: string) => trpcMutate<{ cleared: number }>("admin.queueRetrySource", { sourceKind }),
|
||||
auditLog: (limit = 50, offset = 0, actor?: string, action?: string) =>
|
||||
trpcQuery<Array<{ id: string; actor: string; action: string; target: string | null; ts: string; meta: string | null }>>("admin.auditLog", { limit, offset, ...(actor ? { actor } : {}), ...(action ? { action } : {}) }),
|
||||
},
|
||||
};
|
||||
|
||||
export interface WatchlistEntry {
|
||||
@@ -244,3 +264,31 @@ export interface PortfolioHolding {
|
||||
avg_cost: number;
|
||||
added_at: string;
|
||||
}
|
||||
|
||||
export interface QueueStatus {
|
||||
paused: boolean;
|
||||
queued: number;
|
||||
in_flight: number;
|
||||
failed: number;
|
||||
backoff: number;
|
||||
done: number;
|
||||
backoff_until: string | null;
|
||||
last_error: string | undefined;
|
||||
counts: Record<string, number>;
|
||||
lastErrors: Array<{ key: string; error: string | null }>;
|
||||
}
|
||||
|
||||
export interface ScheduleEntry {
|
||||
sourceKind: string;
|
||||
intervalMs: number;
|
||||
lastEnqueued: string | null;
|
||||
nextEnqueue: string | null;
|
||||
}
|
||||
|
||||
export interface ErrorLogEntry {
|
||||
attempt: number;
|
||||
errorMessage: string | null;
|
||||
errorStack: string | null;
|
||||
ts: string;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user