feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling

- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips
- Error capture: every failed/backoff attempt logged to queue_errors with the
  full stack trace; admin queue page expands a failed job to stream it
- Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs)
- Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop
  (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes
- Startup recovery: interrupted in_flight jobs reset to pending on boot
- fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the
  company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty
- Migration: add queue_errors, queue_schedules, queue_state tables plus error/
  scheduled_for columns on adapter_queue (idempotent ALTER on startup)

Verified: full sec-fetch backfill now succeeds for all watched symbols
(NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.).
503 backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Investor Flow Build
2026-07-12 12:55:59 -04:00
co-authored by Claude
parent 55f07e6b42
commit ca385c1960
10 changed files with 1345 additions and 141 deletions
+12
View File
@@ -44,12 +44,24 @@ export function initSchema(database: DatabaseSync): void {
database.exec(sql);
}
/** Idempotent migrations for existing databases (new columns, tables). */
function runMigrations(db: DatabaseSync): void {
const migrations: string[] = [
`ALTER TABLE adapter_queue ADD COLUMN error TEXT`,
`ALTER TABLE adapter_queue ADD COLUMN scheduled_for TEXT`,
];
for (const sql of migrations) {
try { db.exec(sql); } catch { /* column already exists */ }
}
}
let _db: DatabaseSync | null = null;
/** Process-wide singleton connection (auto-creates + inits schema on first call). */
export function db(): DatabaseSync {
if (!_db) {
_db = createDb();
initSchema(_db);
runMigrations(_db);
// Ensure the anonymous user exists for unauthenticated watchlist/portfolio usage.
try {
_db.prepare("INSERT OR IGNORE INTO users (id, email, pw_hash, created_at) VALUES (?, ?, ?, ?)")