feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling

- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips
- Error capture: every failed/backoff attempt logged to queue_errors with the
  full stack trace; admin queue page expands a failed job to stream it
- Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs)
- Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop
  (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes
- Startup recovery: interrupted in_flight jobs reset to pending on boot
- fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the
  company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty
- Migration: add queue_errors, queue_schedules, queue_state tables plus error/
  scheduled_for columns on adapter_queue (idempotent ALTER on startup)

Verified: full sec-fetch backfill now succeeds for all watched symbols
(NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.).
503 backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Investor Flow Build
2026-07-12 12:55:59 -04:00
co-authored by Claude
parent 55f07e6b42
commit ca385c1960
10 changed files with 1345 additions and 141 deletions
+4 -1
View File
@@ -4,6 +4,7 @@
import { randomUUID, randomBytes, scryptSync, timingSafeEqual, createHmac } from 'node:crypto';
import type { DatabaseSync } from 'node:sqlite';
import type { CacheRepository } from '../cache/CacheRepository.ts';
import type { AdapterQueue } from '../queue/AdapterQueue.ts';
export const SESSION_COOKIE = 'iflow_session';
export const OAUTH_STATE_COOKIE = 'iflow_oauth_state';
@@ -19,6 +20,7 @@ const SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
export interface Context {
db: DatabaseSync;
cache: CacheRepository;
queue: AdapterQueue;
resHeaders: Headers; // mutable; procedures append Set-Cookie here (applied to Response by the fetch adapter)
userId: string | null; // resolved from the session cookie; null = unauthenticated
cookies: Record<string, string>; // parsed request cookies (session + oauth state)
@@ -100,12 +102,13 @@ export function verifyPassword(pw: string, stored: string): boolean {
}
// ----- context factory: 1f wires real db+cache; tests inject in-memory -----
export function makeCreateContext(opts: { db: DatabaseSync; cache: CacheRepository }) {
export function makeCreateContext(opts: { db: DatabaseSync; cache: CacheRepository; queue: AdapterQueue }) {
return ({ req, resHeaders }: CreateContextOpts): Context => {
const cookies = parseCookies(req);
return {
db: opts.db,
cache: opts.cache,
queue: opts.queue,
resHeaders,
cookies,
userId: resolveSessionUserId(opts.db, req),