feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling

- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips
- Error capture: every failed/backoff attempt logged to queue_errors with the
  full stack trace; admin queue page expands a failed job to stream it
- Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs)
- Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop
  (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes
- Startup recovery: interrupted in_flight jobs reset to pending on boot
- fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the
  company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty
- Migration: add queue_errors, queue_schedules, queue_state tables plus error/
  scheduled_for columns on adapter_queue (idempotent ALTER on startup)

Verified: full sec-fetch backfill now succeeds for all watched symbols
(NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.).
503 backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>

app/server/src/index.ts

@@ -5,17 +5,31 @@ import { fetchRequestHandler } from '@trpc/server/adapters/fetch';
 import { db } from './db/client.ts';
 import { createCacheRepository, type SourceKind } from './cache/CacheRepository.ts';
 import { YFinanceAdapter } from './adapters/YFinanceAdapter.ts';
+import { SecFetchAdapter } from './adapters/SecFetchAdapter.ts';
+import type { SourceFetch } from './adapters/SourceAdapter.ts';
 import { AdapterQueue } from './queue/AdapterQueue.ts';
 import { makeCreateContext } from './trpc/context.ts';
 import { appRouter } from './trpc/router.ts';
 
 const PORT = Number(process.env.PORT ?? 3001);
 const database = db();
-const adapters = new Map<SourceKind, YFinanceAdapter>([['yfinance', new YFinanceAdapter()]]);
+const adapters: Map<SourceKind, SourceFetch> = new Map([
+  ['yfinance', new YFinanceAdapter()],
+  ['sec-fetch', new SecFetchAdapter(database)],
+]);
 const queue = new AdapterQueue({ db: database, adapters });
 const cache = createCacheRepository({ db: database, scheduler: queue });
 queue.cache = cache; // break the cache<->scheduler cycle
-const createContext = makeCreateContext({ db: database, cache });
+
+// Seed default schedules (noop if already seeded)
+queue.seedDefaultSchedules();
+
+// Startup recovery: any job left 'in_flight' was interrupted by a restart/crash.
+// Reset to 'pending' so the drain loop reprocesses it.
+const recovered = database.prepare("UPDATE adapter_queue SET status='pending', error=NULL, retry_count=0 WHERE status='in_flight'").run();
+if (Number(recovered.changes) > 0) console.log(`[investor-flow] recovered ${recovered.changes} interrupted in_flight jobs`);
+
+const createContext = makeCreateContext({ db: database, cache, queue });
 
 // Background drain: stale-while-revalidate refreshes are queued by CacheRepository.get;
 // this loop drains them (fetch via adapter -> write to cache), deduped + backed off.
@@ -23,6 +37,11 @@ const DRAIN_MS = Number(process.env.IFLOW_DRAIN_MS ?? 2000);
 const drainTimer = setInterval(() => { queue.drain().catch((e) => console.error('[drain error]', e)); }, DRAIN_MS);
 drainTimer.unref();
 
+// Auto-scheduler: every 30s, enqueue refreshes for due schedules
+const SCHEDULE_MS = 30_000;
+const scheduleTimer = setInterval(() => { queue.enqueueDueSchedules().catch((e) => console.error('[schedule error]', e)); }, SCHEDULE_MS);
+scheduleTimer.unref();
+
 function readBody(req: IncomingMessage): Promise<string> {
   return new Promise((resolve, reject) => {
     let data = '';
@@ -44,7 +63,12 @@ const server = createServer(async (req, res) => {
     for (const [k, v] of Object.entries(req.headers)) if (v != null) headers.set(k, Array.isArray(v) ? v.join(', ') : String(v));
     const method = req.method ?? 'GET';
     const body = method === 'GET' || method === 'HEAD' ? undefined : await readBody(req);
-    const request = new Request(`http://localhost:${PORT}${url.pathname}${url.search}`, { method, headers, body });
+    const requestUrl = `http://localhost:${PORT}${url.pathname}${url.search}`;
+    const init: RequestInit = { method, headers };
+    if (body) {
+      init.body = new Blob([body], { type: 'application/json' });
+    }
+    const request = new Request(requestUrl, init);
     try {
       const response = await fetchRequestHandler({ router: appRouter, createContext, endpoint: '/api/trpc', req: request });
       const buf = Buffer.from(await response.arrayBuffer());
@@ -61,6 +85,7 @@ const server = createServer(async (req, res) => {
   res.end(JSON.stringify({ error: 'not found' }));
 });
 
-server.listen(PORT, () => {
-  console.log(`[investor-flow] backend on http://localhost:${PORT} (tRPC /api/trpc, health /health, drain every ${DRAIN_MS}ms)`);
+const HOST = process.env.HOST ?? '0.0.0.0';
+server.listen(PORT, HOST, () => {
+  console.log(`[investor-flow] backend on http://${HOST}:${PORT} (tRPC /api/trpc, health /health, drain every ${DRAIN_MS}ms)`);
 });