feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling

- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips
- Error capture: every failed/backoff attempt logged to queue_errors with the
  full stack trace; admin queue page expands a failed job to stream it
- Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs)
- Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop
  (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes
- Startup recovery: interrupted in_flight jobs reset to pending on boot
- fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the
  company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty
- Migration: add queue_errors, queue_schedules, queue_state tables plus error/
  scheduled_for columns on adapter_queue (idempotent ALTER on startup)

Verified: full sec-fetch backfill now succeeds for all watched symbols
(NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.).
503 backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>

app/server/src/trpc/context.ts

@@ -4,6 +4,7 @@
 import { randomUUID, randomBytes, scryptSync, timingSafeEqual, createHmac } from 'node:crypto';
 import type { DatabaseSync } from 'node:sqlite';
 import type { CacheRepository } from '../cache/CacheRepository.ts';
+import type { AdapterQueue } from '../queue/AdapterQueue.ts';
 
 export const SESSION_COOKIE = 'iflow_session';
 export const OAUTH_STATE_COOKIE = 'iflow_oauth_state';
@@ -19,6 +20,7 @@ const SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
 export interface Context {
   db: DatabaseSync;
   cache: CacheRepository;
+  queue: AdapterQueue;
   resHeaders: Headers; // mutable; procedures append Set-Cookie here (applied to Response by the fetch adapter)
   userId: string | null; // resolved from the session cookie; null = unauthenticated
   cookies: Record<string, string>; // parsed request cookies (session + oauth state)
@@ -100,12 +102,13 @@ export function verifyPassword(pw: string, stored: string): boolean {
 }
 
 // ----- context factory: 1f wires real db+cache; tests inject in-memory -----
-export function makeCreateContext(opts: { db: DatabaseSync; cache: CacheRepository }) {
+export function makeCreateContext(opts: { db: DatabaseSync; cache: CacheRepository; queue: AdapterQueue }) {
   return ({ req, resHeaders }: CreateContextOpts): Context => {
     const cookies = parseCookies(req);
     return {
       db: opts.db,
       cache: opts.cache,
+      queue: opts.queue,
       resHeaders,
       cookies,
       userId: resolveSessionUserId(opts.db, req),

app/server/src/trpc/router.ts

@@ -9,7 +9,7 @@ import { buildAuthorizeUrl, generateState, exchangeCode, type OAuthProvider } fr
 import { STARTER_WATCHLIST, defaultDrawdownTolerancePct, defaultRiskTolerance, ONBOARDING_DISCLAIMER, type Complexity } from '../onboarding/starter.ts';
 import type { Quote, PriceCandle, SymbolMeta } from '../cache/CacheRepository.ts';
 import { emaFromCandles, rsi as rsiFn, relativeVolume } from '../analysis/indicators.ts';
-import { listUsers, resetPassword, gdprExport, queueHealth, resetQueueBackoff, NotOwnerError } from '../admin/admin.ts';
+import { listUsers, resetPassword, gdprExport, queueHealth, resetQueueBackoff, NotOwnerError, listUserSessions, listAuditLog, queueSecFetch } from '../admin/admin.ts';
 import { EdgarAdapter } from '../adapters/EdgarAdapter.ts';
 import { OptionsAdapter, parseOptionChainRows } from '../adapters/OptionsAdapter.ts';
 import type { OptionChainRow, OptionGreeks } from '../adapters/OptionsAdapter.ts';
@@ -156,6 +156,7 @@ const onboardingRouter = router({
       for (const sym of symbols) {
         const kind = (STARTER_WATCHLIST.find((s) => s.symbol === sym)?.tickerKind ?? 'equity') as 'equity' | 'crypto' | 'etf' | 'index';
         await ctx.cache.subscribe(sym, kind);
+        queueSecFetch(ctx.db, sym);
       }
       if (input.portfolio) {
         const ins = ctx.db.prepare('INSERT INTO portfolio_holdings (id, owner_id, symbol, qty, avg_cost, acquired_at, status) VALUES (?,?,?,?,?,?,?)');
@@ -254,6 +255,79 @@ const adminRouter = router({
   resetBackoff: adminProcedure
     .input(z.object({ sourceKind: z.string().regex(/^[a-z0-9_]+$/i) }))
     .mutation(({ ctx, input }) => resetQueueBackoff(ctx.db, ctx.userId, input.sourceKind)),
+
+  userSessions: adminProcedure
+    .input(z.object({ userId: z.string().uuid() }))
+    .query(({ ctx, input }) => listUserSessions(ctx.db, input.userId)),
+
+  auditLog: adminProcedure
+    .input(z.object({
+      limit: z.number().int().min(1).max(200).default(50),
+      offset: z.number().int().min(0).default(0),
+      actor: z.string().optional(),
+      action: z.string().optional(),
+    }))
+    .query(({ ctx, input }) => listAuditLog(ctx.db, { limit: input.limit, offset: input.offset, actor: input.actor ?? null, action: input.action ?? null })),
+
+  queueSecFetch: adminProcedure
+    .input(z.object({ symbol: z.string().min(1).max(10) }))
+    .mutation(({ ctx, input }) => {
+      try { queueSecFetch(ctx.db, input.symbol); return { ok: true }; }
+      catch (e) { throw new TRPCError({ code: 'BAD_REQUEST', message: e instanceof Error ? e.message : 'Failed to queue fetch.' }); }
+    }),
+
+  queueStatus: adminProcedure.query(({ ctx }) => ctx.queue.health()),
+
+  queuePause: adminProcedure.mutation(({ ctx }) => {
+    ctx.queue.setPaused(true);
+    return { paused: true };
+  }),
+
+  queueResume: adminProcedure.mutation(({ ctx }) => {
+    ctx.queue.setPaused(false);
+    return { paused: false };
+  }),
+
+  queueLogs: adminProcedure
+    .input(z.object({ key: z.string().min(1), limit: z.number().int().min(1).max(100).optional().default(10) }))
+    .query(({ ctx, input }) => ctx.queue.getErrorLog(input.key, input.limit)),
+
+  queueRetryJob: adminProcedure
+    .input(z.object({ key: z.string().min(1) }))
+    .mutation(({ ctx, input }) => {
+      ctx.queue.retryJob(input.key);
+      return { ok: true };
+    }),
+
+  queueRetrySource: adminProcedure
+    .input(z.object({ sourceKind: z.string().min(1) }))
+    .mutation(({ ctx, input }) => {
+      const cleared = ctx.queue.retrySource(input.sourceKind);
+      return { cleared };
+    }),
+
+  queueClearDone: adminProcedure
+    .input(z.object({ olderThanHours: z.number().min(1).max(336).optional().default(24) }))
+    .mutation(({ ctx, input }) => {
+      const cleared = ctx.queue.clearDone(input.olderThanHours * 3600000);
+      return { cleared };
+    }),
+
+  queueSchedules: adminProcedure.query(({ ctx }) => ctx.queue.listSchedules()),
+
+  queueSetSchedule: adminProcedure
+    .input(z.object({ sourceKind: z.string().min(1), intervalMs: z.number().int().min(60000).max(604800000) }))
+    .mutation(({ ctx, input }) => {
+      ctx.queue.setSchedule(input.sourceKind, input.intervalMs);
+      return { ok: true };
+    }),
+
+  queueDeleteSchedule: adminProcedure
+    .input(z.object({ sourceKind: z.string().min(1) }))
+    .mutation(({ ctx, input }) => {
+      ctx.queue.deleteSchedule(input.sourceKind);
+      return { ok: true };
+    }),
 });
 
 
@@ -262,7 +336,7 @@ const dashboardRouter = router({
   rollup: publicProcedure
     .input(z.object({}))
     .query(async ({ ctx }) => {
-      const userId = ctx.userId as string;
+      const userId = ctx.userId ?? 'anonymous';
       // Import lazily to avoid circular deps.
       const { DashboardRollupEngine } = await import('../analysis/dashboardRollup.ts');
       const { InstitutionFlowEngine } = await import('../analysis/institutionFlowEngine.ts');
@@ -410,7 +484,7 @@ const institutionalRouter = router({
       }>;
 
       if (rows.length === 0) {
-        return { symbol, flow: [], quarters: [] };
+        return { symbol, flow: [], quarters: [], filings: [] };
       }
 
       // Group by filer_cik, then by reported_quarter to compute QoQ flow
@@ -468,26 +542,42 @@ const institutionalRouter = router({
       // Collect unique quarters
       const quarters = [...new Set(rows.map(r => r.reported_quarter))].sort().reverse();
 
-      return { symbol, flow: allFlow, quarters };
+      // Return individual filing rows for month-granularity grouping
+      const filings = rows.map(r => ({
+        filerCik: r.filer_cik,
+        filerName: r.filer_name,
+        shares: r.shares ?? 0,
+        valueUsd: r.value_usd ?? 0,
+        reportedQuarter: r.reported_quarter,
+        filedAt: r.filed_at,
+        form: r.form,
+      }));
+
+      return { symbol, flow: allFlow, quarters, filings };
     }),
 
   /** Get insider activity stream for a symbol (M5 quarterly price strip). */
   insiderStream: publicProcedure
     .input(z.object({
       symbol: z.string().min(1),
-      limit: z.number().int().min(1).max(100).optional().default(50),
+      limit: z.number().int().min(1).max(500).optional().default(200),
     }))
     .query(async ({ ctx, input }) => {
       const symbol = input.symbol.toUpperCase();
 
+      // Go back 5 years so the monthly chart has real depth.
+      const fiveYearsAgo = new Date();
+      fiveYearsAgo.setFullYear(fiveYearsAgo.getFullYear() - 5);
+      const since = fiveYearsAgo.toISOString().slice(0, 10);
+
       const transactions = ctx.db.prepare(`
         SELECT form4_id, symbol, insider_name, insider_role, tx_date, tx_code,
                tx_type, shares, price, is_10b5_1, classification, filed_at
         FROM insider_transactions
-        WHERE symbol = ?
+        WHERE symbol = ? AND tx_date >= ?
         ORDER BY tx_date DESC
         LIMIT ?
-      `).all(symbol, input.limit) as Array<{
+      `).all(symbol, since, input.limit) as Array<{
         form4_id: string;
         symbol: string;
         insider_name: string;
@@ -509,11 +599,11 @@ const institutionalRouter = router({
         netShares += shares * direction;
 
         return {
-          insiderName: tx.insider_name,
-          role: tx.insider_role,
-          txDate: tx.tx_date,
-          txCode: tx.tx_code,
-          txType: tx.tx_type,
+          reporter: tx.insider_name,
+          relationship: tx.insider_role,
+          transactionDate: tx.tx_date,
+          transactionCode: tx.tx_code,
+          transactionType: tx.tx_type,
           shares,
           price: tx.price,
           is10b5: tx.is_10b5_1 === 1,
@@ -678,14 +768,8 @@ const watchlistRouter = router({
       const { addSymbol } = await import('../db/watchlistRepository.ts');
       const added = addSymbol(ctx.db, userId, input.symbol);
 
-      // Fire-and-forget: fetch SEC data (13F + Form 4) for this symbol in background.
-      // Don't await — let the response return immediately.
       if (added) {
-        import('../services/secDataFetcher.ts').then(({ fetchAndStoreSecData }) => {
-          fetchAndStoreSecData(ctx.db, input.symbol).catch((e) => {
-            console.error(`[sec fetch] failed for ${input.symbol}:`, e);
-          });
-        }).catch(() => { /* sec fetcher not available */ });
+        queueSecFetch(ctx.db, input.symbol);
       }
 
       return { added };