feat: dealer flow, mirror portfolio (M21), options convexity, FINRA short interest, alert producers, vendor gate

Snapshot of in-progress module work across multiple slices:

- Dealer Flow: dealerExposureEngine, dealerMapService, dealerMapExplain,
  dealerMapIntegrity, dealerMapReplay, dealerStudyEngine, hanStyleLevels
- Mirror Portfolio (M21): fundRepository, captureIngest, mirrorAlertProducers,
  fund holdings strip, live book, position capture ingest
- Options: BSM, NormalizedOptionSurface types, OptionsChainRouter,
  ConvexityGate, option legs panel
- Alert producers: vixLevel, rotation, thesis, unlock, portfolioRisk,
  mirror (fund_capture, fund_13f, mirror_diff)
- FINRA short interest adapter + queue integration
- SEC company tickers adapter + ingest (symbol search index seed)
- Vendor gate (rate-limit-first data plane, ADR-0009)
- CUSIP registry, reverse 13F refresh, stock float service
- LRU cache, portfolio backtest engine
- Frontend: dealer-flow, funds, journal, lab, monitor, plan, portfolio,
  reports, screener, strategies, theses, guided-start, exits, more pages
- Volume profile, workspace profile, visibility-aware poll
- ADRs 0010 (mirror math not advice), 0011 (symbol search index)
- VENDOR_INTEGRATIONS.md, END_USER_TEST.md
- .gitignore: exclude DBs, .DS_Store, local config, agent scratch

app/server/src/index.ts

@@ -5,24 +5,39 @@ import { fetchRequestHandler } from '@trpc/server/adapters/fetch';
 import { db } from './db/client.ts';
 import { createCacheRepository, type SourceKind } from './cache/CacheRepository.ts';
 import { YFinanceAdapter } from './adapters/YFinanceAdapter.ts';
+import { OptionsAdapter } from './adapters/OptionsAdapter.ts';
 import { NasdaqAdapter } from './adapters/NasdaqAdapter.ts';
 import { FinraBulkAdapter } from './adapters/FinraBulkAdapter.ts';
+import { FinraShortInterestAdapter } from './adapters/FinraShortInterestAdapter.ts';
 import { SecFetchAdapter } from './adapters/SecFetchAdapter.ts';
+import { SecCompanyTickersAdapter } from './adapters/SecCompanyTickersAdapter.ts';
 import { SecLintAdapter } from './adapters/SecLintAdapter.ts';
 import { XCookieAdapter } from './adapters/XCookieAdapter.ts';
+import { FredAdapterImpl } from './macro/FredAdapter.ts';
 import type { SourceFetch } from './adapters/SourceAdapter.ts';
+import { composeYFinanceWithOptions } from './options/OptionsChainRouter.ts';
 import cryptoMod from './lib/crypto.ts';
 import { AdapterQueue } from './queue/AdapterQueue.ts';
+import { seedCuratedCusips } from './services/cusipRegistry.ts';
+import { seedAdminDefaultAlertSubscriptions } from './db/alertSubscriptionRepository.ts';
 import { makeCreateContext } from './trpc/context.ts';
 import { appRouter } from './trpc/router.ts';
+import type { Alert } from './alerts/AlertEngine.ts';
 
 const PORT = Number(process.env.PORT ?? 3001);
 const database = db();
+const yfinanceAdapter = composeYFinanceWithOptions(
+  new YFinanceAdapter({ db: database }) as unknown as SourceFetch,
+  new OptionsAdapter() as unknown as SourceFetch,
+);
 const adapters = new Map<SourceKind, SourceFetch>([
-  ['yfinance' as const, new YFinanceAdapter() as unknown as SourceFetch],
+  ['yfinance' as const, yfinanceAdapter],
   ['nasdaq' as const, new NasdaqAdapter() as unknown as SourceFetch],
   ['finra-bulk' as const, new FinraBulkAdapter(database) as unknown as SourceFetch],
+  ['finra-si' as const, new FinraShortInterestAdapter(database) as unknown as SourceFetch],
   ['sec-fetch' as const, new SecFetchAdapter(database) as unknown as SourceFetch],
+  ['sec-sc-fetch' as const, new SecFetchAdapter(database) as unknown as SourceFetch],
+  ['sec-tickers' as const, new SecCompanyTickersAdapter(database) as unknown as SourceFetch],
   ['sec-lint-holders' as const, new SecLintAdapter(() => database, 'sec-lint-holders') as unknown as SourceFetch],
   ['sec-lint-insiders' as const, new SecLintAdapter(() => database, 'sec-lint-insiders') as unknown as SourceFetch],
 ]);
@@ -45,13 +60,59 @@ let xAdapter: XCookieAdapter | null = null;
   console.log('[investor-flow] X adapter registered (credentials configured)');
 })();
 
+// Register FredAdapter when a FRED API key is configured (encrypted in x_credentials).
+// FRED series are warmed by the queue schedule (fred_macro tier), never on the
+// market.condition request path.
+(function initFredAdapter() {
+  const row = database.prepare('SELECT fred_api_key_enc FROM x_credentials WHERE id=?').get('singleton') as { fred_api_key_enc?: string | null } | undefined;
+  if (!row?.fred_api_key_enc) return;
+  let apiKey: string;
+  try { apiKey = cryptoMod.decrypt(row.fred_api_key_enc); } catch { return; }
+  if (!apiKey) return;
+  adapters.set('fred' as const, new FredAdapterImpl(apiKey) as unknown as SourceFetch);
+  console.log('[investor-flow] FRED adapter registered (series warm-up via queue schedule)');
+})();
+
 const queue = new AdapterQueue({ db: database, adapters });
 const cache = createCacheRepository({ db: database, scheduler: queue });
 queue.cache = cache; // break the cache<->scheduler cycle
 
-// Seed default schedules (noop if already seeded)
+// Seed / migrate tiered schedules (safe every boot)
 queue.seedDefaultSchedules();
 
+// Offline CUSIP registry → kv_cache so institutional alerts never depend solely on EFTS.
+try {
+  const seeded = seedCuratedCusips(database);
+  if (seeded > 0) console.log(`[investor-flow] seeded ${seeded} curated sec:cusip cache entries`);
+} catch (e) {
+  console.error('[investor-flow] cusip seed failed', e);
+}
+
+// Admin default alert subscriptions: every catalog alert type ON for is_admin=1
+// (idempotent; respects any toggled preference). Seeded now, not just on the
+// 10-min batch producer tick, so operators receive alerts immediately.
+try {
+  const seeded = seedAdminDefaultAlertSubscriptions(database);
+  if (seeded > 0) console.log(`[investor-flow] seeded ${seeded} admin default alert subscription(s)`);
+} catch (e) {
+  console.error('[investor-flow] admin alert seed failed', e);
+}
+
+// Demand hygiene: junk test symbols + inflated refcounts from page-view subscribe spam
+try {
+  const hygiene = queue.cleanupDemandHygiene();
+  if (hygiene.removedJunk || hygiene.cappedRefcounts) {
+    console.log(`[investor-flow] demand hygiene: removedJunk=${hygiene.removedJunk} cappedRefcounts=${hygiene.cappedRefcounts}`);
+  }
+} catch (e) {
+  console.error('[investor-flow] demand hygiene failed', e);
+}
+
+// Pin rotation universe + SPY + VIX (no refcount inflation)
+queue.pinSystemUniverse().then(() => {
+  console.log('[investor-flow] system pins (rotation universe) ready');
+}).catch((e) => console.error('[investor-flow] pinSystemUniverse failed', e));
+
 // Startup recovery: any job left 'in_flight' was interrupted by a restart/crash.
 // Reset to 'pending' so the drain loop reprocesses it.
 const recovered = database.prepare("UPDATE adapter_queue SET status='pending', error=NULL, retry_count=0 WHERE status='in_flight'").run();
@@ -70,28 +131,49 @@ const SCHEDULE_MS = 30_000;
 const scheduleTimer = setInterval(() => { queue.enqueueDueSchedules().catch((e) => console.error('[schedule error]', e)); }, SCHEDULE_MS);
 scheduleTimer.unref();
 
-// Per-fetch alert check: run alongside the schedule cycle for critical alert types.
+// Realtime + per-fetch alert check: run alongside the schedule cycle for
+// low-latency alert types (e.g. VIX band crosses). Enqueues to the email
+// outbox only; SMTP is drained on its own timer so it never blocks this loop.
 const perFetchAlertTimer = setInterval(async () => {
   try {
-    const alerts = await runProducers(database, 'per-fetch');
+    const alerts: Alert[] = [
+      ...(await runProducers(database, 'realtime')),
+      ...(await runProducers(database, 'per-fetch')),
+    ];
     if (alerts.length > 0) {
-      console.log(`[alert] ${alerts.length} per-fetch alert(s) created`);
-      const { sendAlertEmail } = await import('./services/emailAlertService.ts');
-      alerts.forEach((a) => sendAlertEmail(database, a).catch(() => {}));
+      console.log(`[alert] ${alerts.length} realtime alert(s) created`);
+      const { enqueueAlertEmail } = await import('./services/emailAlertService.ts');
+      alerts.forEach((a) => enqueueAlertEmail(database, a));
     }
   } catch (e) {
-    console.error('[alert] per-fetch check failed:', e);
+    console.error('[alert] realtime check failed:', e);
   }
 }, SCHEDULE_MS);
 perFetchAlertTimer.unref();
 
 // Register alert producers.
-import { registerProducer, runProducers } from './alerts/producers/index.ts';
+import { registerProducer, runProducers, registerMirrorProducers } from './alerts/producers/index.ts';
 import { informedBuyProducer, informedSellProducer } from './alerts/producers/insiderProducer.ts';
-import { new13daProducer } from './alerts/producers/new13daProducer.ts';
+import { new13daProducer, new13fFilingProducer } from './alerts/producers/new13daProducer.ts';
+import { vixLevelProducer } from './alerts/producers/vixLevelProducer.ts';
+import { rotationIncipientProducer, regimeShiftProducer } from './alerts/producers/rotationProducer.ts';
+import { convictionUnlockProducer } from './alerts/producers/unlockProducer.ts';
+import { thesisBrokenProducer, thesisWeakeningProducer } from './alerts/producers/thesisProducer.ts';
+import { clusterBreachProducer, drawdownHaltProducer, asymmetryWarningProducer } from './alerts/producers/portfolioRiskProducer.ts';
 registerProducer(informedBuyProducer);
 registerProducer(informedSellProducer);
 registerProducer(new13daProducer);
+registerProducer(new13fFilingProducer);
+registerProducer(vixLevelProducer);
+registerProducer(rotationIncipientProducer);
+registerProducer(regimeShiftProducer);
+registerProducer(convictionUnlockProducer);
+registerProducer(thesisBrokenProducer);
+registerProducer(thesisWeakeningProducer);
+registerProducer(clusterBreachProducer);
+registerProducer(drawdownHaltProducer);
+registerProducer(asymmetryWarningProducer);
+registerMirrorProducers(); // fund_capture / fund_13f / mirror_diff (lazy, offline-safe)
 
 // Batched alert check: run every 10 minutes for non-critical producers.
 const ALERT_BATCH_MS = 10 * 60 * 1000;
@@ -99,17 +181,45 @@ const alertBatchTimer = setInterval(async () => {
   try {
     const alerts = await runProducers(database, 'batched');
     if (alerts.length > 0) console.log(`[alert] ${alerts.length} batch alert(s) created`);
-    // Send email for each new alert.
-    const { sendAlertEmail } = await import('./services/emailAlertService.ts');
-    for (const alert of alerts) {
-      sendAlertEmail(database, alert).catch((e) => console.error('[alert:email] send error:', e));
-    }
+    // Enqueue email for each new alert (SMTP drained separately).
+    const { enqueueAlertEmail } = await import('./services/emailAlertService.ts');
+    for (const alert of alerts) enqueueAlertEmail(database, alert);
   } catch (e) {
     console.error('[alert] batch check failed:', e);
   }
 }, ALERT_BATCH_MS);
 alertBatchTimer.unref();
 
+// Email outbox drain: decouples SMTP latency/errors from the alert loops.
+const OUTBOX_DRAIN_MS = 60_000;
+const outboxDrainTimer = setInterval(async () => {
+  try {
+    const { drainEmailOutbox } = await import('./services/emailAlertService.ts');
+    const sent = await drainEmailOutbox(database);
+    if (sent > 0) console.log(`[alert:email] drained ${sent} outbox email(s)`);
+  } catch (e) {
+    console.error('[alert:email] outbox drain failed:', e);
+  }
+}, OUTBOX_DRAIN_MS);
+outboxDrainTimer.unref();
+
+// Daily queue housekeeping: purge stale 'done' rows and age out queue_errors.
+// Both tables previously grew unbounded (live DB had 819k error rows).
+const DAILY_HOUSEKEEP_MS = 24 * 60 * 60 * 1000;
+const housekeepTimer = setInterval(() => {
+  try {
+    const cleared = queue.clearDone();
+    const pruned = queue.pruneQueueErrors();
+    if (cleared > 0 || pruned > 0) {
+      console.log(`[queue] housekeeping: cleared ${cleared} done jobs, pruned ${pruned} old queue_errors`);
+    }
+  } catch (e) {
+    console.error('[queue] daily housekeeping failed:', e);
+  }
+}, DAILY_HOUSEKEEP_MS);
+housekeepTimer.unref();
+
+
 function readBody(req: IncomingMessage): Promise<string> {
   return new Promise((resolve, reject) => {
     let data = '';