feat: dealer flow, mirror portfolio (M21), options convexity, FINRA short interest, alert producers, vendor gate

Snapshot of in-progress module work across multiple slices: - Dealer Flow: dealerExposureEngine, dealerMapService, dealerMapExplain, dealerMapIntegrity, dealerMapReplay, dealerStudyEngine, hanStyleLevels - Mirror Portfolio (M21): fundRepository, captureIngest, mirrorAlertProducers, fund holdings strip, live book, position capture ingest - Options: BSM, NormalizedOptionSurface types, OptionsChainRouter, ConvexityGate, option legs panel - Alert producers: vixLevel, rotation, thesis, unlock, portfolioRisk, mirror (fund_capture, fund_13f, mirror_diff) - FINRA short interest adapter + queue integration - SEC company tickers adapter + ingest (symbol search index seed) - Vendor gate (rate-limit-first data plane, ADR-0009) - CUSIP registry, reverse 13F refresh, stock float service - LRU cache, portfolio backtest engine - Frontend: dealer-flow, funds, journal, lab, monitor, plan, portfolio, reports, screener, strategies, theses, guided-start, exits, more pages - Volume profile, workspace profile, visibility-aware poll - ADRs 0010 (mirror math not advice), 0011 (symbol search index) - VENDOR_INTEGRATIONS.md, END_USER_TEST.md - .gitignore: exclude DBs, .DS_Store, local config, agent scratch
2026-08-10 13:36:26 -04:00
parent 04fc11b2fd
commit ac94acf9e3
229 changed files with 32617 additions and 3934 deletions
@@ -0,0 +1,77 @@
+#!/usr/bin/env npx tsx
+/**
+ * CLI: backfill stock_float table with sharesOutstanding / floatShares from Yahoo Finance.
+ *
+ * Usage (from app/server):
+ *   npx tsx scripts/backfill-stock-float.ts
+ *   npx tsx scripts/backfill-stock-float.ts --symbol IREN    # single symbol
+ *   npx tsx scripts/backfill-stock-float.ts --all             # all symbols in DB
+ *
+ * Reads tracked fund positions and 13F filings to find symbols needing float data.
+ * Respects vendor rate limits; skips entries fetched within the last 24h.
+ */
+
+import { DatabaseSync } from 'node:sqlite';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { fetchAndPersistFloat, refreshAllStockFloats } from '../src/services/stockFloatService.ts';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SERVER_ROOT = join(__dirname, '..');
+const DB_PATH = resolve(SERVER_ROOT, 'data', 'investor-flow.db');
+
+function arg(name: string): string | undefined {
+  const i = process.argv.indexOf(name);
+  if (i < 0 || i + 1 >= process.argv.length) return undefined;
+  return process.argv[i + 1];
+}
+
+async function main() {
+  const symbolArg = arg('--symbol');
+  const forceFlag = arg('--force') === 'true' || arg('--force') === '';
+
+  console.log('Backfilling stock float data from Yahoo Finance...');
+  console.log(`DB: ${DB_PATH}`);
+  if (symbolArg) console.log(`Symbol filter: ${symbolArg.toUpperCase()}`);
+  if (forceFlag && !symbolArg) console.log('Mode: force refresh all symbols');
+
+  const db = new DatabaseSync(DB_PATH);
+
+  try {
+    let result;
+    if (symbolArg) {
+      // Single symbol backfill.
+      console.log(`Fetching float for ${symbolArg.toUpperCase()}...`);
+      await fetchAndPersistFloat(db, symbolArg.toUpperCase());
+      const row = db.prepare(
+        `SELECT shares_outstanding, float_shares FROM stock_float WHERE symbol = ? ORDER BY as_of DESC LIMIT 1`
+      ).get(symbolArg.toUpperCase()) as { shares_outstanding: number | null; float_shares: number | null } | undefined;
+
+      console.log(`  sharesOutstanding: ${row?.shares_outstanding ?? 'null'}`);
+      console.log(`  floatShares:       ${row?.float_shares ?? 'null'}`);
+      result = { refreshed: 1, skipped: 0, errors: [] };
+    } else if (forceFlag) {
+      // Force refresh all symbols regardless of TTL.
+      result = await refreshAllStockFloats(db, { forceRefresh: true });
+    } else {
+      // Default: refresh stale entries only.
+      result = await refreshAllStockFloats(db);
+    }
+
+    console.log('\nDone.');
+    console.log(`  Refreshed: ${result.refreshed}`);
+    console.log(`  Skipped (fresh): ${result.skipped}`);
+    if (result.errors.length > 0) {
+      console.log(`\nErrors (${result.errors.length}):`);
+      for (const err of result.errors.slice(0, 10)) console.log(`  - ${err}`);
+      if (result.errors.length > 10) console.log(`  ... and ${result.errors.length - 10} more`);
+    }
+  } finally {
+    db.close();
+  }
+}
+
+main().catch((err) => {
+  console.error('Backfill failed:', err);
+  process.exit(1);
+});
@@ -0,0 +1,293 @@
+#!/usr/bin/env node
+/**
+ * Distill harvested raw posts into:
+ *   1) In-app notes: src/analysis/dealerFlowExplainNotes.ts  (app reads this)
+ *   2) Obsidian pages under your DealerFlow vault           (you read this; app does not)
+ *
+ * Does not call the LLM by default - extracts plain-English candidate lines and
+ * merges into the existing notes structure. Re-run after harvest.
+ *
+ * Usage (from app/server):
+ *   node --experimental-strip-types scripts/dealer-flow-distill.ts
+ */
+
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SERVER_ROOT = join(__dirname, '..');
+const DATA = join(SERVER_ROOT, 'data');
+const SOURCES_PATH = join(SERVER_ROOT, 'config', 'dealer-flow-sources.json');
+const RAW_ROOT = join(DATA, 'dealer-flow-raw');
+const NOTES_OUT = join(SERVER_ROOT, 'src/analysis/dealerFlowExplainNotes.ts');
+
+interface SourcesFile {
+  obsidianVault?: string;
+}
+
+function loadSources(): SourcesFile {
+  if (!existsSync(SOURCES_PATH)) return {};
+  return JSON.parse(readFileSync(SOURCES_PATH, 'utf8')) as SourcesFile;
+}
+
+function walkMd(dir: string, acc: string[] = []): string[] {
+  if (!existsSync(dir)) return acc;
+  for (const name of readdirSync(dir, { withFileTypes: true })) {
+    const p = join(dir, name.name);
+    if (name.isDirectory()) walkMd(p, acc);
+    else if (name.name.endsWith('.md')) acc.push(p);
+  }
+  return acc;
+}
+
+function stripFrontmatter(md: string): string {
+  if (!md.startsWith('---')) return md;
+  const end = md.indexOf('\n---', 3);
+  if (end < 0) return md;
+  return md.slice(end + 4);
+}
+
+/** Lines that sound like trade advice - never enter product notes (Primary Rule). */
+const DIRECTIVE =
+  /\b(buy|sell|long|short|puts?|calls?|grab|grabs|entry|entries|entering|enter|aim for|i wait|target|targets|stop|stops|size|sizing|subs?|subscribe|trade idea|trading rules|your long|your short|go long|go short|load up|trim|add here|looking good|good returns|bounce till|try a bounce|sure bet|shorted|try for|predict anything|tough to say|poor asymmetry|offers poor)\b/i;
+
+/** Strip third-party product names from product-facing snippets. */
+function sanitizeSnippet(s: string): string {
+  return s
+    .replace(/\bHeatseeker\b/gi, 'DealerFlow')
+    .replace(/\bPika\b/gi, '')
+    .replace(/\s{2,}/g, ' ')
+    .trim();
+}
+
+/** Prefer mechanics / definition language (education threads). */
+const MECHANICS =
+  /\b(node|nodes|dealer|hedg|dampen|amplif|mean reversion|momentum|yellow|purple|calm zone|no control|attractor|accelerant|vanna|positive gamma|negative gamma|long gamma|short gamma|concentration|stabiliz|pinning|air pocket|desert|flip|king is|what .* means|how .* work)\b/i;
+
+/** Pull short plain-looking lines that might help explain maps (heuristic, not ML). */
+function extractSnippets(text: string): string[] {
+  const body = stripFrontmatter(text).replace(/\s+/g, ' ').trim();
+  if (body.length < 40) return [];
+  const sentences = body.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter(Boolean);
+  const out: string[] = [];
+  const keywords =
+    /\b(gamma|gex|vex|vanna|pin|pinning|wall|strike|expiry|opex|dealer|hedge|hedging|open interest|flip|king|magnet|cluster|air pocket|desert|stabilize|amplif|mean reversion|momentum|volatility|positive gamma|negative gamma|long gamma|short gamma|node|nodes)\b/i;
+  for (const s of sentences) {
+    if (s.length < 45 || s.length > 260) continue;
+    if (!keywords.test(s)) continue;
+    if (DIRECTIVE.test(s)) continue;
+    // Skip ticker-heavy or price-level callouts (app uses live map numbers)
+    if ((s.match(/\$[A-Z]{1,5}\b/g) ?? []).length >= 1) continue;
+    if (/\$\d/.test(s) || /\b\d{3,5}\b/.test(s)) continue;
+    // Prefer education mechanics; deprioritize bare "VEX …" market chatter
+    if (/^VEX\b/i.test(s) && !MECHANICS.test(s)) continue;
+    if (!MECHANICS.test(s) && !/\b(means|work as|acting as|promotes|dampen|amplif|gravitational|exposure)\b/i.test(s)) {
+      continue;
+    }
+    // Skip live session chatter / unhelpful fragments
+    if (/\bMINS IN\b/i.test(s) || /^Update on nodes:/i.test(s)) continue;
+    out.push(sanitizeSnippet(s.replace(/["']/g, "'")));
+  }
+  return out;
+}
+
+function uniqueSnippets(all: string[], max = 20): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const s of all) {
+    const key = s.toLowerCase().slice(0, 80);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(s);
+    if (out.length >= max) break;
+  }
+  return out;
+}
+
+function buildNotesTs(snippets: string[], version: string, updatedAt: string): string {
+  const snippetsLit = JSON.stringify(snippets, null, 2);
+  return `/**
+ * In-app plain-language notes for Dealer Flow explains.
+ * Runtime reads THIS file only - never Obsidian.
+ *
+ * Regenerated by: \`node --experimental-strip-types scripts/dealer-flow-distill.ts\`
+ * (distill also writes a personal copy into your Obsidian vault for offline reading).
+ */
+
+export interface DealerFlowExplainNotes {
+  /** Bump when regenerating so L1 cache invalidates. */
+  version: string;
+  updatedAt: string;
+  /** One sentence: what the picture is. */
+  whatMapIs: string;
+  plainMetaphors: {
+    cushion: string;
+    spring: string;
+    mixed: string;
+    magnet: string;
+    floor: string;
+    ceiling: string;
+    thinZone: string;
+  };
+  levels: {
+    king: string;
+    flip: string;
+    callWall: string;
+    putWall: string;
+    airPocket: string;
+  };
+  scenarios: Record<string, string>;
+  patterns: Record<string, string>;
+  caveats: string[];
+  forbidden: string[];
+  /** Optional snippets distilled from harvested sources (plain English only). */
+  sourceSnippets: string[];
+}
+
+export const DEALER_FLOW_EXPLAIN_NOTES: DealerFlowExplainNotes = {
+  version: ${JSON.stringify(version)},
+  updatedAt: ${JSON.stringify(updatedAt)},
+  whatMapIs:
+    'This picture shows modeled hedging pressure from options open interest across prices and dates - not a forecast of where the stock must go, and not a live list of any firm\\'s inventory.',
+  plainMetaphors: {
+    cushion:
+      'Cushion: near this area, modeled hedges tend to lean against moves, so big swings may get pushed back toward the middle.',
+    spring:
+      'Spring: near this area, modeled hedges tend to lean with moves, so once price starts moving it can keep going.',
+    mixed: 'Mixed or thin: neither a clear cushion nor a clear spring near today\\'s price.',
+    magnet: 'Magnet: a loud strike the map cares about more than quieter levels nearby.',
+    floor: 'Floor people watch: heavy modeled pressure below the market - not a guarantee price bounces.',
+    ceiling: 'Ceiling people watch: heavy modeled pressure above the market - not a guarantee price stops.',
+    thinZone: 'Thin zone: little modeled pressure - price may travel through with less mechanical friction in the model.',
+  },
+  levels: {
+    king: 'King is the loudest strike on this map (largest absolute modeled pressure).',
+    flip: 'Flip is where the map\\'s personality can switch between cushion and spring territory.',
+    callWall: 'Call wall is a heavy modeled pile above spot - a ceiling people watch.',
+    putWall: 'Put wall is a heavy modeled pile below spot - a floor people watch.',
+    airPocket: 'Air pocket is a thin band of pressure between stronger levels.',
+  },
+  scenarios: {
+    pos_gex_pos_vex:
+      'Reads relatively calm: both price-linked and vol-linked pieces tend to absorb shocks in the model.',
+    neg_gex_neg_vex:
+      'Reads jumpy: price and vol can reinforce each other in the model - swings may feel larger.',
+    gex_dominates: 'Price-level magnets matter more than volatility for this snapshot - start with king and walls.',
+    vex_dominates: 'Volatility shifts may matter more than pin levels for this snapshot.',
+    mixed: 'Signals disagree - treat the map as context, not one clean story.',
+    insufficient_data: 'Not enough open-interest data yet to label a clear scenario.',
+  },
+  patterns: {
+    whipsaw:
+      'Whipsaw shape: two strong magnets with a thinner middle - price can bounce between the poles.',
+    rainbow_road:
+      'Rainbow-road shape: pressure spread across a wide range without one dominant magnet - more wandering.',
+    rug_setup:
+      'Rug-like stack: support-looking structure that can fail and speed moves lower in the model if the soft floor breaks - still not a trade call.',
+  },
+  caveats: [
+    'Quotes may be delayed; open interest is often from the prior session.',
+    'Numbers are modeled dollar hedging pressure, not reported dealer inventory.',
+    'This explains a snapshot, not a price prediction.',
+  ],
+  forbidden: [
+    'Do not say buy, sell, short, or long as instructions.',
+    'Do not give position size, entries, stops, or targets as recommendations.',
+    'Do not claim the map knows where price must go.',
+  ],
+  sourceSnippets: ${snippetsLit},
+};
+
+/** Compact block for LLM system prompts. */
+export function explainNotesForPrompt(notes: DealerFlowExplainNotes = DEALER_FLOW_EXPLAIN_NOTES): string {
+  const lines = [
+    \`version: \${notes.version}\`,
+    \`whatMapIs: \${notes.whatMapIs}\`,
+    \`metaphors: \${Object.values(notes.plainMetaphors).join(' | ')}\`,
+    \`levels: \${Object.values(notes.levels).join(' | ')}\`,
+    \`scenarios: \${Object.entries(notes.scenarios)
+      .map(([k, v]) => \`\${k}=\${v}\`)
+      .join(' | ')}\`,
+    \`patterns: \${Object.entries(notes.patterns)
+      .map(([k, v]) => \`\${k}=\${v}\`)
+      .join(' | ')}\`,
+    \`caveats: \${notes.caveats.join(' | ')}\`,
+    \`forbidden: \${notes.forbidden.join(' | ')}\`,
+  ];
+  if (notes.sourceSnippets.length > 0) {
+    lines.push(
+      'extra plain-English snippets from distilled sources:',
+      ...notes.sourceSnippets.slice(0, 12).map((s) => \`- \${s}\`),
+    );
+  }
+  return lines.join('\\n');
+}
+`;
+}
+
+function writeObsidianPages(vault: string, snippets: string[], version: string): void {
+  const pages = join(vault, 'pages');
+  mkdirSync(pages, { recursive: true });
+
+  const glossary = `# DealerFlow glossary (auto-distilled shell)
+
+Last distill: ${version}
+
+## What the map is
+
+This picture shows modeled hedging pressure from options open interest - not a forecast and not anyone's live inventory.
+
+## Everyday words
+
+- **Cushion** - hedges lean against moves
+- **Spring** - hedges lean with moves
+- **Magnet** - loud strike the map cares about
+- **Floor / ceiling** - heavy piles people watch below / above
+- **Thin zone** - little pressure between stronger levels
+
+## From harvested sources (review & edit)
+
+${snippets.length ? snippets.map((s) => `- ${s}`).join('\n') : '_No keyword snippets yet. Run harvest with handles first._'}
+
+---
+Edit freely. The **app does not read this file**. After you improve wording, re-run distill or paste good lines into in-app notes.
+`;
+
+  writeFileSync(join(pages, 'glossary.md'), glossary, 'utf8');
+  writeFileSync(
+    join(pages, 'source-snippets.md'),
+    `# Source snippets (personal)\n\nVersion ${version}\n\n${snippets.map((s) => `- ${s}`).join('\n') || '_empty_'}\n`,
+    'utf8',
+  );
+}
+
+function main() {
+  const sources = loadSources();
+  const files = walkMd(RAW_ROOT);
+  const all: string[] = [];
+  for (const f of files) {
+    try {
+      all.push(...extractSnippets(readFileSync(f, 'utf8')));
+    } catch {
+      /* skip */
+    }
+  }
+  const snippets = uniqueSnippets(all, 20);
+  const updatedAt = new Date().toISOString();
+  const version = updatedAt.slice(0, 10) + '.' + String(snippets.length);
+
+  writeFileSync(NOTES_OUT, buildNotesTs(snippets, version, updatedAt), 'utf8');
+  console.log(`[distill] wrote in-app notes → ${NOTES_OUT} (${snippets.length} source snippets)`);
+
+  const vault = sources.obsidianVault?.trim();
+  if (vault) {
+    mkdirSync(vault, { recursive: true });
+    writeObsidianPages(vault, snippets, version);
+    console.log(`[distill] wrote personal Obsidian pages under ${vault}/pages/ (app does not read)`);
+  } else {
+    console.log('[distill] no obsidianVault in dealer-flow-sources.json - skipped vault write');
+  }
+}
+
+main();
@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+/**
+ * Harvest X posts for Dealer Flow knowledge (offline).
+ *
+ * Writes ONLY to:
+ *   - app/server/data/dealer-flow-raw/   (gitignored via data/)
+ *   - optional Obsidian vault raw/by-handle/  (personal notebook; app never reads)
+ *
+ * Usage (from app/server):
+ *   node --experimental-strip-types scripts/dealer-flow-harvest.ts
+ *   node --experimental-strip-types scripts/dealer-flow-harvest.ts --handle someuser --count 40
+ *
+ * Auth: uses bird with cookies from x_credentials in investor-flow.db when available,
+ * or bird's own browser cookie extraction.
+ */
+
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { DatabaseSync } from 'node:sqlite';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SERVER_ROOT = join(__dirname, '..');
+const DATA = join(SERVER_ROOT, 'data');
+const SOURCES_PATH = join(SERVER_ROOT, 'config', 'dealer-flow-sources.json');
+const RAW_ROOT = join(DATA, 'dealer-flow-raw');
+const DB_PATH = join(DATA, 'investor-flow.db');
+
+interface SourcesFile {
+  obsidianVault?: string;
+  handles: string[];
+  threads?: string[];
+  notes?: string;
+}
+
+function loadSources(): SourcesFile {
+  if (!existsSync(SOURCES_PATH)) {
+    return { handles: [], threads: [] };
+  }
+  return JSON.parse(readFileSync(SOURCES_PATH, 'utf8')) as SourcesFile;
+}
+
+async function decryptCreds(): Promise<{ authToken: string; ct0: string } | null> {
+  if (!existsSync(DB_PATH)) return null;
+  try {
+    const { default: cryptoMod } = await import('../src/lib/crypto.ts');
+    const db = new DatabaseSync(DB_PATH, { readOnly: true });
+    const row = db.prepare(
+      "SELECT ct0_enc, auth_token_enc FROM x_credentials WHERE id='singleton'",
+    ).get() as { ct0_enc?: string; auth_token_enc?: string } | undefined;
+    db.close();
+    if (!row?.ct0_enc || !row?.auth_token_enc) return null;
+    return {
+      ct0: cryptoMod.decrypt(row.ct0_enc),
+      authToken: cryptoMod.decrypt(row.auth_token_enc),
+    };
+  } catch (e) {
+    console.warn('[harvest] could not load x_credentials:', e instanceof Error ? e.message : e);
+    return null;
+  }
+}
+
+function birdJson(
+  args: string[],
+  creds: { authToken: string; ct0: string } | null,
+): unknown {
+  const full = [...args, '--json'];
+  if (creds) {
+    full.push('--auth-token', creds.authToken, '--ct0', creds.ct0);
+  }
+  const r = spawnSync('bird', full, {
+    encoding: 'utf8',
+    maxBuffer: 20 * 1024 * 1024,
+    timeout: 120_000,
+  });
+  if (r.status !== 0) {
+    throw new Error(r.stderr || r.stdout || `bird exit ${r.status}`);
+  }
+  const out = (r.stdout || '').trim();
+  if (!out) return null;
+  return JSON.parse(out);
+}
+
+function writePost(
+  handle: string,
+  post: { id?: string; text?: string; createdAt?: string; url?: string },
+  vault?: string,
+): void {
+  const id = post.id ?? String(Date.now());
+  const date = (post.createdAt ?? new Date().toISOString()).slice(0, 10);
+  const body = [
+    '---',
+    `handle: "${handle}"`,
+    `post_id: "${id}"`,
+    `date: ${date}`,
+    `url: "${post.url ?? ''}"`,
+    'tags: []',
+    '---',
+    '',
+    post.text ?? '',
+    '',
+  ].join('\n');
+
+  const localDir = join(RAW_ROOT, 'by-handle', handle);
+  mkdirSync(localDir, { recursive: true });
+  const fname = `${date}_${id}.md`;
+  writeFileSync(join(localDir, fname), body, 'utf8');
+
+  if (vault) {
+    const vDir = join(vault, 'raw', 'by-handle', handle);
+    mkdirSync(vDir, { recursive: true });
+    writeFileSync(join(vDir, fname), body, 'utf8');
+  }
+}
+
+function parseArgs(argv: string[]) {
+  const out: { handle?: string; count: number; thread?: string; maxPages: number } = {
+    count: 50,
+    maxPages: 5,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--handle') out.handle = argv[++i]?.replace(/^@/, '');
+    else if (argv[i] === '--count') out.count = Number(argv[++i] ?? 50);
+    else if (argv[i] === '--thread') out.thread = argv[++i];
+    else if (argv[i] === '--max-pages') out.maxPages = Number(argv[++i] ?? 5);
+  }
+  return out;
+}
+
+function normalizeTweets(json: unknown): Array<{ id?: string; text?: string; createdAt?: string; url?: string }> {
+  if (!json) return [];
+  if (Array.isArray(json)) {
+    return json.map((t) => {
+      const o = t as Record<string, unknown>;
+      return {
+        id: String(o.id ?? o.id_str ?? o.rest_id ?? ''),
+        text: String(o.text ?? o.full_text ?? o.body ?? ''),
+        createdAt: String(o.createdAt ?? o.created_at ?? ''),
+        url: o.url ? String(o.url) : undefined,
+      };
+    });
+  }
+  if (typeof json === 'object') {
+    const o = json as Record<string, unknown>;
+    if (Array.isArray(o.tweets)) return normalizeTweets(o.tweets);
+    if (Array.isArray(o.results)) return normalizeTweets(o.results);
+    if (Array.isArray(o.data)) return normalizeTweets(o.data);
+  }
+  return [];
+}
+
+async function main() {
+  const cli = parseArgs(process.argv.slice(2));
+  const sources = loadSources();
+  const handles = cli.handle
+    ? [cli.handle]
+    : (sources.handles ?? []).map((h) => h.replace(/^@/, '').trim()).filter(Boolean);
+  const threads = cli.thread ? [cli.thread] : (sources.threads ?? []);
+  const vault = sources.obsidianVault?.trim() || undefined;
+
+  if (handles.length === 0 && threads.length === 0) {
+    console.error(
+      'No handles or threads. Edit data/dealer-flow-sources.json or pass --handle / --thread.',
+    );
+    process.exit(1);
+  }
+
+  mkdirSync(RAW_ROOT, { recursive: true });
+  const creds = await decryptCreds();
+  if (creds) console.log('[harvest] using x_credentials from DB');
+  else console.log('[harvest] no DB cookies - bird will use its own cookie source if configured');
+
+  let written = 0;
+
+  for (const handle of handles) {
+    console.log(`[harvest] user-tweets @${handle} n=${cli.count} max-pages=${cli.maxPages}`);
+    try {
+      const json = birdJson(
+        [
+          'user-tweets',
+          handle,
+          '-n',
+          String(cli.count),
+          '--max-pages',
+          String(cli.maxPages),
+          '--delay',
+          '1500',
+        ],
+        creds,
+      );
+      const tweets = normalizeTweets(json);
+      for (const t of tweets) {
+        if (!t.text) continue;
+        writePost(handle, t, vault);
+        written++;
+      }
+      console.log(`[harvest] @${handle}: ${tweets.length} posts`);
+    } catch (e) {
+      console.error(`[harvest] @${handle} failed:`, e instanceof Error ? e.message : e);
+    }
+  }
+
+  for (const thread of threads) {
+    console.log(`[harvest] thread ${thread}`);
+    try {
+      const json = birdJson(['thread', thread, '--all'], creds);
+      const tweets = normalizeTweets(json);
+      // unknown author - put under _threads
+      for (const t of tweets) {
+        if (!t.text) continue;
+        writePost('_threads', t, vault);
+        written++;
+      }
+      console.log(`[harvest] thread: ${tweets.length} posts`);
+    } catch (e) {
+      console.error(`[harvest] thread failed:`, e instanceof Error ? e.message : e);
+    }
+  }
+
+  // Touch sources.md in vault if present
+  if (vault && existsSync(vault)) {
+    const sourcesMd = join(vault, 'sources.md');
+    // leave existing; just ensure directory
+    mkdirSync(join(vault, 'raw', 'by-handle'), { recursive: true });
+    if (!existsSync(sourcesMd)) {
+      writeFileSync(
+        sourcesMd,
+        '# Sources\n\nUpdated by dealer-flow-harvest. Edit handles in app `data/dealer-flow-sources.json`.\n',
+        'utf8',
+      );
+    }
+  }
+
+  console.log(`[harvest] done. wrote ${written} post files under ${RAW_ROOT}`);
+  if (vault) console.log(`[harvest] also mirrored under ${vault}/raw/by-handle/ (personal; app does not read)`);
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});
@@ -0,0 +1,84 @@
+#!/usr/bin/env npx tsx
+/**
+ * CLI: backtest GEX/VEX integrity by replaying chains as-of a timestamp.
+ *
+ *   npx tsx scripts/dealer-map-replay.ts --symbol IREN --as-of 2026-08-07T13:14:00.000Z
+ *   npx tsx scripts/dealer-map-replay.ts --symbol IREN --from-snapshots --days 7
+ *
+ * Exit 1 if integrity incomplete or snapshot drift too large.
+ */
+
+import { DatabaseSync } from 'node:sqlite';
+import { resolve } from 'node:path';
+import { replayDealerMapAt, replayFromSnapshots } from '../src/analysis/dealerMapReplay.ts';
+
+function arg(name: string): string | undefined {
+  const i = process.argv.indexOf(name);
+  if (i < 0 || i + 1 >= process.argv.length) return undefined;
+  return process.argv[i + 1];
+}
+
+function has(name: string): boolean {
+  return process.argv.includes(name);
+}
+
+const symbol = (arg('--symbol') ?? 'IREN').toUpperCase();
+const asOf = arg('--as-of');
+const days = Number(arg('--days') ?? '7');
+const fromSnapshots = has('--from-snapshots');
+
+const dbPath =
+  arg('--db') ??
+  resolve(process.cwd(), 'data/investor-flow.db');
+
+const db = new DatabaseSync(dbPath, { readOnly: true });
+
+function printResult(r: ReturnType<typeof replayDealerMapAt>) {
+  console.log('---');
+  console.log(r.symbol, r.asOf);
+  console.log('  ok:', r.ok, '|', r.integrity.status);
+  console.log('  ', r.summary);
+  if (r.map?.levels.king) {
+    console.log(
+      '  king:',
+      r.map.levels.king.strike,
+      r.map.levels.king.expiry,
+      `gex=${(r.map.levels.king.gex / 1000).toFixed(1)}K`,
+    );
+  }
+  console.log(
+    '  metrics:',
+    JSON.stringify(r.integrity.metrics),
+  );
+  const fails = r.integrity.checks.filter((c) => !c.ok);
+  if (fails.length) {
+    console.log('  failed checks:');
+    for (const c of fails) {
+      console.log(`    [${c.level}] ${c.id}: ${c.detail}`);
+    }
+  }
+}
+
+let anyFail = false;
+
+if (fromSnapshots) {
+  const results = replayFromSnapshots(db, symbol, days);
+  if (results.length === 0) {
+    console.error('No snapshots for', symbol);
+    process.exit(2);
+  }
+  for (const r of results) {
+    printResult(r);
+    if (!r.ok) anyFail = true;
+  }
+} else if (asOf) {
+  const r = replayDealerMapAt(db, symbol, asOf);
+  printResult(r);
+  if (!r.ok) anyFail = true;
+} else {
+  console.error('Usage: --symbol SYM --as-of ISO | --from-snapshots [--days N] [--db path]');
+  process.exit(2);
+}
+
+db.close();
+process.exit(anyFail ? 1 : 0);
@@ -0,0 +1,150 @@
+// One-shot: refresh IREN options surface, rebuild dealer map, diff vs Heatseeker snapshot.
+import { db as openDb } from '../src/db/client.ts';
+import { createCacheRepository, type SourceKind } from '../src/cache/CacheRepository.ts';
+import { AdapterQueue } from '../src/queue/AdapterQueue.ts';
+import { composeYFinanceWithOptions } from '../src/options/OptionsChainRouter.ts';
+import { YFinanceAdapter } from '../src/adapters/YFinanceAdapter.ts';
+import { OptionsAdapter } from '../src/adapters/OptionsAdapter.ts';
+import type { SourceFetch } from '../src/adapters/SourceAdapter.ts';
+import { clearVendorRateLimit } from '../src/services/vendorGate.ts';
+import {
+  getDealerMap,
+  recomputeDealerMapFromCacheDetailed,
+  scheduleDealerMapInputs,
+} from '../src/analysis/dealerMapService.ts';
+
+const database = openDb();
+clearVendorRateLimit('yfinance');
+for (const k of database
+  .prepare("SELECT key FROM queue_state WHERE key LIKE 'source_cooldown%yfinance%' OR key LIKE 'source_cooldown:yfinance%'")
+  .all() as Array<{ key: string }>) {
+  database.prepare('DELETE FROM queue_state WHERE key=?').run(k.key);
+}
+
+const yfinanceAdapter = composeYFinanceWithOptions(
+  new YFinanceAdapter({ db: database }) as unknown as SourceFetch,
+  new OptionsAdapter() as unknown as SourceFetch,
+);
+const adapters = new Map<SourceKind, SourceFetch>([['yfinance' as const, yfinanceAdapter]]);
+const queue = new AdapterQueue({ db: database, adapters, rateLimitMs: { yfinance: 0 } });
+const cache = createCacheRepository({ db: database, scheduler: queue });
+queue.cache = cache;
+const deps = { db: database, cache, queue };
+
+console.log('1) schedule quote + chains for IREN...');
+await scheduleDealerMapInputs(deps, 'IREN', 6);
+
+console.log('2) drain until chains land (max ~120s)...');
+const t0 = Date.now();
+let rounds = 0;
+while (Date.now() - t0 < 120_000) {
+  await queue.drain();
+  rounds += 1;
+  const chains = database
+    .prepare(
+      `SELECT expiry, COUNT(*) n, MAX(ts) ts FROM options_chains
+       WHERE symbol='IREN' AND expiry >= date('now') GROUP BY expiry ORDER BY expiry LIMIT 8`,
+    )
+    .all() as Array<{ expiry: string; n: number; ts: string }>;
+  const quote = database.prepare(`SELECT observed_at FROM kv_cache WHERE key='yfinance:quote:IREN'`).get() as
+    | { observed_at: string }
+    | undefined;
+  if (rounds % 4 === 0) {
+    console.log(
+      ' round',
+      rounds,
+      'expiries',
+      chains.map((c) => `${c.expiry}:${c.n}`).join(','),
+      'quote',
+      quote?.observed_at?.slice(0, 19),
+    );
+  }
+  const fresh = chains.filter((c) => c.ts && Date.now() - Date.parse(c.ts) < 45 * 60_000);
+  if (fresh.length >= 4 && quote && Date.now() - Date.parse(quote.observed_at) < 45 * 60_000) break;
+  await new Promise((r) => setTimeout(r, 1200));
+}
+
+const detail = recomputeDealerMapFromCacheDetailed(database, 'IREN', { maxExpiries: 6, write: true });
+console.log('3) recompute', {
+  asOf: detail.map?.asOf,
+  spot: detail.map?.spot,
+  king: detail.map?.levels?.king,
+  keptPrior: detail.keptPriorMap,
+  msg: detail.message,
+  integrity: detail.integrity?.status,
+});
+
+const result = await getDealerMap(deps, 'IREN', { forceSchedule: true, maxExpiries: 6 });
+const m = result.map ?? detail.map;
+if (!m) {
+  console.error('NO MAP after rebuild');
+  process.exit(1);
+}
+
+const fmt = (n: number) => {
+  const k = n / 1000;
+  if (Math.abs(k) >= 1000) return `$${(k / 1000).toFixed(1)}M`;
+  return `$${k.toFixed(1)}K`;
+};
+
+// Heatseeker LIVE ~03:45:52 PM, IREN 40.41, GEX 0DTE column from screenshot
+const hs0: Record<number, { v: number; label: string }> = {
+  48: { v: 742_400, label: '+742.4K' },
+  45: { v: 231_800, label: '+231.8K' },
+  43: { v: 284_600, label: '+284.6K' },
+  42: { v: -455_900, label: '-455.9K' },
+  41.5: { v: 288_300, label: '+288.3K' },
+  41: { v: -605_900, label: '-605.9K' },
+  40.5: { v: 3_192_100, label: '+3.19M' },
+  40: { v: -5_735_300, label: '-5.74M ★' },
+  39.5: { v: 2_021_300, label: '+2.02M' },
+  39: { v: -37_200, label: '-37.2K' },
+  38: { v: 1_502_600, label: '+1.50M' },
+  37.5: { v: 567_400, label: '+567K' },
+  37: { v: 638_000, label: '+638K' },
+  34: { v: -1_402_300, label: '-1.40M' },
+};
+
+console.log('\n=== LIVE rebuild vs HS 0DTE (HS @40.41 ~3:45pm ET) ===');
+console.log('OUR spot', m.spot, 'asOf', m.asOf, 'status', result.status);
+console.log(
+  'strike'.padStart(6),
+  'HS'.padStart(12),
+  'OUR_0DTE'.padStart(12),
+  'ratio'.padStart(8),
+  'sign',
+);
+let ok = 0;
+let flip = 0;
+for (const s of Object.keys(hs0)
+  .map(Number)
+  .sort((a, b) => b - a)) {
+  const cell = m.cells.find((c) => c.strike === s && c.expiry === '2026-08-07');
+  const our = cell?.gex ?? null;
+  const hs = hs0[s];
+  let ratio = '—';
+  let sign = '—';
+  if (our != null && Math.abs(hs.v) > 0) {
+    ratio = (our / hs.v).toFixed(2);
+    if ((our < 0) === (hs.v < 0)) ok += 1;
+    else flip += 1;
+    sign = (our < 0) === (hs.v < 0) ? 'OK' : 'FLIP';
+  }
+  console.log(
+    String(s).padStart(6),
+    hs.label.padStart(12),
+    (our == null ? 'miss' : fmt(our)).padStart(12),
+    ratio.padStart(8),
+    sign,
+  );
+}
+console.log(`\nsign match ${ok} / flip ${flip}`);
+console.log('king', m.levels?.king);
+console.log('flip/walls', { flip: m.levels?.flip, callWall: m.levels?.callWall, putWall: m.levels?.putWall });
+console.log('quality', m.quality);
+const c35 = m.cells.find((c) => c.strike === 35 && c.expiry === '2026-08-21');
+const c34 = m.cells.find((c) => c.strike === 34 && c.expiry === '2026-08-21');
+console.log('Aug21 35', c35 ? fmt(c35.gex) : 'miss', 'HS~+3.3M');
+console.log('Aug21 34', c34 ? fmt(c34.gex) : 'miss', 'HS~+2.9M');
+console.log('\nDone.');
+process.exit(0);