feat: dealer flow, mirror portfolio (M21), options convexity, FINRA short interest, alert producers, vendor gate

Snapshot of in-progress module work across multiple slices: - Dealer Flow: dealerExposureEngine, dealerMapService, dealerMapExplain, dealerMapIntegrity, dealerMapReplay, dealerStudyEngine, hanStyleLevels - Mirror Portfolio (M21): fundRepository, captureIngest, mirrorAlertProducers, fund holdings strip, live book, position capture ingest - Options: BSM, NormalizedOptionSurface types, OptionsChainRouter, ConvexityGate, option legs panel - Alert producers: vixLevel, rotation, thesis, unlock, portfolioRisk, mirror (fund_capture, fund_13f, mirror_diff) - FINRA short interest adapter + queue integration - SEC company tickers adapter + ingest (symbol search index seed) - Vendor gate (rate-limit-first data plane, ADR-0009) - CUSIP registry, reverse 13F refresh, stock float service - LRU cache, portfolio backtest engine - Frontend: dealer-flow, funds, journal, lab, monitor, plan, portfolio, reports, screener, strategies, theses, guided-start, exits, more pages - Volume profile, workspace profile, visibility-aware poll - ADRs 0010 (mirror math not advice), 0011 (symbol search index) - VENDOR_INTEGRATIONS.md, END_USER_TEST.md - .gitignore: exclude DBs, .DS_Store, local config, agent scratch
2026-08-10 13:36:26 -04:00
parent 04fc11b2fd
commit ac94acf9e3
229 changed files with 32617 additions and 3934 deletions
@@ -0,0 +1,293 @@
+#!/usr/bin/env node
+/**
+ * Distill harvested raw posts into:
+ *   1) In-app notes: src/analysis/dealerFlowExplainNotes.ts  (app reads this)
+ *   2) Obsidian pages under your DealerFlow vault           (you read this; app does not)
+ *
+ * Does not call the LLM by default - extracts plain-English candidate lines and
+ * merges into the existing notes structure. Re-run after harvest.
+ *
+ * Usage (from app/server):
+ *   node --experimental-strip-types scripts/dealer-flow-distill.ts
+ */
+
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SERVER_ROOT = join(__dirname, '..');
+const DATA = join(SERVER_ROOT, 'data');
+const SOURCES_PATH = join(SERVER_ROOT, 'config', 'dealer-flow-sources.json');
+const RAW_ROOT = join(DATA, 'dealer-flow-raw');
+const NOTES_OUT = join(SERVER_ROOT, 'src/analysis/dealerFlowExplainNotes.ts');
+
+interface SourcesFile {
+  obsidianVault?: string;
+}
+
+function loadSources(): SourcesFile {
+  if (!existsSync(SOURCES_PATH)) return {};
+  return JSON.parse(readFileSync(SOURCES_PATH, 'utf8')) as SourcesFile;
+}
+
+function walkMd(dir: string, acc: string[] = []): string[] {
+  if (!existsSync(dir)) return acc;
+  for (const name of readdirSync(dir, { withFileTypes: true })) {
+    const p = join(dir, name.name);
+    if (name.isDirectory()) walkMd(p, acc);
+    else if (name.name.endsWith('.md')) acc.push(p);
+  }
+  return acc;
+}
+
+function stripFrontmatter(md: string): string {
+  if (!md.startsWith('---')) return md;
+  const end = md.indexOf('\n---', 3);
+  if (end < 0) return md;
+  return md.slice(end + 4);
+}
+
+/** Lines that sound like trade advice - never enter product notes (Primary Rule). */
+const DIRECTIVE =
+  /\b(buy|sell|long|short|puts?|calls?|grab|grabs|entry|entries|entering|enter|aim for|i wait|target|targets|stop|stops|size|sizing|subs?|subscribe|trade idea|trading rules|your long|your short|go long|go short|load up|trim|add here|looking good|good returns|bounce till|try a bounce|sure bet|shorted|try for|predict anything|tough to say|poor asymmetry|offers poor)\b/i;
+
+/** Strip third-party product names from product-facing snippets. */
+function sanitizeSnippet(s: string): string {
+  return s
+    .replace(/\bHeatseeker\b/gi, 'DealerFlow')
+    .replace(/\bPika\b/gi, '')
+    .replace(/\s{2,}/g, ' ')
+    .trim();
+}
+
+/** Prefer mechanics / definition language (education threads). */
+const MECHANICS =
+  /\b(node|nodes|dealer|hedg|dampen|amplif|mean reversion|momentum|yellow|purple|calm zone|no control|attractor|accelerant|vanna|positive gamma|negative gamma|long gamma|short gamma|concentration|stabiliz|pinning|air pocket|desert|flip|king is|what .* means|how .* work)\b/i;
+
+/** Pull short plain-looking lines that might help explain maps (heuristic, not ML). */
+function extractSnippets(text: string): string[] {
+  const body = stripFrontmatter(text).replace(/\s+/g, ' ').trim();
+  if (body.length < 40) return [];
+  const sentences = body.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter(Boolean);
+  const out: string[] = [];
+  const keywords =
+    /\b(gamma|gex|vex|vanna|pin|pinning|wall|strike|expiry|opex|dealer|hedge|hedging|open interest|flip|king|magnet|cluster|air pocket|desert|stabilize|amplif|mean reversion|momentum|volatility|positive gamma|negative gamma|long gamma|short gamma|node|nodes)\b/i;
+  for (const s of sentences) {
+    if (s.length < 45 || s.length > 260) continue;
+    if (!keywords.test(s)) continue;
+    if (DIRECTIVE.test(s)) continue;
+    // Skip ticker-heavy or price-level callouts (app uses live map numbers)
+    if ((s.match(/\$[A-Z]{1,5}\b/g) ?? []).length >= 1) continue;
+    if (/\$\d/.test(s) || /\b\d{3,5}\b/.test(s)) continue;
+    // Prefer education mechanics; deprioritize bare "VEX …" market chatter
+    if (/^VEX\b/i.test(s) && !MECHANICS.test(s)) continue;
+    if (!MECHANICS.test(s) && !/\b(means|work as|acting as|promotes|dampen|amplif|gravitational|exposure)\b/i.test(s)) {
+      continue;
+    }
+    // Skip live session chatter / unhelpful fragments
+    if (/\bMINS IN\b/i.test(s) || /^Update on nodes:/i.test(s)) continue;
+    out.push(sanitizeSnippet(s.replace(/["']/g, "'")));
+  }
+  return out;
+}
+
+function uniqueSnippets(all: string[], max = 20): string[] {
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const s of all) {
+    const key = s.toLowerCase().slice(0, 80);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(s);
+    if (out.length >= max) break;
+  }
+  return out;
+}
+
+function buildNotesTs(snippets: string[], version: string, updatedAt: string): string {
+  const snippetsLit = JSON.stringify(snippets, null, 2);
+  return `/**
+ * In-app plain-language notes for Dealer Flow explains.
+ * Runtime reads THIS file only - never Obsidian.
+ *
+ * Regenerated by: \`node --experimental-strip-types scripts/dealer-flow-distill.ts\`
+ * (distill also writes a personal copy into your Obsidian vault for offline reading).
+ */
+
+export interface DealerFlowExplainNotes {
+  /** Bump when regenerating so L1 cache invalidates. */
+  version: string;
+  updatedAt: string;
+  /** One sentence: what the picture is. */
+  whatMapIs: string;
+  plainMetaphors: {
+    cushion: string;
+    spring: string;
+    mixed: string;
+    magnet: string;
+    floor: string;
+    ceiling: string;
+    thinZone: string;
+  };
+  levels: {
+    king: string;
+    flip: string;
+    callWall: string;
+    putWall: string;
+    airPocket: string;
+  };
+  scenarios: Record<string, string>;
+  patterns: Record<string, string>;
+  caveats: string[];
+  forbidden: string[];
+  /** Optional snippets distilled from harvested sources (plain English only). */
+  sourceSnippets: string[];
+}
+
+export const DEALER_FLOW_EXPLAIN_NOTES: DealerFlowExplainNotes = {
+  version: ${JSON.stringify(version)},
+  updatedAt: ${JSON.stringify(updatedAt)},
+  whatMapIs:
+    'This picture shows modeled hedging pressure from options open interest across prices and dates - not a forecast of where the stock must go, and not a live list of any firm\\'s inventory.',
+  plainMetaphors: {
+    cushion:
+      'Cushion: near this area, modeled hedges tend to lean against moves, so big swings may get pushed back toward the middle.',
+    spring:
+      'Spring: near this area, modeled hedges tend to lean with moves, so once price starts moving it can keep going.',
+    mixed: 'Mixed or thin: neither a clear cushion nor a clear spring near today\\'s price.',
+    magnet: 'Magnet: a loud strike the map cares about more than quieter levels nearby.',
+    floor: 'Floor people watch: heavy modeled pressure below the market - not a guarantee price bounces.',
+    ceiling: 'Ceiling people watch: heavy modeled pressure above the market - not a guarantee price stops.',
+    thinZone: 'Thin zone: little modeled pressure - price may travel through with less mechanical friction in the model.',
+  },
+  levels: {
+    king: 'King is the loudest strike on this map (largest absolute modeled pressure).',
+    flip: 'Flip is where the map\\'s personality can switch between cushion and spring territory.',
+    callWall: 'Call wall is a heavy modeled pile above spot - a ceiling people watch.',
+    putWall: 'Put wall is a heavy modeled pile below spot - a floor people watch.',
+    airPocket: 'Air pocket is a thin band of pressure between stronger levels.',
+  },
+  scenarios: {
+    pos_gex_pos_vex:
+      'Reads relatively calm: both price-linked and vol-linked pieces tend to absorb shocks in the model.',
+    neg_gex_neg_vex:
+      'Reads jumpy: price and vol can reinforce each other in the model - swings may feel larger.',
+    gex_dominates: 'Price-level magnets matter more than volatility for this snapshot - start with king and walls.',
+    vex_dominates: 'Volatility shifts may matter more than pin levels for this snapshot.',
+    mixed: 'Signals disagree - treat the map as context, not one clean story.',
+    insufficient_data: 'Not enough open-interest data yet to label a clear scenario.',
+  },
+  patterns: {
+    whipsaw:
+      'Whipsaw shape: two strong magnets with a thinner middle - price can bounce between the poles.',
+    rainbow_road:
+      'Rainbow-road shape: pressure spread across a wide range without one dominant magnet - more wandering.',
+    rug_setup:
+      'Rug-like stack: support-looking structure that can fail and speed moves lower in the model if the soft floor breaks - still not a trade call.',
+  },
+  caveats: [
+    'Quotes may be delayed; open interest is often from the prior session.',
+    'Numbers are modeled dollar hedging pressure, not reported dealer inventory.',
+    'This explains a snapshot, not a price prediction.',
+  ],
+  forbidden: [
+    'Do not say buy, sell, short, or long as instructions.',
+    'Do not give position size, entries, stops, or targets as recommendations.',
+    'Do not claim the map knows where price must go.',
+  ],
+  sourceSnippets: ${snippetsLit},
+};
+
+/** Compact block for LLM system prompts. */
+export function explainNotesForPrompt(notes: DealerFlowExplainNotes = DEALER_FLOW_EXPLAIN_NOTES): string {
+  const lines = [
+    \`version: \${notes.version}\`,
+    \`whatMapIs: \${notes.whatMapIs}\`,
+    \`metaphors: \${Object.values(notes.plainMetaphors).join(' | ')}\`,
+    \`levels: \${Object.values(notes.levels).join(' | ')}\`,
+    \`scenarios: \${Object.entries(notes.scenarios)
+      .map(([k, v]) => \`\${k}=\${v}\`)
+      .join(' | ')}\`,
+    \`patterns: \${Object.entries(notes.patterns)
+      .map(([k, v]) => \`\${k}=\${v}\`)
+      .join(' | ')}\`,
+    \`caveats: \${notes.caveats.join(' | ')}\`,
+    \`forbidden: \${notes.forbidden.join(' | ')}\`,
+  ];
+  if (notes.sourceSnippets.length > 0) {
+    lines.push(
+      'extra plain-English snippets from distilled sources:',
+      ...notes.sourceSnippets.slice(0, 12).map((s) => \`- \${s}\`),
+    );
+  }
+  return lines.join('\\n');
+}
+`;
+}
+
+function writeObsidianPages(vault: string, snippets: string[], version: string): void {
+  const pages = join(vault, 'pages');
+  mkdirSync(pages, { recursive: true });
+
+  const glossary = `# DealerFlow glossary (auto-distilled shell)
+
+Last distill: ${version}
+
+## What the map is
+
+This picture shows modeled hedging pressure from options open interest - not a forecast and not anyone's live inventory.
+
+## Everyday words
+
+- **Cushion** - hedges lean against moves
+- **Spring** - hedges lean with moves
+- **Magnet** - loud strike the map cares about
+- **Floor / ceiling** - heavy piles people watch below / above
+- **Thin zone** - little pressure between stronger levels
+
+## From harvested sources (review & edit)
+
+${snippets.length ? snippets.map((s) => `- ${s}`).join('\n') : '_No keyword snippets yet. Run harvest with handles first._'}
+
+---
+Edit freely. The **app does not read this file**. After you improve wording, re-run distill or paste good lines into in-app notes.
+`;
+
+  writeFileSync(join(pages, 'glossary.md'), glossary, 'utf8');
+  writeFileSync(
+    join(pages, 'source-snippets.md'),
+    `# Source snippets (personal)\n\nVersion ${version}\n\n${snippets.map((s) => `- ${s}`).join('\n') || '_empty_'}\n`,
+    'utf8',
+  );
+}
+
+function main() {
+  const sources = loadSources();
+  const files = walkMd(RAW_ROOT);
+  const all: string[] = [];
+  for (const f of files) {
+    try {
+      all.push(...extractSnippets(readFileSync(f, 'utf8')));
+    } catch {
+      /* skip */
+    }
+  }
+  const snippets = uniqueSnippets(all, 20);
+  const updatedAt = new Date().toISOString();
+  const version = updatedAt.slice(0, 10) + '.' + String(snippets.length);
+
+  writeFileSync(NOTES_OUT, buildNotesTs(snippets, version, updatedAt), 'utf8');
+  console.log(`[distill] wrote in-app notes → ${NOTES_OUT} (${snippets.length} source snippets)`);
+
+  const vault = sources.obsidianVault?.trim();
+  if (vault) {
+    mkdirSync(vault, { recursive: true });
+    writeObsidianPages(vault, snippets, version);
+    console.log(`[distill] wrote personal Obsidian pages under ${vault}/pages/ (app does not read)`);
+  } else {
+    console.log('[distill] no obsidianVault in dealer-flow-sources.json - skipped vault write');
+  }
+}
+
+main();