investor-flow/app/server/scripts/dealer-flow-distill.ts

#!/usr/bin/env node
/**
 * Distill harvested raw posts into:
 *   1) In-app notes: src/analysis/dealerFlowExplainNotes.ts  (app reads this)
 *   2) Obsidian pages under your DealerFlow vault           (you read this; app does not)
 *
 * Does not call the LLM by default - extracts plain-English candidate lines and
 * merges into the existing notes structure. Re-run after harvest.
 *
 * Usage (from app/server):
 *   node --experimental-strip-types scripts/dealer-flow-distill.ts
 */

import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const SERVER_ROOT = join(__dirname, '..');
const DATA = join(SERVER_ROOT, 'data');
const SOURCES_PATH = join(SERVER_ROOT, 'config', 'dealer-flow-sources.json');
const RAW_ROOT = join(DATA, 'dealer-flow-raw');
const NOTES_OUT = join(SERVER_ROOT, 'src/analysis/dealerFlowExplainNotes.ts');

interface SourcesFile {
  obsidianVault?: string;
}

function loadSources(): SourcesFile {
  if (!existsSync(SOURCES_PATH)) return {};
  return JSON.parse(readFileSync(SOURCES_PATH, 'utf8')) as SourcesFile;
}

function walkMd(dir: string, acc: string[] = []): string[] {
  if (!existsSync(dir)) return acc;
  for (const name of readdirSync(dir, { withFileTypes: true })) {
    const p = join(dir, name.name);
    if (name.isDirectory()) walkMd(p, acc);
    else if (name.name.endsWith('.md')) acc.push(p);
  }
  return acc;
}

function stripFrontmatter(md: string): string {
  if (!md.startsWith('---')) return md;
  const end = md.indexOf('\n---', 3);
  if (end < 0) return md;
  return md.slice(end + 4);
}

/** Lines that sound like trade advice - never enter product notes (Primary Rule). */
const DIRECTIVE =
  /\b(buy|sell|long|short|puts?|calls?|grab|grabs|entry|entries|entering|enter|aim for|i wait|target|targets|stop|stops|size|sizing|subs?|subscribe|trade idea|trading rules|your long|your short|go long|go short|load up|trim|add here|looking good|good returns|bounce till|try a bounce|sure bet|shorted|try for|predict anything|tough to say|poor asymmetry|offers poor)\b/i;

/** Strip third-party product names from product-facing snippets. */
function sanitizeSnippet(s: string): string {
  return s
    .replace(/\bHeatseeker\b/gi, 'DealerFlow')
    .replace(/\bPika\b/gi, '')
    .replace(/\s{2,}/g, ' ')
    .trim();
}

/** Prefer mechanics / definition language (education threads). */
const MECHANICS =
  /\b(node|nodes|dealer|hedg|dampen|amplif|mean reversion|momentum|yellow|purple|calm zone|no control|attractor|accelerant|vanna|positive gamma|negative gamma|long gamma|short gamma|concentration|stabiliz|pinning|air pocket|desert|flip|king is|what .* means|how .* work)\b/i;

/** Pull short plain-looking lines that might help explain maps (heuristic, not ML). */
function extractSnippets(text: string): string[] {
  const body = stripFrontmatter(text).replace(/\s+/g, ' ').trim();
  if (body.length < 40) return [];
  const sentences = body.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter(Boolean);
  const out: string[] = [];
  const keywords =
    /\b(gamma|gex|vex|vanna|pin|pinning|wall|strike|expiry|opex|dealer|hedge|hedging|open interest|flip|king|magnet|cluster|air pocket|desert|stabilize|amplif|mean reversion|momentum|volatility|positive gamma|negative gamma|long gamma|short gamma|node|nodes)\b/i;
  for (const s of sentences) {
    if (s.length < 45 || s.length > 260) continue;
    if (!keywords.test(s)) continue;
    if (DIRECTIVE.test(s)) continue;
    // Skip ticker-heavy or price-level callouts (app uses live map numbers)
    if ((s.match(/\$[A-Z]{1,5}\b/g) ?? []).length >= 1) continue;
    if (/\$\d/.test(s) || /\b\d{3,5}\b/.test(s)) continue;
    // Prefer education mechanics; deprioritize bare "VEX …" market chatter
    if (/^VEX\b/i.test(s) && !MECHANICS.test(s)) continue;
    if (!MECHANICS.test(s) && !/\b(means|work as|acting as|promotes|dampen|amplif|gravitational|exposure)\b/i.test(s)) {
      continue;
    }
    // Skip live session chatter / unhelpful fragments
    if (/\bMINS IN\b/i.test(s) || /^Update on nodes:/i.test(s)) continue;
    out.push(sanitizeSnippet(s.replace(/["']/g, "'")));
  }
  return out;
}

function uniqueSnippets(all: string[], max = 20): string[] {
  const seen = new Set<string>();
  const out: string[] = [];
  for (const s of all) {
    const key = s.toLowerCase().slice(0, 80);
    if (seen.has(key)) continue;
    seen.add(key);
    out.push(s);
    if (out.length >= max) break;
  }
  return out;
}

function buildNotesTs(snippets: string[], version: string, updatedAt: string): string {
  const snippetsLit = JSON.stringify(snippets, null, 2);
  return `/**
 * In-app plain-language notes for Dealer Flow explains.
 * Runtime reads THIS file only - never Obsidian.
 *
 * Regenerated by: \`node --experimental-strip-types scripts/dealer-flow-distill.ts\`
 * (distill also writes a personal copy into your Obsidian vault for offline reading).
 */

export interface DealerFlowExplainNotes {
  /** Bump when regenerating so L1 cache invalidates. */
  version: string;
  updatedAt: string;
  /** One sentence: what the picture is. */
  whatMapIs: string;
  plainMetaphors: {
    cushion: string;
    spring: string;
    mixed: string;
    magnet: string;
    floor: string;
    ceiling: string;
    thinZone: string;
  };
  levels: {
    king: string;
    flip: string;
    callWall: string;
    putWall: string;
    airPocket: string;
  };
  scenarios: Record<string, string>;
  patterns: Record<string, string>;
  caveats: string[];
  forbidden: string[];
  /** Optional snippets distilled from harvested sources (plain English only). */
  sourceSnippets: string[];
}

export const DEALER_FLOW_EXPLAIN_NOTES: DealerFlowExplainNotes = {
  version: ${JSON.stringify(version)},
  updatedAt: ${JSON.stringify(updatedAt)},
  whatMapIs:
    'This picture shows modeled hedging pressure from options open interest across prices and dates - not a forecast of where the stock must go, and not a live list of any firm\\'s inventory.',
  plainMetaphors: {
    cushion:
      'Cushion: near this area, modeled hedges tend to lean against moves, so big swings may get pushed back toward the middle.',
    spring:
      'Spring: near this area, modeled hedges tend to lean with moves, so once price starts moving it can keep going.',
    mixed: 'Mixed or thin: neither a clear cushion nor a clear spring near today\\'s price.',
    magnet: 'Magnet: a loud strike the map cares about more than quieter levels nearby.',
    floor: 'Floor people watch: heavy modeled pressure below the market - not a guarantee price bounces.',
    ceiling: 'Ceiling people watch: heavy modeled pressure above the market - not a guarantee price stops.',
    thinZone: 'Thin zone: little modeled pressure - price may travel through with less mechanical friction in the model.',
  },
  levels: {
    king: 'King is the loudest strike on this map (largest absolute modeled pressure).',
    flip: 'Flip is where the map\\'s personality can switch between cushion and spring territory.',
    callWall: 'Call wall is a heavy modeled pile above spot - a ceiling people watch.',
    putWall: 'Put wall is a heavy modeled pile below spot - a floor people watch.',
    airPocket: 'Air pocket is a thin band of pressure between stronger levels.',
  },
  scenarios: {
    pos_gex_pos_vex:
      'Reads relatively calm: both price-linked and vol-linked pieces tend to absorb shocks in the model.',
    neg_gex_neg_vex:
      'Reads jumpy: price and vol can reinforce each other in the model - swings may feel larger.',
    gex_dominates: 'Price-level magnets matter more than volatility for this snapshot - start with king and walls.',
    vex_dominates: 'Volatility shifts may matter more than pin levels for this snapshot.',
    mixed: 'Signals disagree - treat the map as context, not one clean story.',
    insufficient_data: 'Not enough open-interest data yet to label a clear scenario.',
  },
  patterns: {
    whipsaw:
      'Whipsaw shape: two strong magnets with a thinner middle - price can bounce between the poles.',
    rainbow_road:
      'Rainbow-road shape: pressure spread across a wide range without one dominant magnet - more wandering.',
    rug_setup:
      'Rug-like stack: support-looking structure that can fail and speed moves lower in the model if the soft floor breaks - still not a trade call.',
  },
  caveats: [
    'Quotes may be delayed; open interest is often from the prior session.',
    'Numbers are modeled dollar hedging pressure, not reported dealer inventory.',
    'This explains a snapshot, not a price prediction.',
  ],
  forbidden: [
    'Do not say buy, sell, short, or long as instructions.',
    'Do not give position size, entries, stops, or targets as recommendations.',
    'Do not claim the map knows where price must go.',
  ],
  sourceSnippets: ${snippetsLit},
};

/** Compact block for LLM system prompts. */
export function explainNotesForPrompt(notes: DealerFlowExplainNotes = DEALER_FLOW_EXPLAIN_NOTES): string {
  const lines = [
    \`version: \${notes.version}\`,
    \`whatMapIs: \${notes.whatMapIs}\`,
    \`metaphors: \${Object.values(notes.plainMetaphors).join(' | ')}\`,
    \`levels: \${Object.values(notes.levels).join(' | ')}\`,
    \`scenarios: \${Object.entries(notes.scenarios)
      .map(([k, v]) => \`\${k}=\${v}\`)
      .join(' | ')}\`,
    \`patterns: \${Object.entries(notes.patterns)
      .map(([k, v]) => \`\${k}=\${v}\`)
      .join(' | ')}\`,
    \`caveats: \${notes.caveats.join(' | ')}\`,
    \`forbidden: \${notes.forbidden.join(' | ')}\`,
  ];
  if (notes.sourceSnippets.length > 0) {
    lines.push(
      'extra plain-English snippets from distilled sources:',
      ...notes.sourceSnippets.slice(0, 12).map((s) => \`- \${s}\`),
    );
  }
  return lines.join('\\n');
}
`;
}

function writeObsidianPages(vault: string, snippets: string[], version: string): void {
  const pages = join(vault, 'pages');
  mkdirSync(pages, { recursive: true });

  const glossary = `# DealerFlow glossary (auto-distilled shell)

Last distill: ${version}

## What the map is

This picture shows modeled hedging pressure from options open interest - not a forecast and not anyone's live inventory.

## Everyday words

- **Cushion** - hedges lean against moves
- **Spring** - hedges lean with moves
- **Magnet** - loud strike the map cares about
- **Floor / ceiling** - heavy piles people watch below / above
- **Thin zone** - little pressure between stronger levels

## From harvested sources (review & edit)

${snippets.length ? snippets.map((s) => `- ${s}`).join('\n') : '_No keyword snippets yet. Run harvest with handles first._'}

---
Edit freely. The **app does not read this file**. After you improve wording, re-run distill or paste good lines into in-app notes.
`;

  writeFileSync(join(pages, 'glossary.md'), glossary, 'utf8');
  writeFileSync(
    join(pages, 'source-snippets.md'),
    `# Source snippets (personal)\n\nVersion ${version}\n\n${snippets.map((s) => `- ${s}`).join('\n') || '_empty_'}\n`,
    'utf8',
  );
}

function main() {
  const sources = loadSources();
  const files = walkMd(RAW_ROOT);
  const all: string[] = [];
  for (const f of files) {
    try {
      all.push(...extractSnippets(readFileSync(f, 'utf8')));
    } catch {
      /* skip */
    }
  }
  const snippets = uniqueSnippets(all, 20);
  const updatedAt = new Date().toISOString();
  const version = updatedAt.slice(0, 10) + '.' + String(snippets.length);

  writeFileSync(NOTES_OUT, buildNotesTs(snippets, version, updatedAt), 'utf8');
  console.log(`[distill] wrote in-app notes → ${NOTES_OUT} (${snippets.length} source snippets)`);

  const vault = sources.obsidianVault?.trim();
  if (vault) {
    mkdirSync(vault, { recursive: true });
    writeObsidianPages(vault, snippets, version);
    console.log(`[distill] wrote personal Obsidian pages under ${vault}/pages/ (app does not read)`);
  } else {
    console.log('[distill] no obsidianVault in dealer-flow-sources.json - skipped vault write');
  }
}

main();