294 lines
12 KiB
JavaScript
294 lines
12 KiB
JavaScript
|
#!/usr/bin/env node
|
||
|
|
/**
|
||
|
|
* Distill harvested raw posts into:
|
||
|
|
* 1) In-app notes: src/analysis/dealerFlowExplainNotes.ts (app reads this)
|
||
|
|
* 2) Obsidian pages under your DealerFlow vault (you read this; app does not)
|
||
|
|
*
|
||
|
|
* Does not call the LLM by default - extracts plain-English candidate lines and
|
||
|
|
* merges into the existing notes structure. Re-run after harvest.
|
||
|
|
*
|
||
|
|
* Usage (from app/server):
|
||
|
|
* node --experimental-strip-types scripts/dealer-flow-distill.ts
|
||
|
|
*/
|
||
|
|
|
||
|
|
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||
|
|
import { dirname, join } from 'node:path';
|
||
|
|
import { fileURLToPath } from 'node:url';
|
||
|
|
|
||
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||
|
|
const SERVER_ROOT = join(__dirname, '..');
|
||
|
|
const DATA = join(SERVER_ROOT, 'data');
|
||
|
|
const SOURCES_PATH = join(SERVER_ROOT, 'config', 'dealer-flow-sources.json');
|
||
|
|
const RAW_ROOT = join(DATA, 'dealer-flow-raw');
|
||
|
|
const NOTES_OUT = join(SERVER_ROOT, 'src/analysis/dealerFlowExplainNotes.ts');
|
||
|
|
|
||
|
|
interface SourcesFile {
|
||
|
|
obsidianVault?: string;
|
||
|
|
}
|
||
|
|
|
||
|
|
function loadSources(): SourcesFile {
|
||
|
|
if (!existsSync(SOURCES_PATH)) return {};
|
||
|
|
return JSON.parse(readFileSync(SOURCES_PATH, 'utf8')) as SourcesFile;
|
||
|
|
}
|
||
|
|
|
||
|
|
function walkMd(dir: string, acc: string[] = []): string[] {
|
||
|
|
if (!existsSync(dir)) return acc;
|
||
|
|
for (const name of readdirSync(dir, { withFileTypes: true })) {
|
||
|
|
const p = join(dir, name.name);
|
||
|
|
if (name.isDirectory()) walkMd(p, acc);
|
||
|
|
else if (name.name.endsWith('.md')) acc.push(p);
|
||
|
|
}
|
||
|
|
return acc;
|
||
|
|
}
|
||
|
|
|
||
|
|
function stripFrontmatter(md: string): string {
|
||
|
|
if (!md.startsWith('---')) return md;
|
||
|
|
const end = md.indexOf('\n---', 3);
|
||
|
|
if (end < 0) return md;
|
||
|
|
return md.slice(end + 4);
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Lines that sound like trade advice - never enter product notes (Primary Rule). */
|
||
|
|
const DIRECTIVE =
|
||
|
|
/\b(buy|sell|long|short|puts?|calls?|grab|grabs|entry|entries|entering|enter|aim for|i wait|target|targets|stop|stops|size|sizing|subs?|subscribe|trade idea|trading rules|your long|your short|go long|go short|load up|trim|add here|looking good|good returns|bounce till|try a bounce|sure bet|shorted|try for|predict anything|tough to say|poor asymmetry|offers poor)\b/i;
|
||
|
|
|
||
|
|
/** Strip third-party product names from product-facing snippets. */
|
||
|
|
function sanitizeSnippet(s: string): string {
|
||
|
|
return s
|
||
|
|
.replace(/\bHeatseeker\b/gi, 'DealerFlow')
|
||
|
|
.replace(/\bPika\b/gi, '')
|
||
|
|
.replace(/\s{2,}/g, ' ')
|
||
|
|
.trim();
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Prefer mechanics / definition language (education threads). */
|
||
|
|
const MECHANICS =
|
||
|
|
/\b(node|nodes|dealer|hedg|dampen|amplif|mean reversion|momentum|yellow|purple|calm zone|no control|attractor|accelerant|vanna|positive gamma|negative gamma|long gamma|short gamma|concentration|stabiliz|pinning|air pocket|desert|flip|king is|what .* means|how .* work)\b/i;
|
||
|
|
|
||
|
|
/** Pull short plain-looking lines that might help explain maps (heuristic, not ML). */
|
||
|
|
function extractSnippets(text: string): string[] {
|
||
|
|
const body = stripFrontmatter(text).replace(/\s+/g, ' ').trim();
|
||
|
|
if (body.length < 40) return [];
|
||
|
|
const sentences = body.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter(Boolean);
|
||
|
|
const out: string[] = [];
|
||
|
|
const keywords =
|
||
|
|
/\b(gamma|gex|vex|vanna|pin|pinning|wall|strike|expiry|opex|dealer|hedge|hedging|open interest|flip|king|magnet|cluster|air pocket|desert|stabilize|amplif|mean reversion|momentum|volatility|positive gamma|negative gamma|long gamma|short gamma|node|nodes)\b/i;
|
||
|
|
for (const s of sentences) {
|
||
|
|
if (s.length < 45 || s.length > 260) continue;
|
||
|
|
if (!keywords.test(s)) continue;
|
||
|
|
if (DIRECTIVE.test(s)) continue;
|
||
|
|
// Skip ticker-heavy or price-level callouts (app uses live map numbers)
|
||
|
|
if ((s.match(/\$[A-Z]{1,5}\b/g) ?? []).length >= 1) continue;
|
||
|
|
if (/\$\d/.test(s) || /\b\d{3,5}\b/.test(s)) continue;
|
||
|
|
// Prefer education mechanics; deprioritize bare "VEX …" market chatter
|
||
|
|
if (/^VEX\b/i.test(s) && !MECHANICS.test(s)) continue;
|
||
|
|
if (!MECHANICS.test(s) && !/\b(means|work as|acting as|promotes|dampen|amplif|gravitational|exposure)\b/i.test(s)) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
// Skip live session chatter / unhelpful fragments
|
||
|
|
if (/\bMINS IN\b/i.test(s) || /^Update on nodes:/i.test(s)) continue;
|
||
|
|
out.push(sanitizeSnippet(s.replace(/["']/g, "'")));
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
function uniqueSnippets(all: string[], max = 20): string[] {
|
||
|
|
const seen = new Set<string>();
|
||
|
|
const out: string[] = [];
|
||
|
|
for (const s of all) {
|
||
|
|
const key = s.toLowerCase().slice(0, 80);
|
||
|
|
if (seen.has(key)) continue;
|
||
|
|
seen.add(key);
|
||
|
|
out.push(s);
|
||
|
|
if (out.length >= max) break;
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
function buildNotesTs(snippets: string[], version: string, updatedAt: string): string {
|
||
|
|
const snippetsLit = JSON.stringify(snippets, null, 2);
|
||
|
|
return `/**
|
||
|
|
* In-app plain-language notes for Dealer Flow explains.
|
||
|
|
* Runtime reads THIS file only - never Obsidian.
|
||
|
|
*
|
||
|
|
* Regenerated by: \`node --experimental-strip-types scripts/dealer-flow-distill.ts\`
|
||
|
|
* (distill also writes a personal copy into your Obsidian vault for offline reading).
|
||
|
|
*/
|
||
|
|
|
||
|
|
export interface DealerFlowExplainNotes {
|
||
|
|
/** Bump when regenerating so L1 cache invalidates. */
|
||
|
|
version: string;
|
||
|
|
updatedAt: string;
|
||
|
|
/** One sentence: what the picture is. */
|
||
|
|
whatMapIs: string;
|
||
|
|
plainMetaphors: {
|
||
|
|
cushion: string;
|
||
|
|
spring: string;
|
||
|
|
mixed: string;
|
||
|
|
magnet: string;
|
||
|
|
floor: string;
|
||
|
|
ceiling: string;
|
||
|
|
thinZone: string;
|
||
|
|
};
|
||
|
|
levels: {
|
||
|
|
king: string;
|
||
|
|
flip: string;
|
||
|
|
callWall: string;
|
||
|
|
putWall: string;
|
||
|
|
airPocket: string;
|
||
|
|
};
|
||
|
|
scenarios: Record<string, string>;
|
||
|
|
patterns: Record<string, string>;
|
||
|
|
caveats: string[];
|
||
|
|
forbidden: string[];
|
||
|
|
/** Optional snippets distilled from harvested sources (plain English only). */
|
||
|
|
sourceSnippets: string[];
|
||
|
|
}
|
||
|
|
|
||
|
|
export const DEALER_FLOW_EXPLAIN_NOTES: DealerFlowExplainNotes = {
|
||
|
|
version: ${JSON.stringify(version)},
|
||
|
|
updatedAt: ${JSON.stringify(updatedAt)},
|
||
|
|
whatMapIs:
|
||
|
|
'This picture shows modeled hedging pressure from options open interest across prices and dates - not a forecast of where the stock must go, and not a live list of any firm\\'s inventory.',
|
||
|
|
plainMetaphors: {
|
||
|
|
cushion:
|
||
|
|
'Cushion: near this area, modeled hedges tend to lean against moves, so big swings may get pushed back toward the middle.',
|
||
|
|
spring:
|
||
|
|
'Spring: near this area, modeled hedges tend to lean with moves, so once price starts moving it can keep going.',
|
||
|
|
mixed: 'Mixed or thin: neither a clear cushion nor a clear spring near today\\'s price.',
|
||
|
|
magnet: 'Magnet: a loud strike the map cares about more than quieter levels nearby.',
|
||
|
|
floor: 'Floor people watch: heavy modeled pressure below the market - not a guarantee price bounces.',
|
||
|
|
ceiling: 'Ceiling people watch: heavy modeled pressure above the market - not a guarantee price stops.',
|
||
|
|
thinZone: 'Thin zone: little modeled pressure - price may travel through with less mechanical friction in the model.',
|
||
|
|
},
|
||
|
|
levels: {
|
||
|
|
king: 'King is the loudest strike on this map (largest absolute modeled pressure).',
|
||
|
|
flip: 'Flip is where the map\\'s personality can switch between cushion and spring territory.',
|
||
|
|
callWall: 'Call wall is a heavy modeled pile above spot - a ceiling people watch.',
|
||
|
|
putWall: 'Put wall is a heavy modeled pile below spot - a floor people watch.',
|
||
|
|
airPocket: 'Air pocket is a thin band of pressure between stronger levels.',
|
||
|
|
},
|
||
|
|
scenarios: {
|
||
|
|
pos_gex_pos_vex:
|
||
|
|
'Reads relatively calm: both price-linked and vol-linked pieces tend to absorb shocks in the model.',
|
||
|
|
neg_gex_neg_vex:
|
||
|
|
'Reads jumpy: price and vol can reinforce each other in the model - swings may feel larger.',
|
||
|
|
gex_dominates: 'Price-level magnets matter more than volatility for this snapshot - start with king and walls.',
|
||
|
|
vex_dominates: 'Volatility shifts may matter more than pin levels for this snapshot.',
|
||
|
|
mixed: 'Signals disagree - treat the map as context, not one clean story.',
|
||
|
|
insufficient_data: 'Not enough open-interest data yet to label a clear scenario.',
|
||
|
|
},
|
||
|
|
patterns: {
|
||
|
|
whipsaw:
|
||
|
|
'Whipsaw shape: two strong magnets with a thinner middle - price can bounce between the poles.',
|
||
|
|
rainbow_road:
|
||
|
|
'Rainbow-road shape: pressure spread across a wide range without one dominant magnet - more wandering.',
|
||
|
|
rug_setup:
|
||
|
|
'Rug-like stack: support-looking structure that can fail and speed moves lower in the model if the soft floor breaks - still not a trade call.',
|
||
|
|
},
|
||
|
|
caveats: [
|
||
|
|
'Quotes may be delayed; open interest is often from the prior session.',
|
||
|
|
'Numbers are modeled dollar hedging pressure, not reported dealer inventory.',
|
||
|
|
'This explains a snapshot, not a price prediction.',
|
||
|
|
],
|
||
|
|
forbidden: [
|
||
|
|
'Do not say buy, sell, short, or long as instructions.',
|
||
|
|
'Do not give position size, entries, stops, or targets as recommendations.',
|
||
|
|
'Do not claim the map knows where price must go.',
|
||
|
|
],
|
||
|
|
sourceSnippets: ${snippetsLit},
|
||
|
|
};
|
||
|
|
|
||
|
|
/** Compact block for LLM system prompts. */
|
||
|
|
export function explainNotesForPrompt(notes: DealerFlowExplainNotes = DEALER_FLOW_EXPLAIN_NOTES): string {
|
||
|
|
const lines = [
|
||
|
|
\`version: \${notes.version}\`,
|
||
|
|
\`whatMapIs: \${notes.whatMapIs}\`,
|
||
|
|
\`metaphors: \${Object.values(notes.plainMetaphors).join(' | ')}\`,
|
||
|
|
\`levels: \${Object.values(notes.levels).join(' | ')}\`,
|
||
|
|
\`scenarios: \${Object.entries(notes.scenarios)
|
||
|
|
.map(([k, v]) => \`\${k}=\${v}\`)
|
||
|
|
.join(' | ')}\`,
|
||
|
|
\`patterns: \${Object.entries(notes.patterns)
|
||
|
|
.map(([k, v]) => \`\${k}=\${v}\`)
|
||
|
|
.join(' | ')}\`,
|
||
|
|
\`caveats: \${notes.caveats.join(' | ')}\`,
|
||
|
|
\`forbidden: \${notes.forbidden.join(' | ')}\`,
|
||
|
|
];
|
||
|
|
if (notes.sourceSnippets.length > 0) {
|
||
|
|
lines.push(
|
||
|
|
'extra plain-English snippets from distilled sources:',
|
||
|
|
...notes.sourceSnippets.slice(0, 12).map((s) => \`- \${s}\`),
|
||
|
|
);
|
||
|
|
}
|
||
|
|
return lines.join('\\n');
|
||
|
|
}
|
||
|
|
`;
|
||
|
|
}
|
||
|
|
|
||
|
|
function writeObsidianPages(vault: string, snippets: string[], version: string): void {
|
||
|
|
const pages = join(vault, 'pages');
|
||
|
|
mkdirSync(pages, { recursive: true });
|
||
|
|
|
||
|
|
const glossary = `# DealerFlow glossary (auto-distilled shell)
|
||
|
|
|
||
|
|
Last distill: ${version}
|
||
|
|
|
||
|
|
## What the map is
|
||
|
|
|
||
|
|
This picture shows modeled hedging pressure from options open interest - not a forecast and not anyone's live inventory.
|
||
|
|
|
||
|
|
## Everyday words
|
||
|
|
|
||
|
|
- **Cushion** - hedges lean against moves
|
||
|
|
- **Spring** - hedges lean with moves
|
||
|
|
- **Magnet** - loud strike the map cares about
|
||
|
|
- **Floor / ceiling** - heavy piles people watch below / above
|
||
|
|
- **Thin zone** - little pressure between stronger levels
|
||
|
|
|
||
|
|
## From harvested sources (review & edit)
|
||
|
|
|
||
|
|
${snippets.length ? snippets.map((s) => `- ${s}`).join('\n') : '_No keyword snippets yet. Run harvest with handles first._'}
|
||
|
|
|
||
|
|
---
|
||
|
|
Edit freely. The **app does not read this file**. After you improve wording, re-run distill or paste good lines into in-app notes.
|
||
|
|
`;
|
||
|
|
|
||
|
|
writeFileSync(join(pages, 'glossary.md'), glossary, 'utf8');
|
||
|
|
writeFileSync(
|
||
|
|
join(pages, 'source-snippets.md'),
|
||
|
|
`# Source snippets (personal)\n\nVersion ${version}\n\n${snippets.map((s) => `- ${s}`).join('\n') || '_empty_'}\n`,
|
||
|
|
'utf8',
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
function main() {
|
||
|
|
const sources = loadSources();
|
||
|
|
const files = walkMd(RAW_ROOT);
|
||
|
|
const all: string[] = [];
|
||
|
|
for (const f of files) {
|
||
|
|
try {
|
||
|
|
all.push(...extractSnippets(readFileSync(f, 'utf8')));
|
||
|
|
} catch {
|
||
|
|
/* skip */
|
||
|
|
}
|
||
|
|
}
|
||
|
|
const snippets = uniqueSnippets(all, 20);
|
||
|
|
const updatedAt = new Date().toISOString();
|
||
|
|
const version = updatedAt.slice(0, 10) + '.' + String(snippets.length);
|
||
|
|
|
||
|
|
writeFileSync(NOTES_OUT, buildNotesTs(snippets, version, updatedAt), 'utf8');
|
||
|
|
console.log(`[distill] wrote in-app notes → ${NOTES_OUT} (${snippets.length} source snippets)`);
|
||
|
|
|
||
|
|
const vault = sources.obsidianVault?.trim();
|
||
|
|
if (vault) {
|
||
|
|
mkdirSync(vault, { recursive: true });
|
||
|
|
writeObsidianPages(vault, snippets, version);
|
||
|
|
console.log(`[distill] wrote personal Obsidian pages under ${vault}/pages/ (app does not read)`);
|
||
|
|
} else {
|
||
|
|
console.log('[distill] no obsidianVault in dealer-flow-sources.json - skipped vault write');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
main();
|