feat: dealer flow, mirror portfolio (M21), options convexity, FINRA short interest, alert producers, vendor gate

Snapshot of in-progress module work across multiple slices: - Dealer Flow: dealerExposureEngine, dealerMapService, dealerMapExplain, dealerMapIntegrity, dealerMapReplay, dealerStudyEngine, hanStyleLevels - Mirror Portfolio (M21): fundRepository, captureIngest, mirrorAlertProducers, fund holdings strip, live book, position capture ingest - Options: BSM, NormalizedOptionSurface types, OptionsChainRouter, ConvexityGate, option legs panel - Alert producers: vixLevel, rotation, thesis, unlock, portfolioRisk, mirror (fund_capture, fund_13f, mirror_diff) - FINRA short interest adapter + queue integration - SEC company tickers adapter + ingest (symbol search index seed) - Vendor gate (rate-limit-first data plane, ADR-0009) - CUSIP registry, reverse 13F refresh, stock float service - LRU cache, portfolio backtest engine - Frontend: dealer-flow, funds, journal, lab, monitor, plan, portfolio, reports, screener, strategies, theses, guided-start, exits, more pages - Volume profile, workspace profile, visibility-aware poll - ADRs 0010 (mirror math not advice), 0011 (symbol search index) - VENDOR_INTEGRATIONS.md, END_USER_TEST.md - .gitignore: exclude DBs, .DS_Store, local config, agent scratch
2026-08-10 13:36:26 -04:00
parent 04fc11b2fd
commit ac94acf9e3
229 changed files with 32617 additions and 3934 deletions
@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+/**
+ * Harvest X posts for Dealer Flow knowledge (offline).
+ *
+ * Writes ONLY to:
+ *   - app/server/data/dealer-flow-raw/   (gitignored via data/)
+ *   - optional Obsidian vault raw/by-handle/  (personal notebook; app never reads)
+ *
+ * Usage (from app/server):
+ *   node --experimental-strip-types scripts/dealer-flow-harvest.ts
+ *   node --experimental-strip-types scripts/dealer-flow-harvest.ts --handle someuser --count 40
+ *
+ * Auth: uses bird with cookies from x_credentials in investor-flow.db when available,
+ * or bird's own browser cookie extraction.
+ */
+
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { DatabaseSync } from 'node:sqlite';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SERVER_ROOT = join(__dirname, '..');
+const DATA = join(SERVER_ROOT, 'data');
+const SOURCES_PATH = join(SERVER_ROOT, 'config', 'dealer-flow-sources.json');
+const RAW_ROOT = join(DATA, 'dealer-flow-raw');
+const DB_PATH = join(DATA, 'investor-flow.db');
+
+interface SourcesFile {
+  obsidianVault?: string;
+  handles: string[];
+  threads?: string[];
+  notes?: string;
+}
+
+function loadSources(): SourcesFile {
+  if (!existsSync(SOURCES_PATH)) {
+    return { handles: [], threads: [] };
+  }
+  return JSON.parse(readFileSync(SOURCES_PATH, 'utf8')) as SourcesFile;
+}
+
+async function decryptCreds(): Promise<{ authToken: string; ct0: string } | null> {
+  if (!existsSync(DB_PATH)) return null;
+  try {
+    const { default: cryptoMod } = await import('../src/lib/crypto.ts');
+    const db = new DatabaseSync(DB_PATH, { readOnly: true });
+    const row = db.prepare(
+      "SELECT ct0_enc, auth_token_enc FROM x_credentials WHERE id='singleton'",
+    ).get() as { ct0_enc?: string; auth_token_enc?: string } | undefined;
+    db.close();
+    if (!row?.ct0_enc || !row?.auth_token_enc) return null;
+    return {
+      ct0: cryptoMod.decrypt(row.ct0_enc),
+      authToken: cryptoMod.decrypt(row.auth_token_enc),
+    };
+  } catch (e) {
+    console.warn('[harvest] could not load x_credentials:', e instanceof Error ? e.message : e);
+    return null;
+  }
+}
+
+function birdJson(
+  args: string[],
+  creds: { authToken: string; ct0: string } | null,
+): unknown {
+  const full = [...args, '--json'];
+  if (creds) {
+    full.push('--auth-token', creds.authToken, '--ct0', creds.ct0);
+  }
+  const r = spawnSync('bird', full, {
+    encoding: 'utf8',
+    maxBuffer: 20 * 1024 * 1024,
+    timeout: 120_000,
+  });
+  if (r.status !== 0) {
+    throw new Error(r.stderr || r.stdout || `bird exit ${r.status}`);
+  }
+  const out = (r.stdout || '').trim();
+  if (!out) return null;
+  return JSON.parse(out);
+}
+
+function writePost(
+  handle: string,
+  post: { id?: string; text?: string; createdAt?: string; url?: string },
+  vault?: string,
+): void {
+  const id = post.id ?? String(Date.now());
+  const date = (post.createdAt ?? new Date().toISOString()).slice(0, 10);
+  const body = [
+    '---',
+    `handle: "${handle}"`,
+    `post_id: "${id}"`,
+    `date: ${date}`,
+    `url: "${post.url ?? ''}"`,
+    'tags: []',
+    '---',
+    '',
+    post.text ?? '',
+    '',
+  ].join('\n');
+
+  const localDir = join(RAW_ROOT, 'by-handle', handle);
+  mkdirSync(localDir, { recursive: true });
+  const fname = `${date}_${id}.md`;
+  writeFileSync(join(localDir, fname), body, 'utf8');
+
+  if (vault) {
+    const vDir = join(vault, 'raw', 'by-handle', handle);
+    mkdirSync(vDir, { recursive: true });
+    writeFileSync(join(vDir, fname), body, 'utf8');
+  }
+}
+
+function parseArgs(argv: string[]) {
+  const out: { handle?: string; count: number; thread?: string; maxPages: number } = {
+    count: 50,
+    maxPages: 5,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--handle') out.handle = argv[++i]?.replace(/^@/, '');
+    else if (argv[i] === '--count') out.count = Number(argv[++i] ?? 50);
+    else if (argv[i] === '--thread') out.thread = argv[++i];
+    else if (argv[i] === '--max-pages') out.maxPages = Number(argv[++i] ?? 5);
+  }
+  return out;
+}
+
+function normalizeTweets(json: unknown): Array<{ id?: string; text?: string; createdAt?: string; url?: string }> {
+  if (!json) return [];
+  if (Array.isArray(json)) {
+    return json.map((t) => {
+      const o = t as Record<string, unknown>;
+      return {
+        id: String(o.id ?? o.id_str ?? o.rest_id ?? ''),
+        text: String(o.text ?? o.full_text ?? o.body ?? ''),
+        createdAt: String(o.createdAt ?? o.created_at ?? ''),
+        url: o.url ? String(o.url) : undefined,
+      };
+    });
+  }
+  if (typeof json === 'object') {
+    const o = json as Record<string, unknown>;
+    if (Array.isArray(o.tweets)) return normalizeTweets(o.tweets);
+    if (Array.isArray(o.results)) return normalizeTweets(o.results);
+    if (Array.isArray(o.data)) return normalizeTweets(o.data);
+  }
+  return [];
+}
+
+async function main() {
+  const cli = parseArgs(process.argv.slice(2));
+  const sources = loadSources();
+  const handles = cli.handle
+    ? [cli.handle]
+    : (sources.handles ?? []).map((h) => h.replace(/^@/, '').trim()).filter(Boolean);
+  const threads = cli.thread ? [cli.thread] : (sources.threads ?? []);
+  const vault = sources.obsidianVault?.trim() || undefined;
+
+  if (handles.length === 0 && threads.length === 0) {
+    console.error(
+      'No handles or threads. Edit data/dealer-flow-sources.json or pass --handle / --thread.',
+    );
+    process.exit(1);
+  }
+
+  mkdirSync(RAW_ROOT, { recursive: true });
+  const creds = await decryptCreds();
+  if (creds) console.log('[harvest] using x_credentials from DB');
+  else console.log('[harvest] no DB cookies - bird will use its own cookie source if configured');
+
+  let written = 0;
+
+  for (const handle of handles) {
+    console.log(`[harvest] user-tweets @${handle} n=${cli.count} max-pages=${cli.maxPages}`);
+    try {
+      const json = birdJson(
+        [
+          'user-tweets',
+          handle,
+          '-n',
+          String(cli.count),
+          '--max-pages',
+          String(cli.maxPages),
+          '--delay',
+          '1500',
+        ],
+        creds,
+      );
+      const tweets = normalizeTweets(json);
+      for (const t of tweets) {
+        if (!t.text) continue;
+        writePost(handle, t, vault);
+        written++;
+      }
+      console.log(`[harvest] @${handle}: ${tweets.length} posts`);
+    } catch (e) {
+      console.error(`[harvest] @${handle} failed:`, e instanceof Error ? e.message : e);
+    }
+  }
+
+  for (const thread of threads) {
+    console.log(`[harvest] thread ${thread}`);
+    try {
+      const json = birdJson(['thread', thread, '--all'], creds);
+      const tweets = normalizeTweets(json);
+      // unknown author - put under _threads
+      for (const t of tweets) {
+        if (!t.text) continue;
+        writePost('_threads', t, vault);
+        written++;
+      }
+      console.log(`[harvest] thread: ${tweets.length} posts`);
+    } catch (e) {
+      console.error(`[harvest] thread failed:`, e instanceof Error ? e.message : e);
+    }
+  }
+
+  // Touch sources.md in vault if present
+  if (vault && existsSync(vault)) {
+    const sourcesMd = join(vault, 'sources.md');
+    // leave existing; just ensure directory
+    mkdirSync(join(vault, 'raw', 'by-handle'), { recursive: true });
+    if (!existsSync(sourcesMd)) {
+      writeFileSync(
+        sourcesMd,
+        '# Sources\n\nUpdated by dealer-flow-harvest. Edit handles in app `data/dealer-flow-sources.json`.\n',
+        'utf8',
+      );
+    }
+  }
+
+  console.log(`[harvest] done. wrote ${written} post files under ${RAW_ROOT}`);
+  if (vault) console.log(`[harvest] also mirrored under ${vault}/raw/by-handle/ (personal; app does not read)`);
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});