investor-flow/app/server/scripts/dealer-flow-harvest.ts

#!/usr/bin/env node
/**
 * Harvest X posts for Dealer Flow knowledge (offline).
 *
 * Writes ONLY to:
 *   - app/server/data/dealer-flow-raw/   (gitignored via data/)
 *   - optional Obsidian vault raw/by-handle/  (personal notebook; app never reads)
 *
 * Usage (from app/server):
 *   node --experimental-strip-types scripts/dealer-flow-harvest.ts
 *   node --experimental-strip-types scripts/dealer-flow-harvest.ts --handle someuser --count 40
 *
 * Auth: uses bird with cookies from x_credentials in investor-flow.db when available,
 * or bird's own browser cookie extraction.
 */

import { spawnSync } from 'node:child_process';
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { DatabaseSync } from 'node:sqlite';

const __dirname = dirname(fileURLToPath(import.meta.url));
const SERVER_ROOT = join(__dirname, '..');
const DATA = join(SERVER_ROOT, 'data');
const SOURCES_PATH = join(SERVER_ROOT, 'config', 'dealer-flow-sources.json');
const RAW_ROOT = join(DATA, 'dealer-flow-raw');
const DB_PATH = join(DATA, 'investor-flow.db');

interface SourcesFile {
  obsidianVault?: string;
  handles: string[];
  threads?: string[];
  notes?: string;
}

function loadSources(): SourcesFile {
  if (!existsSync(SOURCES_PATH)) {
    return { handles: [], threads: [] };
  }
  return JSON.parse(readFileSync(SOURCES_PATH, 'utf8')) as SourcesFile;
}

async function decryptCreds(): Promise<{ authToken: string; ct0: string } | null> {
  if (!existsSync(DB_PATH)) return null;
  try {
    const { default: cryptoMod } = await import('../src/lib/crypto.ts');
    const db = new DatabaseSync(DB_PATH, { readOnly: true });
    const row = db.prepare(
      "SELECT ct0_enc, auth_token_enc FROM x_credentials WHERE id='singleton'",
    ).get() as { ct0_enc?: string; auth_token_enc?: string } | undefined;
    db.close();
    if (!row?.ct0_enc || !row?.auth_token_enc) return null;
    return {
      ct0: cryptoMod.decrypt(row.ct0_enc),
      authToken: cryptoMod.decrypt(row.auth_token_enc),
    };
  } catch (e) {
    console.warn('[harvest] could not load x_credentials:', e instanceof Error ? e.message : e);
    return null;
  }
}

function birdJson(
  args: string[],
  creds: { authToken: string; ct0: string } | null,
): unknown {
  const full = [...args, '--json'];
  if (creds) {
    full.push('--auth-token', creds.authToken, '--ct0', creds.ct0);
  }
  const r = spawnSync('bird', full, {
    encoding: 'utf8',
    maxBuffer: 20 * 1024 * 1024,
    timeout: 120_000,
  });
  if (r.status !== 0) {
    throw new Error(r.stderr || r.stdout || `bird exit ${r.status}`);
  }
  const out = (r.stdout || '').trim();
  if (!out) return null;
  return JSON.parse(out);
}

function writePost(
  handle: string,
  post: { id?: string; text?: string; createdAt?: string; url?: string },
  vault?: string,
): void {
  const id = post.id ?? String(Date.now());
  const date = (post.createdAt ?? new Date().toISOString()).slice(0, 10);
  const body = [
    '---',
    `handle: "${handle}"`,
    `post_id: "${id}"`,
    `date: ${date}`,
    `url: "${post.url ?? ''}"`,
    'tags: []',
    '---',
    '',
    post.text ?? '',
    '',
  ].join('\n');

  const localDir = join(RAW_ROOT, 'by-handle', handle);
  mkdirSync(localDir, { recursive: true });
  const fname = `${date}_${id}.md`;
  writeFileSync(join(localDir, fname), body, 'utf8');

  if (vault) {
    const vDir = join(vault, 'raw', 'by-handle', handle);
    mkdirSync(vDir, { recursive: true });
    writeFileSync(join(vDir, fname), body, 'utf8');
  }
}

function parseArgs(argv: string[]) {
  const out: { handle?: string; count: number; thread?: string; maxPages: number } = {
    count: 50,
    maxPages: 5,
  };
  for (let i = 0; i < argv.length; i++) {
    if (argv[i] === '--handle') out.handle = argv[++i]?.replace(/^@/, '');
    else if (argv[i] === '--count') out.count = Number(argv[++i] ?? 50);
    else if (argv[i] === '--thread') out.thread = argv[++i];
    else if (argv[i] === '--max-pages') out.maxPages = Number(argv[++i] ?? 5);
  }
  return out;
}

function normalizeTweets(json: unknown): Array<{ id?: string; text?: string; createdAt?: string; url?: string }> {
  if (!json) return [];
  if (Array.isArray(json)) {
    return json.map((t) => {
      const o = t as Record<string, unknown>;
      return {
        id: String(o.id ?? o.id_str ?? o.rest_id ?? ''),
        text: String(o.text ?? o.full_text ?? o.body ?? ''),
        createdAt: String(o.createdAt ?? o.created_at ?? ''),
        url: o.url ? String(o.url) : undefined,
      };
    });
  }
  if (typeof json === 'object') {
    const o = json as Record<string, unknown>;
    if (Array.isArray(o.tweets)) return normalizeTweets(o.tweets);
    if (Array.isArray(o.results)) return normalizeTweets(o.results);
    if (Array.isArray(o.data)) return normalizeTweets(o.data);
  }
  return [];
}

async function main() {
  const cli = parseArgs(process.argv.slice(2));
  const sources = loadSources();
  const handles = cli.handle
    ? [cli.handle]
    : (sources.handles ?? []).map((h) => h.replace(/^@/, '').trim()).filter(Boolean);
  const threads = cli.thread ? [cli.thread] : (sources.threads ?? []);
  const vault = sources.obsidianVault?.trim() || undefined;

  if (handles.length === 0 && threads.length === 0) {
    console.error(
      'No handles or threads. Edit data/dealer-flow-sources.json or pass --handle / --thread.',
    );
    process.exit(1);
  }

  mkdirSync(RAW_ROOT, { recursive: true });
  const creds = await decryptCreds();
  if (creds) console.log('[harvest] using x_credentials from DB');
  else console.log('[harvest] no DB cookies - bird will use its own cookie source if configured');

  let written = 0;

  for (const handle of handles) {
    console.log(`[harvest] user-tweets @${handle} n=${cli.count} max-pages=${cli.maxPages}`);
    try {
      const json = birdJson(
        [
          'user-tweets',
          handle,
          '-n',
          String(cli.count),
          '--max-pages',
          String(cli.maxPages),
          '--delay',
          '1500',
        ],
        creds,
      );
      const tweets = normalizeTweets(json);
      for (const t of tweets) {
        if (!t.text) continue;
        writePost(handle, t, vault);
        written++;
      }
      console.log(`[harvest] @${handle}: ${tweets.length} posts`);
    } catch (e) {
      console.error(`[harvest] @${handle} failed:`, e instanceof Error ? e.message : e);
    }
  }

  for (const thread of threads) {
    console.log(`[harvest] thread ${thread}`);
    try {
      const json = birdJson(['thread', thread, '--all'], creds);
      const tweets = normalizeTweets(json);
      // unknown author - put under _threads
      for (const t of tweets) {
        if (!t.text) continue;
        writePost('_threads', t, vault);
        written++;
      }
      console.log(`[harvest] thread: ${tweets.length} posts`);
    } catch (e) {
      console.error(`[harvest] thread failed:`, e instanceof Error ? e.message : e);
    }
  }

  // Touch sources.md in vault if present
  if (vault && existsSync(vault)) {
    const sourcesMd = join(vault, 'sources.md');
    // leave existing; just ensure directory
    mkdirSync(join(vault, 'raw', 'by-handle'), { recursive: true });
    if (!existsSync(sourcesMd)) {
      writeFileSync(
        sourcesMd,
        '# Sources\n\nUpdated by dealer-flow-harvest. Edit handles in app `data/dealer-flow-sources.json`.\n',
        'utf8',
      );
    }
  }

  console.log(`[harvest] done. wrote ${written} post files under ${RAW_ROOT}`);
  if (vault) console.log(`[harvest] also mirrored under ${vault}/raw/by-handle/ (personal; app does not read)`);
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});