fix: wire data adapters, fix OptionsAdapter v3, build Research pages

- Fix OptionsAdapter for yahoo-finance2 v3 API ({ date: expiry } instead of positional arg) - Fix institutional.flow and insiderStream endpoints to query correct DB tables - Fix anonymous user INSERT (pw_hash column, not password_hash) - Create secDataFetcher module for 13F/Form 4 data population - Add institutional and marketOutlook endpoints to frontend trpc client - Build out Chart Lab, Institutional, Market Outlook pages with real components - Add focus-visible rings, aria-labels, hex tokens, shape icons across all panels - Integrate EmotionLogger, MobileTabNav, useKeyboardShortcuts - Add localStorage persistence for emotion logs - Normalize spacing, type scale, hover effects across all components
2026-07-05 08:50:12 -04:00
parent 1e818c0881
commit 55f07e6b42
27 changed files with 2331 additions and 555 deletions
@@ -100,26 +100,61 @@ export class OptionsAdapter implements SourceFetch {
    const yf = await this.yf();

    if (kind === 'expiry_dates') {
-      const rawDates = await yf.optionsExpiryDates(id);
-      // yahoo-finance2 returns string[] (ISO dates). Sort ascending.
-      const dates = [...rawDates].sort() as OptionExpiryDate[];
+      // yahoo-finance2 v3: options() returns expirationDates as Date objects
+      const result = await yf.options(id);
+      const dates = (result.expirationDates ?? [])
+        .map((d: Date) => d.toISOString().slice(0, 10))
+        .sort() as OptionExpiryDate[];
      return {
        value: dates,
-        ttlClass: 'intraday', // 5 min TTL — short-lived, shifts around events
+        ttlClass: 'intraday',
        provenance: { fetchedAt, sourceKind: 'yfinance', rawSourceId: `options:expiry:${id}` },
      };
    }

    if (kind === 'chain') {
      const [symbol, expiry] = id.split(':');
-      const rawChain = await yf.options(symbol, expiry);
+      const rawChain = await yf.options(symbol, { date: expiry });
+      // v3: options is indexed by number, extract calls/puts from first entry
+      const optionsObj = rawChain.options;
+      const firstKey = Object.keys(optionsObj)[0];
+      const expiryData = firstKey ? optionsObj[firstKey] : null;
+      const allRows = [
+        ...(expiryData?.calls ?? []).map((c: Record<string, unknown>) => ({ ...c, right: 'call' as const })),
+        ...(expiryData?.puts ?? []).map((p: Record<string, unknown>) => ({ ...p, right: 'put' as const })),
+      ];
+      const rows = parseOptionChainRows(symbol, allRows);
      return {
-        value: parseOptionChain(symbol, rawChain),
-        ttlClass: 'options_snapshot', // 15 min (matches CacheRepository TTL_MS)
+        value: rows,
+        ttlClass: 'options_snapshot',
        provenance: { fetchedAt, sourceKind: 'yfinance', rawSourceId: `options:chain:${symbol}:${expiry}` },
      };
    }

+    if (kind === 'greeks') {
+      const parts = id.split(':');
+      const symbol = parts[0];
+      const expiry = parts[1];
+      const strike = parseFloat(parts[2] ?? '0');
+      const rawChain = await yf.options(symbol, { date: expiry });
+      const optionsObj = rawChain.options;
+      const firstKey = Object.keys(optionsObj)[0];
+      const expiryData = firstKey ? optionsObj[firstKey] : null;
+      const allRows = [
+        ...(expiryData?.calls ?? []).map((c: Record<string, unknown>) => ({ ...c, right: 'call' as const })),
+        ...(expiryData?.puts ?? []).map((p: Record<string, unknown>) => ({ ...p, right: 'put' as const })),
+      ];
+      const rows = parseOptionChainRows(symbol, allRows);
+      const target = strike > 0
+        ? rows.find((r) => r.strike === strike)
+        : rows[0];
+      return {
+        value: target ?? null,
+        ttlClass: 'options_snapshot',
+        provenance: { fetchedAt, sourceKind: 'yfinance', rawSourceId: `options:greeks:${symbol}:${expiry}:${strike}` },
+      };
+    }
+
    throw new Error(`OptionsAdapter: unknown kind '${kind}'`);
  }

@@ -133,7 +168,9 @@ export class OptionsAdapter implements SourceFetch {
  async chain(symbol: string, expiry: OptionExpiryDate): Promise<OptionChain> {
    const key = `yfinance:chain:${symbol}:${expiry}`;
    const result = await this.fetchOne(key);
-    return result.value as OptionChain;
+    // fetchOne returns OptionChainRow[] (the cached shape). Re-wrap into OptionChain.
+    const rows = result.value as OptionChainRow[];
+    return { symbol, expiration: expiry, rows };
  }
 }

@@ -141,15 +178,25 @@ export class OptionsAdapter implements SourceFetch {

 /**
 * Parse the raw option chain from yahoo-finance2 into typed OptionChainRow[].
- * yfinance2 returns `{ calls: [...], puts: [...] }` — we flatten and tag each row.
+ * Handles both formats:
+ *   - v2: `{ calls: [...], puts: [...] }` object
+ *   - v3: array of row objects with `right` field already set
 */
-export function parseOptionChain(symbol: string, raw: Record<string, unknown>): OptionChain {
-  const calls = (raw.calls ?? []) as Array<Record<string, unknown>>;
-  const puts = (raw.puts ?? []) as Array<Record<string, unknown>>;
+export function parseOptionChainRows(symbol: string, raw: Record<string, unknown> | Array<Record<string, unknown>>): OptionChainRow[] {
+  let rows: OptionChainRow[] = [];

-  const rows: OptionChainRow[] = [];
-  for (const r of [...calls, ...puts]) {
-    rows.push(parseOneRow(r));
+  if (Array.isArray(raw)) {
+    // Already flattened array with right field
+    for (const r of raw) {
+      rows.push(parseOneRow(r));
+    }
+  } else {
+    // Legacy format: { calls: [...], puts: [...] }
+    const calls = (raw.calls ?? []) as Array<Record<string, unknown>>;
+    const puts = (raw.puts ?? []) as Array<Record<string, unknown>>;
+    for (const r of [...calls, ...puts]) {
+      rows.push(parseOneRow(r));
+    }
  }

  // Sort by strike ascending, calls first then puts at same strike (standard convention).
@@ -162,9 +209,16 @@ export function parseOptionChain(symbol: string, raw: Record<string, unknown>):
    return 0;
  });

-  // Derive expiration from the first row (all rows in a chain share it).
-  const expiration = rows[0]?.expiration ?? '';
+  return rows;
+}

+/**
+ * Parse the raw option chain and return a full OptionChain wrapper.
+ * Used when callers need the OptionChain shape (e.g. direct API responses).
+ */
+export function parseOptionChain(symbol: string, raw: Record<string, unknown>): OptionChain {
+  const rows = parseOptionChainRows(symbol, raw);
+  const expiration = rows[0]?.expiration ?? '';
  return { symbol, expiration, rows };
 }

@@ -50,6 +50,11 @@ export function db(): DatabaseSync {
  if (!_db) {
    _db = createDb();
    initSchema(_db);
+    // Ensure the anonymous user exists for unauthenticated watchlist/portfolio usage.
+    try {
+      _db.prepare("INSERT OR IGNORE INTO users (id, email, pw_hash, created_at) VALUES (?, ?, ?, ?)")
+        .run('anonymous', 'anonymous@investor-flow.local', '', new Date().toISOString());
+    } catch { /* table may not exist yet or already exists */ }
  }
  return _db;
 }
@@ -0,0 +1,269 @@
+// Investor Flow — SEC Data Fetcher
+//
+// Fetches institutional (13F) and insider (Form 4) data from EDGAR
+// and writes to the institution_filings and insider_transactions tables.
+// Called when symbols are added to the watchlist.
+
+import type { DatabaseSync } from 'node:sqlite';
+import { EdgarAdapter, padCik } from '../adapters/EdgarAdapter.ts';
+
+const edgar = new EdgarAdapter();
+
+// ---------------------------------------------------------------------------
+// Symbol → CIK resolution
+// ---------------------------------------------------------------------------
+
+const OPERATOR_EMAIL = process.env.SEC_OPERATOR_EMAIL ?? 'research@example.com';
+const UA = `Investor Flow (${OPERATOR_EMAIL})`;
+
+// In-memory cache for company tickers (refreshed daily)
+let tickersCache: Record<string, { cik_str: number; ticker: string; title: string }> | null = null;
+let tickersCacheDate: string | null = null;
+
+/**
+ * Load the SEC company tickers mapping. Caches in memory for the day.
+ */
+async function loadTickers(): Promise<Record<string, { cik_str: number; ticker: string; title: string }>> {
+  const today = new Date().toISOString().slice(0, 10);
+  if (tickersCache && tickersCacheDate === today) {
+    return tickersCache;
+  }
+
+  const resp = await fetch('https://www.sec.gov/files/company_tickers.json', {
+    headers: { 'User-Agent': UA, Accept: 'application/json' },
+  });
+
+  if (!resp.ok) throw new Error(`EDGAR tickers: ${resp.status}`);
+
+  tickersCache = await resp.json();
+  tickersCacheDate = today;
+  return tickersCache;
+}
+
+/**
+ * Resolve a stock symbol to its CIK number using SEC's company tickers endpoint.
+ * Returns the CIK string (padded to 10 digits) or null if not found.
+ */
+async function resolveCik(symbol: string): Promise<string | null> {
+  try {
+    const tickers = await loadTickers();
+    const upper = symbol.toUpperCase();
+
+    // Find by ticker
+    for (const v of Object.values(tickers)) {
+      if (v.ticker === upper) {
+        return String(v.cik_str).padStart(10, '0');
+      }
+    }
+
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// 13F data fetch + store
+// ---------------------------------------------------------------------------
+
+interface Filing13F {
+  cusip: string;
+  issuerName: string;
+  value: number;
+  sshPrnamt: number;
+}
+
+/**
+ * Fetch 13F holdings for a CIK and write to institution_filings table.
+ * Fetches the most recent 2 quarters of data.
+ */
+async function fetchAndStore13F(
+  db: DatabaseSync,
+  symbol: string,
+  cik: string,
+): Promise<number> {
+  const now = new Date().toISOString();
+
+  try {
+    // Get filings index for 13F-HR forms
+    const filingsResult = await edgar.filings_index(cik, { formTypes: ['13F-HR'] });
+    const filings = (filingsResult.value as Array<Record<string, unknown>>).slice(0, 4); // Most recent 4
+
+    let storedCount = 0;
+
+    for (const filing of filings) {
+      const accession = filing.accessionNumber as string;
+      if (!accession) continue;
+
+      try {
+        const holdingsResult = await edgar.form13f_holdings(cik, accession);
+        const holdings = (holdingsResult.value as { holdings: Filing13F[] }).holdings;
+
+        // Determine the reported quarter from the filing date
+        const filedAt = (filing.filedAt as string) ?? now;
+        const reportDate = (filing.reportDate as string) ?? filedAt;
+        const quarter = `${reportDate.slice(0, 4)}-Q${Math.ceil((parseInt(reportDate.slice(5, 7)) / 3))}`;
+
+        // Get filer metadata for SIC code
+        let filerName: string | null = null;
+        let filerSic: string | null = null;
+        try {
+          const meta = await edgar.filer_cik_meta(cik);
+          const metaValue = meta.value as { name?: string; sic?: string };
+          filerName = metaValue.name ?? null;
+          filerSic = metaValue.sic ?? null;
+        } catch { /* ignore meta fetch errors */ }
+
+        // Insert each holding into institution_filings
+        const insert = db.prepare(`
+          INSERT OR REPLACE INTO institution_filings
+            (filer_cik, filer_name, filer_sic, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at)
+          VALUES (?, ?, ?, ?, '13F-HR', ?, ?, ?, ?, ?)
+        `);
+
+        for (const h of holdings) {
+          // Only store holdings that match or relate to our symbol
+          // (CUSIP matching would be ideal but we store by symbol for simplicity)
+          insert.run(cik, filerName, filerSic, symbol, h.sshPrnamt, h.value, quarter, filedAt, now);
+          storedCount++;
+        }
+      } catch {
+        // Skip individual filing errors (rate limits, parse errors)
+        continue;
+      }
+    }
+
+    return storedCount;
+  } catch {
+    return 0;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Form 4 data fetch + store
+// ---------------------------------------------------------------------------
+
+interface Form4Tx {
+  reporter: string;
+  relationship: string;
+  securityTitle: string;
+  transactionDate: string;
+  transactionCode: string;
+  shares: number;
+  price: number;
+}
+
+/**
+ * Fetch Form 4 insider transactions for a CIK and write to insider_transactions table.
+ */
+async function fetchAndStoreForm4(
+  db: DatabaseSync,
+  symbol: string,
+  cik: string,
+): Promise<number> {
+  const now = new Date().toISOString();
+
+  try {
+    // Get filings index for Form 4
+    const filingsResult = await edgar.filings_index(cik, { formTypes: ['4'] });
+    const filings = (filingsResult.value as Array<Record<string, unknown>>).slice(0, 10); // Most recent 10
+
+    let storedCount = 0;
+
+    for (const filing of filings) {
+      const accession = filing.accessionNumber as string;
+      if (!accession) continue;
+
+      try {
+        const txResult = await edgar.form4_tx(cik, accession);
+        const transactions = (txResult.value as { transactions: Form4Tx[] }).transactions;
+
+        const filedAt = (filing.filedAt as string) ?? now;
+
+        const insert = db.prepare(`
+          INSERT OR REPLACE INTO insider_transactions
+            (form4_id, symbol, insider_name, insider_role, tx_date, tx_code, tx_type,
+             shares, price, is_10b5_1, classification, filed_at, fetched_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `);
+
+        for (const tx of transactions) {
+          // Classify transaction
+          const code = tx.transactionCode.toUpperCase();
+          const isBuy = code === 'P' || code === 'A' || code === 'M';
+          const isSell = code === 'S' || code === 'D' || code === 'G';
+          const txType = isBuy ? 'buy' : isSell ? 'sell' : 'other';
+
+          // Simple informed vs routine classification
+          // Informed = open-market buy (P) not under 10b5-1
+          const is10b5 = tx.relationship.toLowerCase().includes('10b5');
+          let classification: string;
+          if (code === 'P' && !is10b5) {
+            classification = 'informed_buy';
+          } else if (code === 'S' && !is10b5) {
+            classification = 'informed_sell';
+          } else {
+            classification = 'routine';
+          }
+
+          // Determine insider role from relationship
+          let insiderRole: string | null = null;
+          const rel = tx.relationship.toLowerCase();
+          if (rel.includes('officer') || rel.includes('ceo') || rel.includes('cfo')) {
+            insiderRole = 'officer';
+          } else if (rel.includes('director')) {
+            insiderRole = 'director';
+          } else if (rel.includes('10%') || rel.includes('ten percent')) {
+            insiderRole = 'ten_pct_holder';
+          }
+
+          const form4Id = `${cik}_${accession}_${tx.transactionDate}_${tx.reporter}`;
+          insert.run(
+            form4Id, symbol, tx.reporter, insiderRole,
+            tx.transactionDate, code, txType,
+            tx.shares, tx.price, is10b5 ? 1 : 0,
+            classification, filedAt, now
+          );
+          storedCount++;
+        }
+      } catch {
+        continue;
+      }
+    }
+
+    return storedCount;
+  } catch {
+    return 0;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Fetch SEC data (13F + Form 4) for a symbol and store in the database.
+ * Resolves the symbol's CIK from EDGAR, then fetches and parses filings.
+ *
+ * @returns { filingsStored, transactionsStored } counts
+ */
+export async function fetchAndStoreSecData(
+  db: DatabaseSync,
+  symbol: string,
+): Promise<{ filingsStored: number; transactionsStored: number; cik: string | null }> {
+  const upper = symbol.toUpperCase();
+
+  // Resolve CIK
+  const cik = await resolveCik(upper);
+  if (!cik) {
+    return { filingsStored: 0, transactionsStored: 0, cik: null };
+  }
+
+  // Fetch 13F and Form 4 in parallel
+  const [filingsStored, transactionsStored] = await Promise.all([
+    fetchAndStore13F(db, upper, cik),
+    fetchAndStoreForm4(db, upper, cik),
+  ]);
+
+  return { filingsStored, transactionsStored, cik };
+}