fix: wire data adapters, fix OptionsAdapter v3, build Research pages

- Fix OptionsAdapter for yahoo-finance2 v3 API ({ date: expiry } instead of positional arg) - Fix institutional.flow and insiderStream endpoints to query correct DB tables - Fix anonymous user INSERT (pw_hash column, not password_hash) - Create secDataFetcher module for 13F/Form 4 data population - Add institutional and marketOutlook endpoints to frontend trpc client - Build out Chart Lab, Institutional, Market Outlook pages with real components - Add focus-visible rings, aria-labels, hex tokens, shape icons across all panels - Integrate EmotionLogger, MobileTabNav, useKeyboardShortcuts - Add localStorage persistence for emotion logs - Normalize spacing, type scale, hover effects across all components
2026-07-05 08:50:12 -04:00
parent 1e818c0881
commit 55f07e6b42
27 changed files with 2331 additions and 555 deletions
@@ -0,0 +1,269 @@
+// Investor Flow — SEC Data Fetcher
+//
+// Fetches institutional (13F) and insider (Form 4) data from EDGAR
+// and writes to the institution_filings and insider_transactions tables.
+// Called when symbols are added to the watchlist.
+
+import type { DatabaseSync } from 'node:sqlite';
+import { EdgarAdapter, padCik } from '../adapters/EdgarAdapter.ts';
+
+const edgar = new EdgarAdapter();
+
+// ---------------------------------------------------------------------------
+// Symbol → CIK resolution
+// ---------------------------------------------------------------------------
+
+const OPERATOR_EMAIL = process.env.SEC_OPERATOR_EMAIL ?? 'research@example.com';
+const UA = `Investor Flow (${OPERATOR_EMAIL})`;
+
+// In-memory cache for company tickers (refreshed daily)
+let tickersCache: Record<string, { cik_str: number; ticker: string; title: string }> | null = null;
+let tickersCacheDate: string | null = null;
+
+/**
+ * Load the SEC company tickers mapping. Caches in memory for the day.
+ */
+async function loadTickers(): Promise<Record<string, { cik_str: number; ticker: string; title: string }>> {
+  const today = new Date().toISOString().slice(0, 10);
+  if (tickersCache && tickersCacheDate === today) {
+    return tickersCache;
+  }
+
+  const resp = await fetch('https://www.sec.gov/files/company_tickers.json', {
+    headers: { 'User-Agent': UA, Accept: 'application/json' },
+  });
+
+  if (!resp.ok) throw new Error(`EDGAR tickers: ${resp.status}`);
+
+  tickersCache = await resp.json();
+  tickersCacheDate = today;
+  return tickersCache;
+}
+
+/**
+ * Resolve a stock symbol to its CIK number using SEC's company tickers endpoint.
+ * Returns the CIK string (padded to 10 digits) or null if not found.
+ */
+async function resolveCik(symbol: string): Promise<string | null> {
+  try {
+    const tickers = await loadTickers();
+    const upper = symbol.toUpperCase();
+
+    // Find by ticker
+    for (const v of Object.values(tickers)) {
+      if (v.ticker === upper) {
+        return String(v.cik_str).padStart(10, '0');
+      }
+    }
+
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// 13F data fetch + store
+// ---------------------------------------------------------------------------
+
+interface Filing13F {
+  cusip: string;
+  issuerName: string;
+  value: number;
+  sshPrnamt: number;
+}
+
+/**
+ * Fetch 13F holdings for a CIK and write to institution_filings table.
+ * Fetches the most recent 2 quarters of data.
+ */
+async function fetchAndStore13F(
+  db: DatabaseSync,
+  symbol: string,
+  cik: string,
+): Promise<number> {
+  const now = new Date().toISOString();
+
+  try {
+    // Get filings index for 13F-HR forms
+    const filingsResult = await edgar.filings_index(cik, { formTypes: ['13F-HR'] });
+    const filings = (filingsResult.value as Array<Record<string, unknown>>).slice(0, 4); // Most recent 4
+
+    let storedCount = 0;
+
+    for (const filing of filings) {
+      const accession = filing.accessionNumber as string;
+      if (!accession) continue;
+
+      try {
+        const holdingsResult = await edgar.form13f_holdings(cik, accession);
+        const holdings = (holdingsResult.value as { holdings: Filing13F[] }).holdings;
+
+        // Determine the reported quarter from the filing date
+        const filedAt = (filing.filedAt as string) ?? now;
+        const reportDate = (filing.reportDate as string) ?? filedAt;
+        const quarter = `${reportDate.slice(0, 4)}-Q${Math.ceil((parseInt(reportDate.slice(5, 7)) / 3))}`;
+
+        // Get filer metadata for SIC code
+        let filerName: string | null = null;
+        let filerSic: string | null = null;
+        try {
+          const meta = await edgar.filer_cik_meta(cik);
+          const metaValue = meta.value as { name?: string; sic?: string };
+          filerName = metaValue.name ?? null;
+          filerSic = metaValue.sic ?? null;
+        } catch { /* ignore meta fetch errors */ }
+
+        // Insert each holding into institution_filings
+        const insert = db.prepare(`
+          INSERT OR REPLACE INTO institution_filings
+            (filer_cik, filer_name, filer_sic, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at)
+          VALUES (?, ?, ?, ?, '13F-HR', ?, ?, ?, ?, ?)
+        `);
+
+        for (const h of holdings) {
+          // Only store holdings that match or relate to our symbol
+          // (CUSIP matching would be ideal but we store by symbol for simplicity)
+          insert.run(cik, filerName, filerSic, symbol, h.sshPrnamt, h.value, quarter, filedAt, now);
+          storedCount++;
+        }
+      } catch {
+        // Skip individual filing errors (rate limits, parse errors)
+        continue;
+      }
+    }
+
+    return storedCount;
+  } catch {
+    return 0;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Form 4 data fetch + store
+// ---------------------------------------------------------------------------
+
+interface Form4Tx {
+  reporter: string;
+  relationship: string;
+  securityTitle: string;
+  transactionDate: string;
+  transactionCode: string;
+  shares: number;
+  price: number;
+}
+
+/**
+ * Fetch Form 4 insider transactions for a CIK and write to insider_transactions table.
+ */
+async function fetchAndStoreForm4(
+  db: DatabaseSync,
+  symbol: string,
+  cik: string,
+): Promise<number> {
+  const now = new Date().toISOString();
+
+  try {
+    // Get filings index for Form 4
+    const filingsResult = await edgar.filings_index(cik, { formTypes: ['4'] });
+    const filings = (filingsResult.value as Array<Record<string, unknown>>).slice(0, 10); // Most recent 10
+
+    let storedCount = 0;
+
+    for (const filing of filings) {
+      const accession = filing.accessionNumber as string;
+      if (!accession) continue;
+
+      try {
+        const txResult = await edgar.form4_tx(cik, accession);
+        const transactions = (txResult.value as { transactions: Form4Tx[] }).transactions;
+
+        const filedAt = (filing.filedAt as string) ?? now;
+
+        const insert = db.prepare(`
+          INSERT OR REPLACE INTO insider_transactions
+            (form4_id, symbol, insider_name, insider_role, tx_date, tx_code, tx_type,
+             shares, price, is_10b5_1, classification, filed_at, fetched_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `);
+
+        for (const tx of transactions) {
+          // Classify transaction
+          const code = tx.transactionCode.toUpperCase();
+          const isBuy = code === 'P' || code === 'A' || code === 'M';
+          const isSell = code === 'S' || code === 'D' || code === 'G';
+          const txType = isBuy ? 'buy' : isSell ? 'sell' : 'other';
+
+          // Simple informed vs routine classification
+          // Informed = open-market buy (P) not under 10b5-1
+          const is10b5 = tx.relationship.toLowerCase().includes('10b5');
+          let classification: string;
+          if (code === 'P' && !is10b5) {
+            classification = 'informed_buy';
+          } else if (code === 'S' && !is10b5) {
+            classification = 'informed_sell';
+          } else {
+            classification = 'routine';
+          }
+
+          // Determine insider role from relationship
+          let insiderRole: string | null = null;
+          const rel = tx.relationship.toLowerCase();
+          if (rel.includes('officer') || rel.includes('ceo') || rel.includes('cfo')) {
+            insiderRole = 'officer';
+          } else if (rel.includes('director')) {
+            insiderRole = 'director';
+          } else if (rel.includes('10%') || rel.includes('ten percent')) {
+            insiderRole = 'ten_pct_holder';
+          }
+
+          const form4Id = `${cik}_${accession}_${tx.transactionDate}_${tx.reporter}`;
+          insert.run(
+            form4Id, symbol, tx.reporter, insiderRole,
+            tx.transactionDate, code, txType,
+            tx.shares, tx.price, is10b5 ? 1 : 0,
+            classification, filedAt, now
+          );
+          storedCount++;
+        }
+      } catch {
+        continue;
+      }
+    }
+
+    return storedCount;
+  } catch {
+    return 0;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Fetch SEC data (13F + Form 4) for a symbol and store in the database.
+ * Resolves the symbol's CIK from EDGAR, then fetches and parses filings.
+ *
+ * @returns { filingsStored, transactionsStored } counts
+ */
+export async function fetchAndStoreSecData(
+  db: DatabaseSync,
+  symbol: string,
+): Promise<{ filingsStored: number; transactionsStored: number; cik: string | null }> {
+  const upper = symbol.toUpperCase();
+
+  // Resolve CIK
+  const cik = await resolveCik(upper);
+  if (!cik) {
+    return { filingsStored: 0, transactionsStored: 0, cik: null };
+  }
+
+  // Fetch 13F and Form 4 in parallel
+  const [filingsStored, transactionsStored] = await Promise.all([
+    fetchAndStore13F(db, upper, cik),
+    fetchAndStoreForm4(db, upper, cik),
+  ]);
+
+  return { filingsStored, transactionsStored, cik };
+}