investor-flow/app/server/src/cache/CacheRepository.ts

// Investor Flow — CacheRepository deep module (DESIGN.md §3a Module 1).
// The ONLY interface the SPA (via tRPC) touches for cached data. Owns staleness windows,
// refcount/demand-set, and stale-while-revalidate. Does NOT talk to external sources —
// that is SourceAdapter's job; CacheRepository only schedules background refreshes via
// the injected scheduler (SourceAdapter/AdapterQueue satisfy `CacheScheduler`).
import { DatabaseSync } from 'node:sqlite';
import { db as defaultDb } from '../db/client.ts';
import {
  CANDLE_FRESH_MS,
  quoteTtlMs,
  tieredQuoteTtlMs,
  SYMBOL_META_INCOMPLETE_TTL_MS,
} from '../queue/sourceRatePolicy.ts';
import { KvReadCache } from './LruCache.ts';

export type SourceKind = 'yfinance' | 'nasdaq' | 'finra-bulk' | 'finra-si' | 'sec' | 'sec-fetch' | 'sec-sc-fetch' | 'sec-tickers' | 'reddit' | 'x' | 'macro' | 'llm' | 'sec-lint-holders' | 'sec-lint-insiders' | 'fred' | 'cot';
export type TickerKind = 'equity' | 'crypto' | 'etf' | 'index';
export type CacheKey = string; // `${SourceKind}:${kind}:${id}` e.g. 'yfinance:quote:NVDA', 'yfinance:candles:NVDA:1d'
export type TtlClass =
  | 'live_quote' | 'intraday' | 'daily_permanent' | 'options_snapshot'
  | 'filing_immutable' | 'quarterly_immutable' | 'thread_7d'
  | 'macro_event' | 'regime_classification' | 'llm_summary' | 'symbol_meta'
  | 'short_interest' | 'dividend_fundamentals' | 'fred_macro' | 'cot_weekly';

export interface Provenance { fetchedAt: string; sourceKind: SourceKind; rawSourceId?: string; }

export interface Quote {
  symbol: string;
  price: number;
  bid?: number | null;
  ask?: number | null;
  change?: number | null;
  changePercent?: number | null;
  iv?: number | null;
  /** Which session the `price` print came from (pre/post when Yahoo has extended data). */
  session?: 'regular' | 'pre' | 'post' | null;
  /** RTH last (close during extended); useful when `price` is pre/post. */
  regularPrice?: number | null;
}
export interface PriceCandle { ts: string; o: number; h: number; l: number; c: number; v: number; adjClose?: number | null; }
export interface SymbolMeta { symbol: string; name?: string | null; sector?: string | null; industry?: string | null; exchange?: string | null; tickerKind: TickerKind; peers?: string[] | null; description?: string | null; website?: string | null; marketCap?: number | null; currency?: string | null; employees?: number | null; country?: string | null; city?: string | null; }
export interface PriceAdjustment { symbol: string; exDate: string; type: "split" | "dividend"; ratio: number }

/** Port CacheRepository depends on to schedule background refreshes. SourceAdapter/AdapterQueue satisfy this. */
export interface CacheScheduler { queue(key: CacheKey): Promise<void>; }

export interface CacheEntry<T> { value: T | null; provenance: Provenance | null; isStale: boolean; }

// Module-level LRU cache for kv_cache reads — shared across all handler instances
let kvReadCache = new KvReadCache({ max: 2000, ttl: 5 * 60 * 1000 });

/** Set the LRU cache instance (for testing or runtime configuration) */
export function setKvReadCache(cache: KvReadCache): void {
  kvReadCache = cache;
}

export const TTL_MS: Record<TtlClass, number> = {
  live_quote: 60_000,                       // 1min (mkt hrs); after-hours uses quoteTtlMs()
  intraday: 5 * 60_000,
  daily_permanent: Number.POSITIVE_INFINITY, // full history kept; EOD refresh is schedule-driven
  options_snapshot: 15 * 60_000,
  filing_immutable: Number.POSITIVE_INFINITY,
  quarterly_immutable: Number.POSITIVE_INFINITY,
  thread_7d: 7 * 24 * 60 * 60_000,
  macro_event: Number.POSITIVE_INFINITY,
  regime_classification: 24 * 60 * 60_000,
  llm_summary: Number.POSITIVE_INFINITY,    // cached forever by prompt-hash
  symbol_meta: 7 * 24 * 60 * 60_000,         // weekly (slow-changing sector/industry)
  short_interest: 24 * 60 * 60_000,      // refreshed twice/month per source
  dividend_fundamentals: 7 * 24 * 60 * 60_000, // weekly (yield/payout change slowly)
  fred_macro: 24 * 60 * 60_000,          // daily (rates move daily; other series slower)
  cot_weekly: 7 * 24 * 60 * 60_000,      // COT arrives weekly on Fridays
};

/** Parse 'source:kind:id...' into { source, kind, id } (id may contain colons). */
export function parseCacheKey(key: CacheKey): { source: SourceKind; kind: string; id: string } {
  const parts = key.split(':');
  if (parts.length < 3) throw new Error(`invalid cache key: ${key}`);
  const source = parts[0] as SourceKind;
  const kind = parts[1];
  const id = parts.slice(2).join(':');
  return { source, kind, id };
}

interface KindHandler {
  ttlClass: TtlClass;
  /** Read cached value + the timestamp to judge staleness against. null if not cached. */
  read(d: DatabaseSync, id: string): { value: unknown; stalenessTs: string } | null;
  /** Write value to cache, stamping provenance. */
  write(d: DatabaseSync, id: string, value: unknown, provenance: Provenance): void;
  /** Staleness verdict given the staleness timestamp (null = never cached) and now. */
  isStale(stalenessTs: string | null, now: number): boolean;
}

function tsAgeMs(ts: string | null, now: number): number {
  if (!ts) return Number.POSITIVE_INFINITY;
  const t = Date.parse(ts);
  if (Number.isNaN(t)) return Number.POSITIVE_INFINITY;
  return now - t;
}

// ----- Kind handlers (slice 1: quote, candles, symbol). Later slices register more. -----
const quoteHandler: KindHandler = {
  ttlClass: 'live_quote',
  read(d, symbol) {
    const r = d.prepare(
      'SELECT symbol,price,bid,ask,change,change_pct,iv,session,regular_price,observed_at FROM quotes WHERE symbol=?',
    ).get(symbol) as Record<string, unknown> | undefined;
    if (!r) return null;
    const sessionRaw = r.session == null ? null : String(r.session);
    const session =
      sessionRaw === 'regular' || sessionRaw === 'pre' || sessionRaw === 'post'
        ? sessionRaw
        : null;
    return {
      value: {
        symbol: r.symbol,
        price: r.price,
        bid: r.bid,
        ask: r.ask,
        change: r.change,
        changePercent: r.change_pct,
        iv: r.iv,
        session,
        regularPrice: r.regular_price as number | null | undefined,
      } as Quote,
      stalenessTs: r.observed_at as string,
    };
  },
  write(d, symbol, value, provenance) {
    const q = value as Quote;
    d.prepare(
      'INSERT OR REPLACE INTO quotes (symbol,price,bid,ask,change,change_pct,iv,session,regular_price,observed_at) VALUES (?,?,?,?,?,?,?,?,?,?)',
    ).run(
      symbol,
      q.price,
      q.bid ?? null,
      q.ask ?? null,
      q.change ?? null,
      q.changePercent ?? null,
      q.iv ?? null,
      q.session ?? null,
      q.regularPrice ?? null,
      provenance.fetchedAt,
    );
  },
  isStale(ts, now) { return tsAgeMs(ts, now) > quoteTtlMs(new Date(now)); },
};

const candlesHandler: KindHandler = {
  ttlClass: 'daily_permanent',
  read(d, id) {
    const [symbol, timeframe] = id.split(':');
    if (!timeframe) return null;
    const rows = d.prepare('SELECT ts,o,h,l,c,v,adj_close,observed_at FROM price_candles WHERE symbol=? AND timeframe=? ORDER BY ts ASC').all(symbol, timeframe) as Array<Record<string, unknown>>;
    if (!rows.length) return null;
    const value: PriceCandle[] = rows.map((r) => ({ ts: r.ts as string, o: r.o as number, h: r.h as number, l: r.l as number, c: r.c as number, v: r.v as number, adjClose: r.adj_close as number | null }));
    // Staleness for SWR uses last *bar* ts (not observed_at) so schedule + get agree on lag.
    return { value, stalenessTs: rows[rows.length - 1].ts as string };
  },
  write(d, id, value, provenance) {
    const [symbol, timeframe] = id.split(':');
    const ins = d.prepare('INSERT OR REPLACE INTO price_candles (symbol,timeframe,ts,o,h,l,c,v,adj_close,observed_at) VALUES (?,?,?,?,?,?,?,?,?,?)');
    for (const c of value as PriceCandle[]) ins.run(symbol, timeframe, c.ts, c.o, c.h, c.l, c.c, c.v, c.adjClose ?? null, provenance.fetchedAt);
  },
  // Do not thrash on every read: schedule owns EOD refresh. Stale only when absent.
  isStale(ts) { return ts === null; },
};

/** True when daily candles are missing or last bar is older than CANDLE_FRESH_MS. */
export function needsCandleRefresh(d: DatabaseSync, symbol: string, timeframe = '1d', now = Date.now()): boolean {
  const row = d.prepare(
    'SELECT MAX(ts) AS last_ts FROM price_candles WHERE symbol=? AND timeframe=?',
  ).get(symbol, timeframe) as { last_ts: string | null } | undefined;
  if (!row?.last_ts) return true;
  return tsAgeMs(row.last_ts, now) > CANDLE_FRESH_MS;
}

/** Last daily candle bar timestamp, or null if none. */
export function lastCandleTs(d: DatabaseSync, symbol: string, timeframe = '1d'): string | null {
  const row = d.prepare(
    'SELECT MAX(ts) AS last_ts FROM price_candles WHERE symbol=? AND timeframe=?',
  ).get(symbol, timeframe) as { last_ts: string | null } | undefined;
  return row?.last_ts ?? null;
}

/** True when quote is missing or past session-aware TTL. */
export function needsQuoteRefresh(d: DatabaseSync, symbol: string, now = Date.now()): boolean {
  const row = d.prepare('SELECT observed_at FROM quotes WHERE symbol=?').get(symbol) as { observed_at: string } | undefined;
  if (!row?.observed_at) return true;
  return tsAgeMs(row.observed_at, now) > quoteTtlMs(new Date(now));
}

/** Tier-aware quote freshness check. Portfolio (T0) gets the tightest TTL. */
export function needsTieredQuoteRefresh(d: DatabaseSync, symbol: string, tier: number, now = Date.now()): boolean {
  const row = d.prepare('SELECT observed_at FROM quotes WHERE symbol=?').get(symbol) as { observed_at: string } | undefined;
  if (!row?.observed_at) return true;
  return tsAgeMs(row.observed_at, now) > tieredQuoteTtlMs(tier, new Date(now));
}

/** True when symbol meta missing, incomplete (no name), or past weekly TTL. */
export function needsSymbolMetaRefresh(d: DatabaseSync, symbol: string, now = Date.now()): boolean {
  const row = d.prepare('SELECT name, sector, updated_at FROM symbols WHERE symbol=?').get(symbol) as
    | { name: string | null; sector: string | null; updated_at: string | null }
    | undefined;
  if (!row?.updated_at) return true;
  const incomplete = !row.name;
  const ttl = incomplete ? SYMBOL_META_INCOMPLETE_TTL_MS : TTL_MS.symbol_meta;
  return tsAgeMs(row.updated_at, now) > ttl;
}

const adjustmentsHandler: KindHandler = {
  ttlClass: 'daily_permanent',
  read(d, symbol) {
    const rows = d.prepare('SELECT ex_date, type, ratio FROM price_adjustments WHERE symbol=? ORDER BY ex_date ASC').all(symbol) as Array<Record<string, unknown>>;
    if (!rows.length) return null;
    const value: PriceAdjustment[] = rows.map((r) => ({ symbol, exDate: r.ex_date as string, type: r.type as "split" | "dividend", ratio: r.ratio as number }));
    return { value, stalenessTs: rows[rows.length - 1].ex_date as string };
  },
  write(d, _symbol, value, provenance) {
    const ins = d.prepare('INSERT OR REPLACE INTO price_adjustments (symbol, ex_date, type, ratio) VALUES (?,?,?,?)');
    for (const a of value as PriceAdjustment[]) ins.run(a.symbol, a.exDate, a.type, a.ratio);
  },
  isStale(ts) { return ts === null; }, // permanent: stale only when absent
};

const symbolHandler: KindHandler = {
  ttlClass: 'symbol_meta',
  read(d, symbol) {
    const r = d.prepare('SELECT symbol,name,sector,industry,exchange,ticker_kind,peers,updated_at FROM symbols WHERE symbol=?').get(symbol) as Record<string, unknown> | undefined;
    if (!r) return null;
    let peers: string[] | null = null;
    if (typeof r.peers === 'string') { try { peers = JSON.parse(r.peers); } catch { peers = null; } }
    return {
      value: { symbol: r.symbol, name: r.name, sector: r.sector, industry: r.industry, exchange: r.exchange, tickerKind: r.ticker_kind, peers } as SymbolMeta,
      stalenessTs: r.updated_at as string,
    };
  },
  write(d, symbol, value, provenance) {
    const s = value as SymbolMeta;
    d.prepare('INSERT OR REPLACE INTO symbols (symbol,name,sector,industry,exchange,ticker_kind,peers,updated_at) VALUES (?,?,?,?,?,?,?,?)')
      .run(symbol, s.name ?? null, s.sector ?? null, s.industry ?? null, s.exchange ?? null, s.tickerKind, s.peers ? JSON.stringify(s.peers) : null, provenance.fetchedAt);
  },
  isStale(ts, now) {
    // Age-only here; incomplete rows (null name) are handled in needsSymbolMetaRefresh + get override.
    return tsAgeMs(ts, now) > TTL_MS.symbol_meta;
  },
};

// ----- Options handlers (slice 15) -----
const optionsChainHandler: KindHandler = {
  ttlClass: 'options_snapshot',
  read(d, id) {
    const [symbol, expiry] = id.split(':');
    if (!expiry) return null;
    const rows = d.prepare(
      'SELECT symbol,expiry,strike,type,bid,ask,iv,delta,gamma,theta,vega,open_interest,volume,ts FROM options_chains WHERE symbol=? AND expiry=? ORDER BY strike ASC, type ASC'
    ).all(symbol, expiry) as Array<Record<string, unknown>>;
    if (!rows.length) return null;
    const value = rows.map((r) => ({
      contractSymbol: `${r.symbol}_${r.expiry}_${r.strike}_${r.type}`,
      strike: r.strike as number,
      right: r.type as 'call' | 'put',
      expiration: r.expiry as string,
      bid: r.bid as number | null,
      ask: r.ask as number | null,
      impliedVolatility: r.iv as number | null,
      delta: r.delta as number | null,
      gamma: r.gamma as number | null,
      theta: r.theta as number | null,
      vega: r.vega as number | null,
      openInterest: r.open_interest as number | null,
      volume: r.volume as number | null,
    }));
    return { value, stalenessTs: rows[rows.length - 1].ts as string };
  },
  write(d, id, value, provenance) {
    const [symbol, expiry] = id.split(':');
    if (!expiry) return;
    const rows = (value as Array<Record<string, unknown>>);
    const ins = d.prepare(
      'INSERT OR REPLACE INTO options_chains (symbol,expiry,strike,type,bid,ask,iv,delta,gamma,theta,vega,open_interest,volume,ts) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
    );
    for (const r of rows) {
      const strike = typeof r.strike === 'number' ? r.strike : 0;
      const right = r.right === 'put' ? 'put' : 'call';
      // OptionChainRow nests greeks; accept flat fields too for older writers.
      const g = (r.greeks && typeof r.greeks === 'object')
        ? (r.greeks as Record<string, unknown>)
        : null;
      ins.run(
        symbol, expiry, strike, right,
        numOrNull(r.bid), numOrNull(r.ask), numOrNull(r.impliedVolatility),
        numOrNull(g?.delta ?? r.delta), numOrNull(g?.gamma ?? r.gamma),
        numOrNull(g?.theta ?? r.theta), numOrNull(g?.vega ?? r.vega),
        numOrNull(r.openInterest), numOrNull(r.volume),
        provenance.fetchedAt
      );
    }
  },
  isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.options_snapshot; },
};

// Helper: LRU cache-aware read wrapper for handlers that use kv_cache
function withKvCacheRead(
  handler: KindHandler,
  getCacheKey: (id: string) => string | null,
): KindHandler {
  return {
    ...handler,
    read(d, id) {
      const cacheKey = getCacheKey(id);
      if (!cacheKey) return handler.read(d, id);
      
      // Check LRU cache first
      const cached = kvReadCache.get(cacheKey);
      if (cached) return cached as { value: unknown; stalenessTs: string } | null;
      
      const result = handler.read(d, id);
      
      // Populate LRU cache on hit
      if (result && !cacheKey.includes(':*')) {
        kvReadCache.set(cacheKey, result.value as unknown, result.stalenessTs);
      }
      
      return result;
    },
  };
}

const optionsExpiryDatesHandler: KindHandler = {
  ttlClass: 'intraday',
  read(d, symbol) {
    const r = d.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(`options_expiry:${symbol}`) as Record<string, unknown> | undefined;
    if (!r) return null;
    try {
      const value = JSON.parse(r.value as string);
      return { value, stalenessTs: r.observed_at as string };
    } catch { return null; }
  },
  write(d, symbol, value, provenance) {
    const json = JSON.stringify(value);
    d.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)')
      .run(`options_expiry:${symbol}`, json, provenance.fetchedAt);
  },
  isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.intraday; },
};

/** Coerce a possibly-undefined/unknown value to number | null for SQL binding. */
function numOrNull(v: unknown): number | null {
  return typeof v === 'number' ? v : null;
}

const greeksHandler: KindHandler = {
  ttlClass: 'options_snapshot',
  read(d, id) {
    const { symbol, expiry, strike } = parseGreeksId(id);
    const r = d.prepare(
      'SELECT delta, gamma, theta, vega, strike, open_interest, iv, ts, type FROM options_chains WHERE symbol=? AND expiry=? AND strike=? LIMIT 1'
    ).get(symbol, expiry, parseFloat(strike ?? '0')) as Record<string, unknown> | undefined;
    if (!r) return null;
    return {
      value: {
        delta: numOrNull(r.delta),
        gamma: numOrNull(r.gamma),
        theta: numOrNull(r.theta),
        vega: numOrNull(r.vega),
        strike: typeof r.strike === 'number' ? r.strike : 0,
        openInterest: numOrNull(r.open_interest),
        impliedVolatility: numOrNull(r.iv),
        right: r.type as 'call' | 'put',
      },
      stalenessTs: r.ts as string,
    };
  },
  write(d, id, value, provenance) {
    const { symbol, expiry, strike } = parseGreeksId(id);
    const v = value as Record<string, unknown>;
    const right = v.right === 'put' ? 'put' : 'call';
    d.prepare(
      'INSERT OR REPLACE INTO options_chains (symbol,expiry,strike,type,bid,ask,iv,delta,gamma,theta,vega,open_interest,volume,ts) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
    ).run(
      symbol, expiry, parseFloat(strike ?? '0'), right,
      numOrNull(v.bid), numOrNull(v.ask), numOrNull(v.impliedVolatility),
      numOrNull(v.delta), numOrNull(v.gamma), numOrNull(v.theta), numOrNull(v.vega),
      numOrNull(v.openInterest), numOrNull(v.volume),
      provenance.fetchedAt
    );
  },
  isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.options_snapshot; },
};

/** Parse 'symbol:expiry:strike' from greeks cache key id. */
function parseGreeksId(id: string): { symbol: string; expiry: string; strike: string } {
  const parts = id.split(':');
  return { symbol: parts[0] ?? '', expiry: parts[1] ?? '', strike: parts[2] ?? '0' };
}

const fetchHandler: KindHandler = {
  ttlClass: 'daily_permanent',
  read(d, id) {
    const cacheKey = `sec-fetch:${id}`;
    const r = d.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(cacheKey) as { value: string; observed_at: string } | undefined;
    if (!r) return null;
    try {
      const value = JSON.parse(r.value);
      return { value, stalenessTs: r.observed_at };
    } catch { return null; }
  },
  write(d, id, value, provenance) {
    d.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)').run(`sec-fetch:${id}`, JSON.stringify(value), provenance.fetchedAt);
  },
  isStale(ts) { return ts === null; }, // never stale once written
};

const lintHoldersHandler: KindHandler = {
  ttlClass: 'daily_permanent',
  read() { return null; }, // never read — work happens in DB tables directly
  write(d, key, _value, provenance) {
    d.prepare('INSERT OR REPLACE INTO kv_cache (key,value,observed_at) VALUES (?,?,?)').run(`sec-lint-holders:${key}`, JSON.stringify({ ok: true }), provenance.fetchedAt);
  },
  isStale() { return false; }, // never stale once written (lint writes are permanent)
};

const lintInsidersHandler: KindHandler = {
  ttlClass: 'daily_permanent',
  read() { return null; },
  write(d, key, _value, provenance) {
    d.prepare('INSERT OR REPLACE INTO kv_cache (key,value,observed_at) VALUES (?,?,?)').run(`sec-lint-insiders:${key}`, JSON.stringify({ ok: true }), provenance.fetchedAt);
  },
  isStale() { return false; },
};

const shortInterestHandler: KindHandler = {
  ttlClass: 'short_interest',
  read(d, id) {
    const cacheKey = `yfinance:shortinterest:${id}`;
    const r = d.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(cacheKey) as { value: string; observed_at: string } | undefined;
    if (!r) return null;
    try {
      const value = JSON.parse(r.value);
      return { value, stalenessTs: r.observed_at };
    } catch { return null; }
  },
  write(d, id, value, provenance) {
    d.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)').run(`yfinance:shortinterest:${id}`, JSON.stringify(value), provenance.fetchedAt);
  },
  isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.short_interest; },
};

const nasdaqShortinterestHandler: KindHandler = {
  ttlClass: 'short_interest',
  read(d, id) {
    const cacheKey = `nasdaq:nasdaqShortinterest:${id}`;
    const r = d.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(cacheKey) as { value: string; observed_at: string } | undefined;
    if (!r) return null;
    try {
      const value = JSON.parse(r.value);
      return { value, stalenessTs: r.observed_at };
    } catch { return null; }
  },
  write(d, id, value, provenance) {
    d.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)').run(`nasdaq:nasdaqShortinterest:${id}`, JSON.stringify(value), provenance.fetchedAt);
  },
  isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.short_interest; },
};

const finraShortInterestHandler: KindHandler = {
  ttlClass: 'short_interest',
  read(d, symbol) {
    // Return the most recent settlement date's data for a given symbol
    const r = d.prepare(
      `SELECT short_volume, short_exempt, total_volume, avg_daily_vol, days_to_cover, settlement_date, exchange, ingested_at
       FROM finra_short_interest WHERE symbol=? ORDER BY settlement_date DESC LIMIT 1`
    ).get(symbol) as Record<string, unknown> | undefined;
    if (!r) return null;
    return {
      value: {
        shortVolume: r.short_volume as number,
        shortExempt: r.short_exempt as number,
        totalVolume: r.total_volume as number,
        avgDailyVol: r.avg_daily_vol as number | null,
        daysToCover: r.days_to_cover as number | null,
        settlementDate: r.settlement_date as string,
        exchange: r.exchange as string | null,
      },
      stalenessTs: r.ingested_at as string,
    };
  },
  write(_d, _symbol, _value, _provenance) {
    // FINRA data is bulk-ingested via FinraIngestService; no per-symbol write needed.
  },
  isStale(ts, now) {
    if (ts === null) return false; // schedule-only; never queue per-symbol fetches
    return tsAgeMs(ts, now) > TTL_MS.short_interest;
  },
};

const finraSiBiweeklyHandler: KindHandler = {
  ttlClass: 'short_interest',
  read(d, symbol) {
    const r = d.prepare(
      `SELECT current_short_position, previous_short_position, avg_daily_volume, days_to_cover, change_percent, settlement_date, ingested_at
       FROM finra_short_interest_biweekly WHERE symbol=? ORDER BY settlement_date DESC LIMIT 1`
    ).get(symbol) as Record<string, unknown> | undefined;
    if (!r) return null;
    return {
      value: {
        currentShortPosition: r.current_short_position as number,
        previousShortPosition: r.previous_short_position as number | null,
        avgDailyVolume: r.avg_daily_volume as number | null,
        daysToCover: r.days_to_cover as number | null,
        changePercent: r.change_percent as number | null,
        settlementDate: r.settlement_date as string,
      },
      stalenessTs: r.ingested_at as string,
    };
  },
  write(_d, _symbol, _value, _provenance) {},
  isStale(ts, now) {
    if (ts === null) return false;
    return tsAgeMs(ts, now) > TTL_MS.short_interest;
  },
};

const dividendFundamentalsHandler: KindHandler = {
  ttlClass: 'dividend_fundamentals',
  read(d, symbol) {
    const r = d.prepare(
      `SELECT dividend_yield, payout_ratio, dividend_rate, ex_dividend_date, trailing_eps, forward_pe, growth_streak_years, observed_at FROM dividend_fundamentals WHERE symbol=?`
    ).get(symbol) as Record<string, unknown> | undefined;
    if (!r) return null;
    return { value: { symbol, dividendYield: r.dividend_yield, payoutRatio: r.payout_ratio, dividendRate: r.dividend_rate, exDividendDate: r.ex_dividend_date, trailingEps: r.trailing_eps, forwardPE: r.forward_pe, growthStreakYears: r.growth_streak_years }, stalenessTs: r.observed_at as string };
  },
  write(d, symbol, value, provenance) {
    d.prepare('INSERT OR REPLACE INTO dividend_fundamentals (symbol, dividend_yield, payout_ratio, dividend_rate, ex_dividend_date, trailing_eps, forward_pe, growth_streak_years, observed_at) VALUES (?,?,?,?,?,?,?,?)').run(symbol, (value as any).dividendYield ?? null, (value as any).payoutRatio ?? null, (value as any).dividendRate ?? null, (value as any).exDividendDate ?? null, (value as any).trailingEps ?? null, (value as any).forwardPE ?? null, (value as any).growthStreakYears ?? null, provenance.fetchedAt);
  },
  isStale(ts, now) { return ts ? tsAgeMs(ts, now) > 7 * 86400000 : true; },
};

/** ETF composition — same kv key as market.sectorHoldings for cache sharing. */
const topHoldingsHandler: KindHandler = {
  ttlClass: 'symbol_meta',
  read(d, symbol) {
    const key = `etf:topHoldings:${symbol.toUpperCase()}`;
    const r = d.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(key) as
      | { value: string; observed_at: string }
      | undefined;
    if (!r?.value) return null;
    try {
      const parsed = JSON.parse(r.value) as { holdings?: unknown };
      return { value: parsed, stalenessTs: r.observed_at };
    } catch {
      return null;
    }
  },
  write(d, symbol, value, provenance) {
    const key = `etf:topHoldings:${symbol.toUpperCase()}`;
    const payload = typeof value === 'string' ? value : JSON.stringify(value);
    d.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)')
      .run(key, payload, provenance.fetchedAt);
  },
  isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.symbol_meta; },
};

// Wrap kv_cache-reading handlers with LRU cache
const wrappedOptionsExpiryDates = withKvCacheRead(optionsExpiryDatesHandler, (id) => `options_expiry:${id}`);
const wrappedFetch = withKvCacheRead(fetchHandler, (id) => `sec-fetch:${id}`);
const wrappedShortInterest = withKvCacheRead(shortInterestHandler, (id) => `yfinance:shortinterest:${id}`);
const wrappedNasdaqShortInterest = withKvCacheRead(nasdaqShortinterestHandler, (id) => `nasdaq:nasdaqShortinterest:${id}`);
const wrappedTopHoldings = withKvCacheRead(topHoldingsHandler, (symbol) => `etf:topHoldings:${(symbol as string).toUpperCase()}`);

/** FRED macro series — kv_cache keyed `fred:series:{seriesId}` (warmed by the queue, off request path). */
const seriesHandler: KindHandler = {
  ttlClass: 'fred_macro',
  read(d, seriesId) {
    const cacheKey = `fred:series:${seriesId}`;
    const r = d.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(cacheKey) as
      | { value: string; observed_at: string }
      | undefined;
    if (!r?.value) return null;
    try {
      return { value: JSON.parse(r.value), stalenessTs: r.observed_at };
    } catch { return null; }
  },
  write(d, seriesId, value, provenance) {
    d.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)')
      .run(`fred:series:${seriesId}`, JSON.stringify(value), provenance.fetchedAt);
  },
  isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.fred_macro; },
};

const HANDLERS = new Map<string, KindHandler>([
  ['quote', quoteHandler],
  ['candles', candlesHandler],
  ['symbol', symbolHandler],
  ['adjustments', adjustmentsHandler],
  ['chain', optionsChainHandler],
  ['expiry_dates', wrappedOptionsExpiryDates],
  ['greeks', greeksHandler],
  ['fetch', wrappedFetch],
  ['holders', lintHoldersHandler],
  ['insiders', lintInsidersHandler],
  ['shortinterest', wrappedShortInterest],
  ['nasdaqShortinterest', wrappedNasdaqShortInterest],
  ['finraShortinterest', finraShortInterestHandler],
  ['finraSiBiweekly', finraSiBiweeklyHandler],
  ['dividendFundamentals', dividendFundamentalsHandler],
  ['topHoldings', wrappedTopHoldings],
  ['series', seriesHandler],
]);

export interface CacheRepository {
  get<T>(key: CacheKey): Promise<CacheEntry<T>>;
  set<T>(key: CacheKey, value: T, ttlClass: TtlClass, provenance: Provenance): Promise<void>;
  stale(key: CacheKey): boolean;
  /**
   * Watchlist/portfolio ownership: bumps refcount and seeds cache on first demand.
   * Do NOT call from page-view paths (rotation, seasonality, condition) — use ensureInDemand.
   */
  subscribe(symbol: string, tickerKind: TickerKind): Promise<void>;
  unsubscribe(symbol: string): Promise<void>;
  /**
   * Request-path pin: mark symbol in demand + queue missing/stale data without inflating refcount.
   * Safe to call on every Market Outlook / ticker context load.
   */
  ensureInDemand(symbol: string, tickerKind: TickerKind): Promise<void>;
  /** Bump symbol to watched tier (2) on page view. Decays back after 10 min. */
  bumpToWatched(symbol: string, tickerKind: TickerKind): Promise<void>;
  /** Permanent system pin (rotation universe, SPY, VIX) — survives unsubscribe. */
  pinSystemSymbol(symbol: string, tickerKind: TickerKind): Promise<void>;
  demandSet(): Promise<string[]>;
  getMany<T>(keys: CacheKey[]): Promise<Array<{ key: CacheKey; value: T | null; isStale: boolean }>>;
  /** Delete a cache entry by key (or, for wildcard keys ending in `:*`, all matching entries). */
  del(key: CacheKey): Promise<void>;
  /** Underlying DB for schedule TTL checks (queue only). */
  readonly db: DatabaseSync;
}

export class CacheRepositoryImpl implements CacheRepository {
  private readonly _db: DatabaseSync;
  private readonly _scheduler: CacheScheduler;
  constructor(opts: { db: DatabaseSync; scheduler: CacheScheduler }) {
    this._db = opts.db;
    this._scheduler = opts.scheduler;
  }
  get db(): DatabaseSync { return this._db; }
  private handler(kind: string): KindHandler {
    const h = HANDLERS.get(kind);
    if (!h) throw new Error(`unknown cache kind: ${kind}`);
    return h;
  }
  async get<T>(key: CacheKey): Promise<CacheEntry<T>> {
    const { source, kind, id } = parseCacheKey(key);
    const h = this.handler(kind);
    const row = h.read(this._db, id);
    const now = Date.now();
    let stale = h.isStale(row ? row.stalenessTs : null, now);
    // Incomplete symbol meta (null name) is always treated as stale for SWR re-fetch.
    if (kind === 'symbol' && row) {
      const meta = row.value as SymbolMeta;
      if (!meta?.name && tsAgeMs(row.stalenessTs, now) > SYMBOL_META_INCOMPLETE_TTL_MS) {
        stale = true;
      }
    }
    if (stale) {
      try { await this._scheduler.queue(key); } catch { /* background refresh; never block readers */ }
    }
    return {
      value: (row ? row.value : null) as T | null,
      provenance: row ? { fetchedAt: row.stalenessTs, sourceKind: source } : null,
      isStale: stale,
    };
  }
  async set<T>(key: CacheKey, value: T, ttlClass: TtlClass, provenance: Provenance): Promise<void> {
    const { kind, id } = parseCacheKey(key);
    const h = this.handler(kind);
    if (h.ttlClass !== ttlClass) throw new Error(`ttlClass mismatch for kind '${kind}': expected ${h.ttlClass}, got ${ttlClass}`);
    h.write(this._db, id, value, provenance);
    // Touch demand last_refreshed when a quote lands.
    if (kind === 'quote') {
      try {
        this._db.prepare('UPDATE symbol_demand SET last_refreshed_at=? WHERE symbol=?').run(provenance.fetchedAt, id);
      } catch { /* ignore */ }
    }
  }
  stale(key: CacheKey): boolean {
    const { kind, id } = parseCacheKey(key);
    const h = this.handler(kind);
    const row = h.read(this._db, id);
    if (kind === 'symbol' && row) {
      const meta = row.value as SymbolMeta;
      if (!meta?.name) return true;
    }
    return h.isStale(row ? row.stalenessTs : null, Date.now());
  }

  private ensureDemandRow(symbol: string, tickerKind: TickerKind): void {
    this._db.prepare(
      'INSERT OR IGNORE INTO symbol_demand (symbol,refcount,ticker_kind,in_demand,last_refreshed_at,system_pin) VALUES (?,?,?,?,?,?)',
    ).run(symbol, 0, tickerKind, 1, null, 0);
  }

  /** Queue only what is missing or past TTL — no refcount change. */
  private async queueIfNeeded(symbol: string): Promise<void> {
    const d = this._db;
    if (needsQuoteRefresh(d, symbol)) {
      try { await this._scheduler.queue(`yfinance:quote:${symbol}`); } catch { /* ignore */ }
    }
    if (needsCandleRefresh(d, symbol)) {
      try { await this._scheduler.queue(`yfinance:candles:${symbol}:1d`); } catch { /* ignore */ }
    }
    if (needsSymbolMetaRefresh(d, symbol)) {
      try { await this._scheduler.queue(`yfinance:symbol:${symbol}`); } catch { /* ignore */ }
    }
  }

  async subscribe(symbol: string, tickerKind: TickerKind): Promise<void> {
    const d = this._db;
    d.prepare('INSERT OR IGNORE INTO symbol_demand (symbol,refcount,ticker_kind,in_demand,last_refreshed_at) VALUES (?,?,?,?,?)').run(symbol, 0, tickerKind, 1, null);
    const prev = d.prepare('SELECT refcount FROM symbol_demand WHERE symbol=?').get(symbol) as { refcount: number } | undefined;
    const before = prev?.refcount ?? 0;
    d.prepare('UPDATE symbol_demand SET refcount = refcount + 1, in_demand = 1, ticker_kind=? WHERE symbol=?').run(tickerKind, symbol);
    if (before === 0) {
      // First user demand: full seed once (not every schedule tick).
      for (const k of [
        `yfinance:quote:${symbol}`,
        `yfinance:symbol:${symbol}`,
        `yfinance:candles:${symbol}:1d`,
        `yfinance:adjustments:${symbol}`,
        `yfinance:shortinterest:${symbol}`,
        `nasdaq:nasdaqShortinterest:${symbol}`,
      ]) {
        try { await this._scheduler.queue(k); } catch { /* ignore */ }
      }
    } else {
      // Subsequent demand: re-check staleness and queue missing kinds (no refcount change).
      await this.queueIfNeeded(symbol);
    }
  }

  async ensureInDemand(symbol: string, tickerKind: TickerKind): Promise<void> {
    this.ensureDemandRow(symbol, tickerKind);
    this._db.prepare('UPDATE symbol_demand SET in_demand = 1, ticker_kind=COALESCE(ticker_kind, ?) WHERE symbol=?').run(tickerKind, symbol);
    await this.queueIfNeeded(symbol);
  }

  /** Bump a symbol to watched tier (2) on page view. The periodic tier
   *  recompute decays it back to background after ~10 min of inactivity. */
  async bumpToWatched(symbol: string, tickerKind: TickerKind): Promise<void> {
    this.ensureDemandRow(symbol, tickerKind);
    const now = new Date().toISOString();
    // Only lower tier (raise priority) — never raise tier above current.
    this._db.prepare(
      "UPDATE symbol_demand SET tier = MIN(tier, 2), last_viewed_at = ?, in_demand = 1 WHERE symbol=?",
    ).run(now, symbol);
    await this.queueIfNeeded(symbol);
  }

  async pinSystemSymbol(symbol: string, tickerKind: TickerKind): Promise<void> {
    this.ensureDemandRow(symbol, tickerKind);
    this._db.prepare(
      'UPDATE symbol_demand SET in_demand = 1, system_pin = 1, ticker_kind=? WHERE symbol=?',
    ).run(tickerKind, symbol);
    await this.queueIfNeeded(symbol);
  }

  async unsubscribe(symbol: string): Promise<void> {
    const d = this._db;
    d.prepare('UPDATE symbol_demand SET refcount = MAX(refcount - 1, 0) WHERE symbol=?').run(symbol);
    // Keep system pins in demand even when no user owns them.
    d.prepare(
      'UPDATE symbol_demand SET in_demand = 0 WHERE symbol=? AND refcount = 0 AND COALESCE(system_pin, 0) = 0',
    ).run(symbol);
  }
  async demandSet(): Promise<string[]> {
    return (this._db.prepare(
      'SELECT symbol FROM symbol_demand WHERE in_demand = 1 OR COALESCE(system_pin, 0) = 1 ORDER BY symbol',
    ).all() as Array<{ symbol: string }>).map((r) => r.symbol);
  }
  async getMany<T>(keys: CacheKey[]): Promise<Array<{ key: CacheKey; value: T | null; isStale: boolean }>> {
    return Promise.all(keys.map(async (key) => {
      const e = await this.get<T>(key);
      return { key, value: e.value, isStale: e.isStale };
    }));
  }
  private clearKvCachePrefix(prefix: string): void {
    // Invalidate all LRU cache entries matching the prefix
    for (const key of kvReadCache.keys()) {
      if (key.startsWith(prefix)) {
        kvReadCache.delete(key);
      }
    }
  }
  async del(key: CacheKey): Promise<void> {
    const { source, kind, id } = parseCacheKey(key);
    const d = this._db;
    switch (kind) {
      case 'quote':
        d.prepare('DELETE FROM quotes WHERE symbol=?').run(id);
        break;
      case 'candles': {
        const [symbol, tf] = id.split(':');
        d.prepare('DELETE FROM price_candles WHERE symbol=? AND timeframe=?').run(symbol, tf);
        break;
      }
      case 'symbol':
        d.prepare('DELETE FROM symbols WHERE symbol=?').run(id);
        break;
      case 'adjustments':
        d.prepare('DELETE FROM price_adjustments WHERE symbol=?').run(id);
        break;
      case 'chain':
      case 'greeks': {
        const [symbol, expiry] = id.split(':');
        d.prepare('DELETE FROM options_chains WHERE symbol=? AND expiry=?').run(symbol, expiry);
        break;
      }
      case 'expiry_dates': {
        const cacheKey = `options_expiry:${id}`;
        d.prepare('DELETE FROM kv_cache WHERE key=?').run(cacheKey);
        kvReadCache.delete(cacheKey);
        break;
      }
      case 'shortinterest': {
        const cacheKey = `yfinance:shortinterest:${id}`;
        d.prepare('DELETE FROM kv_cache WHERE key=?').run(cacheKey);
        kvReadCache.delete(cacheKey);
        break;
      }
      case 'nasdaqShortinterest': {
        const cacheKey = `nasdaq:nasdaqShortinterest:${id}`;
        d.prepare('DELETE FROM kv_cache WHERE key=?').run(cacheKey);
        kvReadCache.delete(cacheKey);
        break;
      }
      case 'finraShortinterest':
        d.prepare('DELETE FROM finra_short_interest WHERE symbol=?').run(id);
        break;
      case 'finraSiBiweekly':
        d.prepare('DELETE FROM finra_short_interest_biweekly WHERE symbol=?').run(id);
        break;
      case 'dividendFundamentals':
        d.prepare('DELETE FROM dividend_fundamentals WHERE symbol=?').run(id);
        break;
      default: {
        // Unknown/wildcard kind (e.g. `x:cashtag:*`): best-effort delete from kv_cache via LIKE.
        const like = key.endsWith(':*') ? `${key.slice(0, -1)}%` : key;
        d.prepare('DELETE FROM kv_cache WHERE key LIKE ?').run(like);
        // Invalidate LRU cache for this prefix
        if (key.endsWith(':*')) {
          this.clearKvCachePrefix(key.slice(0, -1));
        }
      }
    }
    void source;
  }
}

export function createCacheRepository(opts: { db: DatabaseSync; scheduler: CacheScheduler }): CacheRepository {
  return new CacheRepositoryImpl(opts);
}

// Prod singleton — wired in slice 1f once AdapterQueue (the scheduler) exists.
let _cache: CacheRepository | null = null;
export function cacheRepository(scheduler: CacheScheduler): CacheRepository {
  if (!_cache) _cache = createCacheRepository({ db: defaultDb(), scheduler });
  return _cache;
}