feat: Phase 3 FINRA bulk adapter + Phase 4 three-way merge with discrepancy flagging

Phase 3 — FINRA bulk short-interest ingest:
- Add finra_short_interest table to schema (per-symbol, per-settlement-date, per-exchange, with short/exempt/total volume, avg daily vol, days to cover)
- Create FinraIngestService: downloads FINRA consolidated pipe-delimited file from configurable base URL, parses Market|Symbol|ShortVolume|ShortExemptVolume|TotalVolume, stores in finra_short_interest table
- Create FinraBulkAdapter: SourceFetch that calls downloadAndIngestFinra, registers as finra-bulk source kind with finraShortinterest cache handler
- finraShortinterest handler reads latest settlement row per symbol from finra_short_interest table (no per-symbol kv_cache write; data is bulk-ingested)
- Register in index.ts adapter map + HANDLERS + del case

Phase 4 — three-way merge with discrepancy detection:
- shortInterest tRPC procedure now reads all 3 caches (yfinance, nasdaq, finra-bulk) in parallel
- Reconciliation hierarchy: FINRA (shares short) > NASDAQ > Yahoo
- daysToCover: NASDAQ (specific) > FINRA (computed) > Yahoo (short ratio fallback)
- settlementDate: FINRA > NASDAQ > Yahoo
- Discrepancy detection: compares sharesShort across available sources, flags >10% difference with discrepancyPct + discrepancyBetween
- Updated ShortInterestPanel: FINRA source badge, discrepancy warning banner, three-source disclaimer
- Updated trpc.ts client type for new shape
This commit is contained in:
Investor Flow Build
2026-07-25 13:03:49 -04:00
parent 76f60dc766
commit 3fa6b32916
8 changed files with 308 additions and 15 deletions
+33 -1
View File
@@ -6,7 +6,7 @@
import { DatabaseSync } from 'node:sqlite';
import { db as defaultDb } from '../db/client.ts';
export type SourceKind = 'yfinance' | 'nasdaq' | 'sec' | 'sec-fetch' | 'reddit' | 'x' | 'macro' | 'llm' | 'sec-lint-holders' | 'sec-lint-insiders' | 'fred';
export type SourceKind = 'yfinance' | 'nasdaq' | 'finra-bulk' | 'sec' | 'sec-fetch' | 'reddit' | 'x' | 'macro' | 'llm' | 'sec-lint-holders' | 'sec-lint-insiders' | 'fred';
export type TickerKind = 'equity' | 'crypto' | 'etf' | 'index';
export type CacheKey = string; // `${SourceKind}:${kind}:${id}` e.g. 'yfinance:quote:NVDA', 'yfinance:candles:NVDA:1d'
export type TtlClass =
@@ -315,6 +315,34 @@ const nasdaqShortinterestHandler: KindHandler = {
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.short_interest; },
};
const finraShortInterestHandler: KindHandler = {
ttlClass: 'short_interest',
read(d, symbol) {
// Return the most recent settlement date's data for a given symbol
const r = d.prepare(
`SELECT short_volume, short_exempt, total_volume, avg_daily_vol, days_to_cover, settlement_date, exchange, ingested_at
FROM finra_short_interest WHERE symbol=? ORDER BY settlement_date DESC LIMIT 1`
).get(symbol) as Record<string, unknown> | undefined;
if (!r) return null;
return {
value: {
shortVolume: r.short_volume as number,
shortExempt: r.short_exempt as number,
totalVolume: r.total_volume as number,
avgDailyVol: r.avg_daily_vol as number | null,
daysToCover: r.days_to_cover as number | null,
settlementDate: r.settlement_date as string,
exchange: r.exchange as string | null,
},
stalenessTs: r.ingested_at as string,
};
},
write(_d, _symbol, _value, _provenance) {
// FINRA data is bulk-ingested via FinraIngestService; no per-symbol write needed.
},
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.short_interest; },
};
const HANDLERS = new Map<string, KindHandler>([
['quote', quoteHandler],
['candles', candlesHandler],
@@ -328,6 +356,7 @@ const HANDLERS = new Map<string, KindHandler>([
['insiders', lintInsidersHandler],
['shortinterest', shortInterestHandler],
['nasdaqShortinterest', nasdaqShortinterestHandler],
['finraShortinterest', finraShortInterestHandler],
]);
export interface CacheRepository {
@@ -441,6 +470,9 @@ export class CacheRepositoryImpl implements CacheRepository {
case 'nasdaqShortinterest':
d.prepare('DELETE FROM kv_cache WHERE key=?').run(`nasdaq:nasdaqShortinterest:${id}`);
break;
case 'finraShortinterest':
d.prepare('DELETE FROM finra_short_interest WHERE symbol=?').run(id);
break;
default: {
// Unknown/wildcard kind (e.g. `x:cashtag:*`): best-effort delete from kv_cache via LIKE.
const like = key.endsWith(':*') ? `${key.slice(0, -1)}%` : key;