feat: Phase 3 FINRA bulk adapter + Phase 4 three-way merge with discrepancy flagging
Phase 3 — FINRA bulk short-interest ingest: - Add finra_short_interest table to schema (per-symbol, per-settlement-date, per-exchange, with short/exempt/total volume, avg daily vol, days to cover) - Create FinraIngestService: downloads FINRA consolidated pipe-delimited file from configurable base URL, parses Market|Symbol|ShortVolume|ShortExemptVolume|TotalVolume, stores in finra_short_interest table - Create FinraBulkAdapter: SourceFetch that calls downloadAndIngestFinra, registers as finra-bulk source kind with finraShortinterest cache handler - finraShortinterest handler reads latest settlement row per symbol from finra_short_interest table (no per-symbol kv_cache write; data is bulk-ingested) - Register in index.ts adapter map + HANDLERS + del case Phase 4 — three-way merge with discrepancy detection: - shortInterest tRPC procedure now reads all 3 caches (yfinance, nasdaq, finra-bulk) in parallel - Reconciliation hierarchy: FINRA (shares short) > NASDAQ > Yahoo - daysToCover: NASDAQ (specific) > FINRA (computed) > Yahoo (short ratio fallback) - settlementDate: FINRA > NASDAQ > Yahoo - Discrepancy detection: compares sharesShort across available sources, flags >10% difference with discrepancyPct + discrepancyBetween - Updated ShortInterestPanel: FINRA source badge, discrepancy warning banner, three-source disclaimer - Updated trpc.ts client type for new shape
This commit is contained in:
@@ -0,0 +1,65 @@
|
|||||||
|
import type { DatabaseSync } from 'node:sqlite';
|
||||||
|
import type { CacheKey, SourceKind } from '../cache/CacheRepository.ts';
|
||||||
|
import { parseCacheKey } from '../cache/CacheRepository.ts';
|
||||||
|
import type { SourceFetch, FetchResult } from './SourceAdapter.ts';
|
||||||
|
import { downloadAndIngestFinra, latestFinraSettlementDate } from '../services/FinraIngestService.ts';
|
||||||
|
|
||||||
|
/** FINRA bulk short-interest adapter.
|
||||||
|
* Key format: finra-bulk:finraShortinterest:latest (auto-settle-date)
|
||||||
|
* finra-bulk:finraShortinterest:2026-07-10 (explicit settle date)
|
||||||
|
*
|
||||||
|
* On fetchOne: downloads the FINRA consolidated file, parses it, stores in
|
||||||
|
* finra_short_interest table. Per-symbol reads happen via CacheRepository handler
|
||||||
|
* reading from that table. */
|
||||||
|
export class FinraBulkAdapter implements SourceFetch {
|
||||||
|
readonly sourceKind: SourceKind = 'finra-bulk';
|
||||||
|
private _db: DatabaseSync;
|
||||||
|
|
||||||
|
constructor(db: DatabaseSync) {
|
||||||
|
this._db = db;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fetchOne(key: CacheKey): Promise<FetchResult> {
|
||||||
|
const { kind, id } = parseCacheKey(key);
|
||||||
|
if (kind !== 'finraShortinterest') throw new Error(`FinraBulkAdapter: unknown kind '${kind}'`);
|
||||||
|
|
||||||
|
const settlementDate = id === 'latest' ? this._guessSettlementDate() : id;
|
||||||
|
|
||||||
|
const result = await downloadAndIngestFinra(this._db, settlementDate);
|
||||||
|
return {
|
||||||
|
value: {
|
||||||
|
ok: true,
|
||||||
|
symbolsStored: result.symbolsStored,
|
||||||
|
sourceFile: result.sourceFile,
|
||||||
|
exchanges: result.exchanges,
|
||||||
|
settlementDate,
|
||||||
|
},
|
||||||
|
ttlClass: 'short_interest',
|
||||||
|
provenance: {
|
||||||
|
fetchedAt: new Date().toISOString(),
|
||||||
|
sourceKind: 'finra-bulk',
|
||||||
|
rawSourceId: settlementDate,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Guess the most recent FINRA settlement date (around 10th and 24th of each month). */
|
||||||
|
private _guessSettlementDate(): string {
|
||||||
|
const existing = latestFinraSettlementDate(this._db);
|
||||||
|
if (existing) return existing;
|
||||||
|
|
||||||
|
// If no data exists, guess the most recent past 10th/24th
|
||||||
|
const now = new Date();
|
||||||
|
const d = now.getDate();
|
||||||
|
const y = now.getFullYear();
|
||||||
|
const m = String(now.getMonth() + 1).padStart(2, '0');
|
||||||
|
|
||||||
|
if (d >= 25) return `${y}-${m}-24`;
|
||||||
|
if (d >= 11) return `${y}-${m}-10`;
|
||||||
|
// Previous month
|
||||||
|
const prev = new Date(now.getFullYear(), now.getMonth(), 0);
|
||||||
|
const pm = String(prev.getMonth() + 1).padStart(2, '0');
|
||||||
|
const pd = prev.getDate() >= 25 ? '24' : '10';
|
||||||
|
return `${prev.getFullYear()}-${pm}-${pd}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
+33
-1
@@ -6,7 +6,7 @@
|
|||||||
import { DatabaseSync } from 'node:sqlite';
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
import { db as defaultDb } from '../db/client.ts';
|
import { db as defaultDb } from '../db/client.ts';
|
||||||
|
|
||||||
export type SourceKind = 'yfinance' | 'nasdaq' | 'sec' | 'sec-fetch' | 'reddit' | 'x' | 'macro' | 'llm' | 'sec-lint-holders' | 'sec-lint-insiders' | 'fred';
|
export type SourceKind = 'yfinance' | 'nasdaq' | 'finra-bulk' | 'sec' | 'sec-fetch' | 'reddit' | 'x' | 'macro' | 'llm' | 'sec-lint-holders' | 'sec-lint-insiders' | 'fred';
|
||||||
export type TickerKind = 'equity' | 'crypto' | 'etf' | 'index';
|
export type TickerKind = 'equity' | 'crypto' | 'etf' | 'index';
|
||||||
export type CacheKey = string; // `${SourceKind}:${kind}:${id}` e.g. 'yfinance:quote:NVDA', 'yfinance:candles:NVDA:1d'
|
export type CacheKey = string; // `${SourceKind}:${kind}:${id}` e.g. 'yfinance:quote:NVDA', 'yfinance:candles:NVDA:1d'
|
||||||
export type TtlClass =
|
export type TtlClass =
|
||||||
@@ -315,6 +315,34 @@ const nasdaqShortinterestHandler: KindHandler = {
|
|||||||
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.short_interest; },
|
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.short_interest; },
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const finraShortInterestHandler: KindHandler = {
|
||||||
|
ttlClass: 'short_interest',
|
||||||
|
read(d, symbol) {
|
||||||
|
// Return the most recent settlement date's data for a given symbol
|
||||||
|
const r = d.prepare(
|
||||||
|
`SELECT short_volume, short_exempt, total_volume, avg_daily_vol, days_to_cover, settlement_date, exchange, ingested_at
|
||||||
|
FROM finra_short_interest WHERE symbol=? ORDER BY settlement_date DESC LIMIT 1`
|
||||||
|
).get(symbol) as Record<string, unknown> | undefined;
|
||||||
|
if (!r) return null;
|
||||||
|
return {
|
||||||
|
value: {
|
||||||
|
shortVolume: r.short_volume as number,
|
||||||
|
shortExempt: r.short_exempt as number,
|
||||||
|
totalVolume: r.total_volume as number,
|
||||||
|
avgDailyVol: r.avg_daily_vol as number | null,
|
||||||
|
daysToCover: r.days_to_cover as number | null,
|
||||||
|
settlementDate: r.settlement_date as string,
|
||||||
|
exchange: r.exchange as string | null,
|
||||||
|
},
|
||||||
|
stalenessTs: r.ingested_at as string,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
write(_d, _symbol, _value, _provenance) {
|
||||||
|
// FINRA data is bulk-ingested via FinraIngestService; no per-symbol write needed.
|
||||||
|
},
|
||||||
|
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.short_interest; },
|
||||||
|
};
|
||||||
|
|
||||||
const HANDLERS = new Map<string, KindHandler>([
|
const HANDLERS = new Map<string, KindHandler>([
|
||||||
['quote', quoteHandler],
|
['quote', quoteHandler],
|
||||||
['candles', candlesHandler],
|
['candles', candlesHandler],
|
||||||
@@ -328,6 +356,7 @@ const HANDLERS = new Map<string, KindHandler>([
|
|||||||
['insiders', lintInsidersHandler],
|
['insiders', lintInsidersHandler],
|
||||||
['shortinterest', shortInterestHandler],
|
['shortinterest', shortInterestHandler],
|
||||||
['nasdaqShortinterest', nasdaqShortinterestHandler],
|
['nasdaqShortinterest', nasdaqShortinterestHandler],
|
||||||
|
['finraShortinterest', finraShortInterestHandler],
|
||||||
]);
|
]);
|
||||||
|
|
||||||
export interface CacheRepository {
|
export interface CacheRepository {
|
||||||
@@ -441,6 +470,9 @@ export class CacheRepositoryImpl implements CacheRepository {
|
|||||||
case 'nasdaqShortinterest':
|
case 'nasdaqShortinterest':
|
||||||
d.prepare('DELETE FROM kv_cache WHERE key=?').run(`nasdaq:nasdaqShortinterest:${id}`);
|
d.prepare('DELETE FROM kv_cache WHERE key=?').run(`nasdaq:nasdaqShortinterest:${id}`);
|
||||||
break;
|
break;
|
||||||
|
case 'finraShortinterest':
|
||||||
|
d.prepare('DELETE FROM finra_short_interest WHERE symbol=?').run(id);
|
||||||
|
break;
|
||||||
default: {
|
default: {
|
||||||
// Unknown/wildcard kind (e.g. `x:cashtag:*`): best-effort delete from kv_cache via LIKE.
|
// Unknown/wildcard kind (e.g. `x:cashtag:*`): best-effort delete from kv_cache via LIKE.
|
||||||
const like = key.endsWith(':*') ? `${key.slice(0, -1)}%` : key;
|
const like = key.endsWith(':*') ? `${key.slice(0, -1)}%` : key;
|
||||||
|
|||||||
@@ -676,3 +676,21 @@ CREATE TABLE IF NOT EXISTS rotation_state (
|
|||||||
sub_group TEXT, -- the strongest sub-group (e.g. "Financials")
|
sub_group TEXT, -- the strongest sub-group (e.g. "Financials")
|
||||||
checked_at TEXT
|
checked_at TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- ===== Slice 22 — FINRA Short Interest (bulk ingest, twice-monthly) =====
|
||||||
|
CREATE TABLE IF NOT EXISTS finra_short_interest (
|
||||||
|
symbol TEXT NOT NULL,
|
||||||
|
settlement_date TEXT NOT NULL, -- YYYY-MM-DD of the short interest report
|
||||||
|
exchange TEXT, -- NASDAQ | NYSE | NYSE_AMERICAN | NYSE_ARCA | ALL
|
||||||
|
short_volume REAL, -- shares sold short
|
||||||
|
short_exempt REAL, -- short exempt volume
|
||||||
|
total_volume REAL, -- short + exempt
|
||||||
|
avg_daily_vol REAL, -- average daily volume (for days-to-cover)
|
||||||
|
days_to_cover REAL, -- computed: short_volume / avg_daily_vol
|
||||||
|
source_file TEXT, -- origin filename for audit
|
||||||
|
ingested_at TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (symbol, settlement_date, exchange)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_finra_symbol ON finra_short_interest(symbol);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_finra_settlement ON finra_short_interest(settlement_date);
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import { db } from './db/client.ts';
|
|||||||
import { createCacheRepository, type SourceKind } from './cache/CacheRepository.ts';
|
import { createCacheRepository, type SourceKind } from './cache/CacheRepository.ts';
|
||||||
import { YFinanceAdapter } from './adapters/YFinanceAdapter.ts';
|
import { YFinanceAdapter } from './adapters/YFinanceAdapter.ts';
|
||||||
import { NasdaqAdapter } from './adapters/NasdaqAdapter.ts';
|
import { NasdaqAdapter } from './adapters/NasdaqAdapter.ts';
|
||||||
|
import { FinraBulkAdapter } from './adapters/FinraBulkAdapter.ts';
|
||||||
import { SecFetchAdapter } from './adapters/SecFetchAdapter.ts';
|
import { SecFetchAdapter } from './adapters/SecFetchAdapter.ts';
|
||||||
import { SecLintAdapter } from './adapters/SecLintAdapter.ts';
|
import { SecLintAdapter } from './adapters/SecLintAdapter.ts';
|
||||||
import { XCookieAdapter } from './adapters/XCookieAdapter.ts';
|
import { XCookieAdapter } from './adapters/XCookieAdapter.ts';
|
||||||
@@ -20,6 +21,7 @@ const database = db();
|
|||||||
const adapters = new Map<SourceKind, SourceFetch>([
|
const adapters = new Map<SourceKind, SourceFetch>([
|
||||||
['yfinance' as const, new YFinanceAdapter() as unknown as SourceFetch],
|
['yfinance' as const, new YFinanceAdapter() as unknown as SourceFetch],
|
||||||
['nasdaq' as const, new NasdaqAdapter() as unknown as SourceFetch],
|
['nasdaq' as const, new NasdaqAdapter() as unknown as SourceFetch],
|
||||||
|
['finra-bulk' as const, new FinraBulkAdapter(database) as unknown as SourceFetch],
|
||||||
['sec-fetch' as const, new SecFetchAdapter(database) as unknown as SourceFetch],
|
['sec-fetch' as const, new SecFetchAdapter(database) as unknown as SourceFetch],
|
||||||
['sec-lint-holders' as const, new SecLintAdapter(() => database, 'sec-lint-holders') as unknown as SourceFetch],
|
['sec-lint-holders' as const, new SecLintAdapter(() => database, 'sec-lint-holders') as unknown as SourceFetch],
|
||||||
['sec-lint-insiders' as const, new SecLintAdapter(() => database, 'sec-lint-insiders') as unknown as SourceFetch],
|
['sec-lint-insiders' as const, new SecLintAdapter(() => database, 'sec-lint-insiders') as unknown as SourceFetch],
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
import type { DatabaseSync } from 'node:sqlite';
|
||||||
|
|
||||||
|
const FINRA_BASE_URL = process.env.FINRA_BASE_URL ?? 'https://www.finra.org/sites/default/files';
|
||||||
|
|
||||||
|
/** Format a FINRA consolidated-short-interest filename: CAshvol{YYYYMMDD}.txt */
|
||||||
|
function finraFilename(settlementDate: string): string {
|
||||||
|
const d = settlementDate.replace(/-/g, '');
|
||||||
|
const ym = settlementDate.slice(0, 7).replace(/-/, '-');
|
||||||
|
return `${ym}/CAshvol${d}.txt`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parse a FINRA consolidated-short-interest file body (pipe-delimited).
|
||||||
|
* Expected columns: Market|Symbol|ShortVolume|ShortExemptVolume|TotalVolume
|
||||||
|
* Returns per-symbol rows aggregated across all exchanges. */
|
||||||
|
function parseFinraFile(
|
||||||
|
body: string,
|
||||||
|
settlementDate: string,
|
||||||
|
ingestedAt: string,
|
||||||
|
sourceFile: string,
|
||||||
|
): Array<{
|
||||||
|
symbol: string;
|
||||||
|
exchange: string;
|
||||||
|
shortVolume: number;
|
||||||
|
shortExempt: number;
|
||||||
|
totalVolume: number;
|
||||||
|
}> {
|
||||||
|
const lines = body.split(/\r?\n/);
|
||||||
|
const rows: Array<{
|
||||||
|
symbol: string;
|
||||||
|
exchange: string;
|
||||||
|
shortVolume: number;
|
||||||
|
shortExempt: number;
|
||||||
|
totalVolume: number;
|
||||||
|
}> = [];
|
||||||
|
let headerFound = false;
|
||||||
|
|
||||||
|
for (const raw of lines) {
|
||||||
|
const line = raw.trim();
|
||||||
|
if (!line || line.startsWith('#')) continue;
|
||||||
|
if (line.startsWith('Date Range') || line.startsWith('Period')) continue;
|
||||||
|
if (line.includes('Market|Symbol|')) { headerFound = true; continue; }
|
||||||
|
if (!headerFound) continue;
|
||||||
|
|
||||||
|
const cols = line.split('|').map((c) => c.trim());
|
||||||
|
if (cols.length < 4) continue;
|
||||||
|
|
||||||
|
const market = cols[0];
|
||||||
|
const symbol = cols[1];
|
||||||
|
const shortVolume = parseFloat(cols[2]?.replace(/,/g, ''));
|
||||||
|
const shortExempt = cols[3] ? parseFloat(cols[3].replace(/,/g, '')) : 0;
|
||||||
|
const totalVolume = cols[4] ? parseFloat(cols[4].replace(/,/g, '')) : shortVolume + shortExempt;
|
||||||
|
|
||||||
|
if (!symbol || Number.isNaN(shortVolume)) continue;
|
||||||
|
|
||||||
|
rows.push({
|
||||||
|
symbol: symbol.toUpperCase(),
|
||||||
|
exchange: market.toUpperCase(),
|
||||||
|
shortVolume,
|
||||||
|
shortExempt,
|
||||||
|
totalVolume,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Download and ingest a FINRA consolidated-short-interest file.
|
||||||
|
* Returns count of symbols stored. */
|
||||||
|
export async function downloadAndIngestFinra(
|
||||||
|
db: DatabaseSync,
|
||||||
|
settlementDate: string,
|
||||||
|
): Promise<{ symbolsStored: number; sourceFile: string; exchanges: string[] }> {
|
||||||
|
const filename = finraFilename(settlementDate);
|
||||||
|
const url = `${FINRA_BASE_URL}/${filename}`;
|
||||||
|
const ingestedAt = new Date().toISOString();
|
||||||
|
|
||||||
|
console.log(`[finra] downloading ${url}`);
|
||||||
|
const resp = await fetch(url, {
|
||||||
|
headers: { 'User-Agent': 'InvestorFlow/1.0 (research) node' },
|
||||||
|
signal: AbortSignal.timeout(30_000),
|
||||||
|
});
|
||||||
|
if (!resp.ok) throw new Error(`FINRA download failed: ${resp.status} ${resp.statusText}`);
|
||||||
|
const body = await resp.text();
|
||||||
|
if (!body.trim()) throw new Error('FINRA file is empty');
|
||||||
|
|
||||||
|
const rows = parseFinraFile(body, settlementDate, ingestedAt, filename);
|
||||||
|
if (!rows.length) throw new Error('No FINRA short interest rows parsed');
|
||||||
|
|
||||||
|
const exchanges = [...new Set(rows.map((r) => r.exchange))];
|
||||||
|
const exchangeMap: Record<string, string> = {};
|
||||||
|
exchanges.forEach((e) => { exchangeMap[e] = e; });
|
||||||
|
|
||||||
|
const upsert = db.prepare(
|
||||||
|
`INSERT OR REPLACE INTO finra_short_interest
|
||||||
|
(symbol, settlement_date, exchange, short_volume, short_exempt, total_volume, avg_daily_vol, days_to_cover, source_file, ingested_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)`
|
||||||
|
);
|
||||||
|
|
||||||
|
const tx = db.transaction(() => {
|
||||||
|
for (const r of rows) {
|
||||||
|
upsert.run(
|
||||||
|
r.symbol,
|
||||||
|
settlementDate,
|
||||||
|
r.exchange,
|
||||||
|
r.shortVolume,
|
||||||
|
r.shortExempt,
|
||||||
|
r.totalVolume,
|
||||||
|
filename,
|
||||||
|
ingestedAt,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
tx();
|
||||||
|
|
||||||
|
console.log(`[finra] ingested ${rows.length} symbols from ${filename} (exchanges: ${exchanges.join(', ')})`);
|
||||||
|
return { symbolsStored: rows.length, sourceFile: filename, exchanges };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Compute days-to-cover for finra_short_interest rows that have avg_daily_vol set.
|
||||||
|
* Called after avg_daily_vol is populated from external volume data. */
|
||||||
|
export function computeDaysToCover(db: DatabaseSync): number {
|
||||||
|
const r = db.exec(
|
||||||
|
`UPDATE finra_short_interest
|
||||||
|
SET days_to_cover = CASE
|
||||||
|
WHEN avg_daily_vol IS NOT NULL AND avg_daily_vol > 0 THEN short_volume / avg_daily_vol
|
||||||
|
ELSE NULL
|
||||||
|
END
|
||||||
|
WHERE days_to_cover IS NULL AND avg_daily_vol IS NOT NULL`
|
||||||
|
);
|
||||||
|
return r.changes;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get the latest settlement date available in the finra_short_interest table. */
|
||||||
|
export function latestFinraSettlementDate(db: DatabaseSync): string | null {
|
||||||
|
const r = db.prepare(
|
||||||
|
'SELECT settlement_date FROM finra_short_interest ORDER BY settlement_date DESC LIMIT 1'
|
||||||
|
).get() as { settlement_date: string } | undefined;
|
||||||
|
return r?.settlement_date ?? null;
|
||||||
|
}
|
||||||
@@ -1900,27 +1900,54 @@ const institutionalRouter = router({
|
|||||||
return { symbol, ratings: result.ratings, consensus: result.consensus };
|
return { symbol, ratings: result.ratings, consensus: result.consensus };
|
||||||
}),
|
}),
|
||||||
|
|
||||||
/** Short interest reconciled across Yahoo Finance + NASDAQ API.
|
/** Short interest reconciled across Yahoo Finance + NASDAQ API + FINRA.
|
||||||
* Yahoo covers all US-listed tickers (basic: shares short, % float, short ratio).
|
* Yahoo covers all US-listed tickers (basic: shares short, % float, short ratio).
|
||||||
* NASDAQ covers NASDAQ-listed only but adds days-to-cover + 24mo history.
|
* NASDAQ covers NASDAQ-listed only but adds days-to-cover + 24mo history.
|
||||||
* Reconciliation: prefer NASDAQ's sharesShort/daysToCover when available (richer),
|
* FINRA covers all exchanges via consolidated twice-monthly files.
|
||||||
* fall back to Yahoo for NYSE/other tickers. */
|
* Reconciliation hierarchy (most → least authoritative for each field):
|
||||||
|
* sharesShort: FINRA > NASDAQ > Yahoo
|
||||||
|
* daysToCover: NASDAQ (specific) > FINRA (computed) > Yahoo (ratio)
|
||||||
|
* % of Float: Yahoo only (has floatShares)
|
||||||
|
* Discrepancy detection: flags >10% difference between top-2 available sources. */
|
||||||
shortInterest: publicProcedure
|
shortInterest: publicProcedure
|
||||||
.input(z.object({ symbol: z.string().min(1) }))
|
.input(z.object({ symbol: z.string().min(1) }))
|
||||||
.query(async ({ ctx, input }) => {
|
.query(async ({ ctx, input }) => {
|
||||||
const symbol = input.symbol.toUpperCase();
|
const symbol = input.symbol.toUpperCase();
|
||||||
const [yfEntry, ndqEntry] = await Promise.all([
|
const [yfEntry, ndqEntry, frEntry] = await Promise.all([
|
||||||
ctx.cache.get<Record<string, unknown>>(`yfinance:shortinterest:${symbol}`),
|
ctx.cache.get<Record<string, unknown>>(`yfinance:shortinterest:${symbol}`),
|
||||||
ctx.cache.get<Record<string, unknown>>(`nasdaq:nasdaqShortinterest:${symbol}`),
|
ctx.cache.get<Record<string, unknown>>(`nasdaq:nasdaqShortinterest:${symbol}`),
|
||||||
|
ctx.cache.get<Record<string, unknown>>(`finra-bulk:finraShortinterest:${symbol}`),
|
||||||
]);
|
]);
|
||||||
const yf = yfEntry.value;
|
const yf = yfEntry.value;
|
||||||
const ndq = ndqEntry.value;
|
const ndq = ndqEntry.value;
|
||||||
|
const fr = frEntry.value;
|
||||||
|
|
||||||
// Prefer NASDAQ's sharesShort + daysToCover when available (NASDAQ-listed);
|
// Collect sharesShort from all sources for discrepancy detection
|
||||||
// otherwise fall back to Yahoo (covers NYSE/other tickers).
|
const sharesSources: Array<{ source: string; sharesShort: number | null }> = [
|
||||||
const sharesShort = ndq?.sharesShort ?? yf?.sharesShort ?? null;
|
{ source: 'finra', sharesShort: (fr?.shortVolume as number | undefined) ?? null },
|
||||||
const daysToCover = ndq?.daysToCover ?? null;
|
{ source: 'nasdaq', sharesShort: ndq?.sharesShort as number | null },
|
||||||
const settlementDate = ndq?.settlementDate ?? null;
|
{ source: 'yfinance', sharesShort: yf?.sharesShort as number | null },
|
||||||
|
];
|
||||||
|
const availableSources = sharesSources.filter((s) => s.sharesShort !== null);
|
||||||
|
|
||||||
|
let discrepancyPct: number | null = null;
|
||||||
|
let discrepancyBetween: string | null = null;
|
||||||
|
if (availableSources.length >= 2) {
|
||||||
|
const sorted = [...availableSources].sort((a, b) => (b.sharesShort ?? 0) - (a.sharesShort ?? 0));
|
||||||
|
const hi = sorted[0].sharesShort!;
|
||||||
|
const lo = sorted[sorted.length - 1].sharesShort!;
|
||||||
|
if (lo > 0) {
|
||||||
|
const pct = Math.abs(hi - lo) / lo;
|
||||||
|
if (pct > 0.10) {
|
||||||
|
discrepancyPct = Math.round(pct * 100);
|
||||||
|
discrepancyBetween = `${sorted[0].source} vs ${sorted[sorted.length - 1].source}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const sharesShort = fr?.shortVolume ?? ndq?.sharesShort ?? yf?.sharesShort ?? null;
|
||||||
|
const daysToCover = ndq?.daysToCover ?? fr?.daysToCover ?? yf?.shortRatio ?? null;
|
||||||
|
const settlementDate = fr?.settlementDate ?? ndq?.settlementDate ?? yf?.dateShortInterest ?? null;
|
||||||
const history = ndq?.history ?? null;
|
const history = ndq?.history ?? null;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -1936,11 +1963,14 @@ const institutionalRouter = router({
|
|||||||
daysToCover,
|
daysToCover,
|
||||||
settlementDate,
|
settlementDate,
|
||||||
history,
|
history,
|
||||||
|
discrepancyPct,
|
||||||
|
discrepancyBetween,
|
||||||
sources: {
|
sources: {
|
||||||
yfinance: { available: !!yf, stale: yfEntry.isStale },
|
yfinance: { available: !!yf, stale: yfEntry.isStale },
|
||||||
nasdaq: { available: !!ndq, stale: ndqEntry.isStale },
|
nasdaq: { available: !!ndq, stale: ndqEntry.isStale },
|
||||||
|
finra: { available: !!fr, stale: frEntry.isStale },
|
||||||
},
|
},
|
||||||
stale: yfEntry.isStale && ndqEntry.isStale,
|
stale: yfEntry.isStale && ndqEntry.isStale && frEntry.isStale,
|
||||||
};
|
};
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
'use client';
|
'use client';
|
||||||
import { useState, useEffect, useMemo } from 'react';
|
import { useState, useEffect } from 'react';
|
||||||
import { api } from '@/lib/trpc';
|
import { api } from '@/lib/trpc';
|
||||||
|
|
||||||
interface HistoryEntry {
|
interface HistoryEntry {
|
||||||
@@ -22,7 +22,9 @@ interface ShortInterestData {
|
|||||||
daysToCover: number | null;
|
daysToCover: number | null;
|
||||||
settlementDate: string | null;
|
settlementDate: string | null;
|
||||||
history: HistoryEntry[] | null;
|
history: HistoryEntry[] | null;
|
||||||
sources: { yfinance: { available: boolean; stale: boolean }; nasdaq: { available: boolean; stale: boolean } };
|
discrepancyPct: number | null;
|
||||||
|
discrepancyBetween: string | null;
|
||||||
|
sources: { yfinance: { available: boolean; stale: boolean }; nasdaq: { available: boolean; stale: boolean }; finra: { available: boolean; stale: boolean } };
|
||||||
stale: boolean;
|
stale: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -80,6 +82,7 @@ export function ShortInterestPanel({ symbol }: { symbol: string | null }) {
|
|||||||
|
|
||||||
const sourceTags = [] as string[];
|
const sourceTags = [] as string[];
|
||||||
if (data.sources.nasdaq.available) sourceTags.push('NASDAQ');
|
if (data.sources.nasdaq.available) sourceTags.push('NASDAQ');
|
||||||
|
if (data.sources.finra.available) sourceTags.push('FINRA');
|
||||||
if (data.sources.yfinance.available) sourceTags.push('Yahoo');
|
if (data.sources.yfinance.available) sourceTags.push('Yahoo');
|
||||||
const sourceLabel = sourceTags.length ? sourceTags.join(' + ') : '—';
|
const sourceLabel = sourceTags.length ? sourceTags.join(' + ') : '—';
|
||||||
|
|
||||||
@@ -91,6 +94,11 @@ export function ShortInterestPanel({ symbol }: { symbol: string | null }) {
|
|||||||
{dateLabel && (
|
{dateLabel && (
|
||||||
<p className="text-[10px] text-fg-muted">{dateLabel}{data.stale ? ' — stale' : ''} ({sourceLabel})</p>
|
<p className="text-[10px] text-fg-muted">{dateLabel}{data.stale ? ' — stale' : ''} ({sourceLabel})</p>
|
||||||
)}
|
)}
|
||||||
|
{data.discrepancyPct !== null && (
|
||||||
|
<div className="bg-warning/10 border border-warning/30 rounded px-2 py-1 text-[10px] text-warning">
|
||||||
|
{data.discrepancyPct}% discrepancy in shares short between sources ({data.discrepancyBetween})
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
{data.sharesShort !== null && (
|
{data.sharesShort !== null && (
|
||||||
<dl className="grid grid-cols-2 gap-x-6 gap-y-1 text-sm">
|
<dl className="grid grid-cols-2 gap-x-6 gap-y-1 text-sm">
|
||||||
<div className="flex justify-between"><dt className="text-fg-muted">Shares Short</dt><dd className="text-fg font-mono">{formatNumber(data.sharesShort)}</dd></div>
|
<div className="flex justify-between"><dt className="text-fg-muted">Shares Short</dt><dd className="text-fg font-mono">{formatNumber(data.sharesShort)}</dd></div>
|
||||||
@@ -130,7 +138,7 @@ export function ShortInterestPanel({ symbol }: { symbol: string | null }) {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
<p className="text-[10px] text-fg-muted mt-1">
|
<p className="text-[10px] text-fg-muted mt-1">
|
||||||
Reconciled from NASDAQ (days-to-cover, history) + Yahoo Finance (all tickers). Educational display only.
|
Reconciled across FINRA (regulatory, shares short), NASDAQ (days-to-cover, history), and Yahoo Finance (% float, universal coverage). Educational display only.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
+1
-1
@@ -623,7 +623,7 @@ export const api = {
|
|||||||
analystRatings: (symbol: string) =>
|
analystRatings: (symbol: string) =>
|
||||||
trpcQuery<{ symbol: string; ratings: Array<{ firm: string; action: string | null; gradeFrom: string | null; gradeTo: string | null; ratingDate: string; targetFrom: number | null; targetTo: number | null }>; consensus: { strongBuy: number; buy: number; hold: number; sell: number; strongSell: number } | null; error?: string }>("institutional.analystRatings", { symbol }),
|
trpcQuery<{ symbol: string; ratings: Array<{ firm: string; action: string | null; gradeFrom: string | null; gradeTo: string | null; ratingDate: string; targetFrom: number | null; targetTo: number | null }>; consensus: { strongBuy: number; buy: number; hold: number; sell: number; strongSell: number } | null; error?: string }>("institutional.analystRatings", { symbol }),
|
||||||
shortInterest: (symbol: string) =>
|
shortInterest: (symbol: string) =>
|
||||||
trpcQuery<{ symbol: string; sharesShort: number | null; sharesShortPriorMonth: number | null; sharesPercentSharesOut: number | null; shortRatio: number | null; shortPercentOfFloat: number | null; dateShortInterest: string | null; floatShares: number | null; sharesOutstanding: number | null; daysToCover: number | null; settlementDate: string | null; history: Array<{ settlementDate: string; sharesShort: number | null; avgDailyVolume: number | null; daysToCover: number }> | null; sources: { yfinance: { available: boolean; stale: boolean }; nasdaq: { available: boolean; stale: boolean } }; stale: boolean }>("institutional.shortInterest", { symbol }),
|
trpcQuery<{ symbol: string; sharesShort: number | null; sharesShortPriorMonth: number | null; sharesPercentSharesOut: number | null; shortRatio: number | null; shortPercentOfFloat: number | null; dateShortInterest: string | null; floatShares: number | null; sharesOutstanding: number | null; daysToCover: number | null; settlementDate: string | null; history: Array<{ settlementDate: string; sharesShort: number | null; avgDailyVolume: number | null; daysToCover: number }> | null; discrepancyPct: number | null; discrepancyBetween: string | null; sources: { yfinance: { available: boolean; stale: boolean }; nasdaq: { available: boolean; stale: boolean }; finra: { available: boolean; stale: boolean } }; stale: boolean }>("institutional.shortInterest", { symbol }),
|
||||||
},
|
},
|
||||||
emotionLogger: {
|
emotionLogger: {
|
||||||
add: (input: { tradeExecutionId: string; priceAtEvent?: number; emotion: string; note?: string }) =>
|
add: (input: { tradeExecutionId: string; priceAtEvent?: number; emotion: string; note?: string }) =>
|
||||||
|
|||||||
Reference in New Issue
Block a user