feat(sec-lint): lint+backfill system for SEC data gaps (B1-B4)

- SecLintAdapter implements SourceFetch, runs via shared queue/drain loop
- Two new SourceKinds: sec-lint-holders, sec-lint-insiders (weekly schedules)
- No-op cache handlers so drain->cache.set doesn't throw on lint keys
- tRPC admin.queueLint(symbol, kind) — run lint for one symbol, returns LintResult
- tRPC admin.queueLintAll(kind) — backfill ALL watched symbols at once
- tRPC admin.dataQualityList() — query data_quality rows (filterable by symbol/kind)
- InstitutionalDashboard: 'Lint holders' button + status badge in detail panel header
- Admin queue page: Data Quality section with per-row status badges, 'Lint all' buttons
- DEFAULT_RATE_MS includes 167ms (~6 req/s) for lint kinds matching EDGAR limiter
This commit is contained in:
Investor Flow Build
2026-07-12 20:19:12 -04:00
parent ca385c1960
commit 5ef2b2f060
9 changed files with 1154 additions and 47 deletions
+90 -4
View File
@@ -6,7 +6,7 @@
import { DatabaseSync } from 'node:sqlite';
import { db as defaultDb } from '../db/client.ts';
export type SourceKind = 'yfinance' | 'sec' | 'reddit' | 'x' | 'macro' | 'llm';
export type SourceKind = 'yfinance' | 'sec' | 'sec-fetch' | 'reddit' | 'x' | 'macro' | 'llm' | 'sec-lint-holders' | 'sec-lint-insiders';
export type TickerKind = 'equity' | 'crypto' | 'etf' | 'index';
export type CacheKey = string; // `${SourceKind}:${kind}:${id}` e.g. 'yfinance:quote:NVDA', 'yfinance:candles:NVDA:1d'
export type TtlClass =
@@ -139,6 +139,7 @@ const symbolHandler: KindHandler = {
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.symbol_meta; },
};
// ----- Options handlers (slice 15) -----
const optionsChainHandler: KindHandler = {
ttlClass: 'options_snapshot',
read(d, id) {
@@ -177,9 +178,9 @@ const optionsChainHandler: KindHandler = {
const right = r.right === 'put' ? 'put' : 'call';
ins.run(
symbol, expiry, strike, right,
r.bid ?? null, r.ask ?? null, r.impliedVolatility ?? null,
r.delta ?? null, r.gamma ?? null, r.theta ?? null, r.vega ?? null,
r.openInterest ?? null, r.volume ?? null,
numOrNull(r.bid), numOrNull(r.ask), numOrNull(r.impliedVolatility),
numOrNull(r.delta), numOrNull(r.gamma), numOrNull(r.theta), numOrNull(r.vega),
numOrNull(r.openInterest), numOrNull(r.volume),
provenance.fetchedAt
);
}
@@ -205,6 +206,87 @@ const optionsExpiryDatesHandler: KindHandler = {
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.intraday; },
};
/** Coerce a possibly-undefined/unknown value to number | null for SQL binding. */
function numOrNull(v: unknown): number | null {
return typeof v === 'number' ? v : null;
}
const greeksHandler: KindHandler = {
ttlClass: 'options_snapshot',
read(d, id) {
const { symbol, expiry, strike } = parseGreeksId(id);
const r = d.prepare(
'SELECT delta, gamma, theta, vega, strike, open_interest, iv, ts, type FROM options_chains WHERE symbol=? AND expiry=? AND strike=? LIMIT 1'
).get(symbol, expiry, parseFloat(strike ?? '0')) as Record<string, unknown> | undefined;
if (!r) return null;
return {
value: {
delta: numOrNull(r.delta),
gamma: numOrNull(r.gamma),
theta: numOrNull(r.theta),
vega: numOrNull(r.vega),
strike: typeof r.strike === 'number' ? r.strike : 0,
openInterest: numOrNull(r.open_interest),
impliedVolatility: numOrNull(r.iv),
right: r.type as 'call' | 'put',
},
stalenessTs: r.ts as string,
};
},
write(d, id, value, provenance) {
const { symbol, expiry, strike } = parseGreeksId(id);
const v = value as Record<string, unknown>;
const right = v.right === 'put' ? 'put' : 'call';
d.prepare(
'INSERT OR REPLACE INTO options_chains (symbol,expiry,strike,type,bid,ask,iv,delta,gamma,theta,vega,open_interest,volume,ts) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
).run(
symbol, expiry, parseFloat(strike ?? '0'), right,
numOrNull(v.bid), numOrNull(v.ask), numOrNull(v.impliedVolatility),
numOrNull(v.delta), numOrNull(v.gamma), numOrNull(v.theta), numOrNull(v.vega),
numOrNull(v.openInterest), numOrNull(v.volume),
provenance.fetchedAt
);
},
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.options_snapshot; },
};
/** Parse 'symbol:expiry:strike' from greeks cache key id. */
function parseGreeksId(id: string): { symbol: string; expiry: string; strike: string } {
const parts = id.split(':');
return { symbol: parts[0] ?? '', expiry: parts[1] ?? '', strike: parts[2] ?? '0' };
}
const fetchHandler: KindHandler = {
ttlClass: 'daily_permanent',
read(d, id) {
const r = d.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(`sec-fetch:${id}`) as { value: string; observed_at: string } | undefined;
if (!r) return null;
try { return { value: JSON.parse(r.value), stalenessTs: r.observed_at }; } catch { return null; }
},
write(d, id, value, provenance) {
d.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)').run(`sec-fetch:${id}`, JSON.stringify(value), provenance.fetchedAt);
},
isStale(ts) { return ts === null; }, // never stale once written
};
const lintHoldersHandler: KindHandler = {
ttlClass: 'daily_permanent',
read() { return null; }, // never read — work happens in DB tables directly
write(d, key, _value, provenance) {
d.prepare('INSERT OR REPLACE INTO kv_cache (key,value,observed_at) VALUES (?,?,?)').run(`sec-lint-holders:${key}`, JSON.stringify({ ok: true }), provenance.fetchedAt);
},
isStale() { return false; }, // never stale once written (lint writes are permanent)
};
const lintInsidersHandler: KindHandler = {
ttlClass: 'daily_permanent',
read() { return null; }, // never read — work happens in DB tables directly
write(d, key, _value, provenance) {
d.prepare('INSERT OR REPLACE INTO kv_cache (key,value,observed_at) VALUES (?,?,?)').run(`sec-lint-insiders:${key}`, JSON.stringify({ ok: true }), provenance.fetchedAt);
},
isStale() { return false; }, // never stale once written
};
const HANDLERS = new Map<string, KindHandler>([
['quote', quoteHandler],
['candles', candlesHandler],
@@ -212,6 +294,10 @@ const HANDLERS = new Map<string, KindHandler>([
['adjustments', adjustmentsHandler],
['chain', optionsChainHandler],
['expiry_dates', optionsExpiryDatesHandler],
['greeks', greeksHandler],
['fetch', fetchHandler],
['holders', lintHoldersHandler],
['insiders', lintInsidersHandler],
]);
export interface CacheRepository {