feat(sec-lint): lint+backfill system for SEC data gaps (B1-B4)
- SecLintAdapter implements SourceFetch, runs via shared queue/drain loop - Two new SourceKinds: sec-lint-holders, sec-lint-insiders (weekly schedules) - No-op cache handlers so drain->cache.set doesn't throw on lint keys - tRPC admin.queueLint(symbol, kind) — run lint for one symbol, returns LintResult - tRPC admin.queueLintAll(kind) — backfill ALL watched symbols at once - tRPC admin.dataQualityList() — query data_quality rows (filterable by symbol/kind) - InstitutionalDashboard: 'Lint holders' button + status badge in detail panel header - Admin queue page: Data Quality section with per-row status badges, 'Lint all' buttons - DEFAULT_RATE_MS includes 167ms (~6 req/s) for lint kinds matching EDGAR limiter
This commit is contained in:
+90
-4
@@ -6,7 +6,7 @@
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { db as defaultDb } from '../db/client.ts';
|
||||
|
||||
export type SourceKind = 'yfinance' | 'sec' | 'reddit' | 'x' | 'macro' | 'llm';
|
||||
export type SourceKind = 'yfinance' | 'sec' | 'sec-fetch' | 'reddit' | 'x' | 'macro' | 'llm' | 'sec-lint-holders' | 'sec-lint-insiders';
|
||||
export type TickerKind = 'equity' | 'crypto' | 'etf' | 'index';
|
||||
export type CacheKey = string; // `${SourceKind}:${kind}:${id}` e.g. 'yfinance:quote:NVDA', 'yfinance:candles:NVDA:1d'
|
||||
export type TtlClass =
|
||||
@@ -139,6 +139,7 @@ const symbolHandler: KindHandler = {
|
||||
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.symbol_meta; },
|
||||
};
|
||||
|
||||
// ----- Options handlers (slice 15) -----
|
||||
const optionsChainHandler: KindHandler = {
|
||||
ttlClass: 'options_snapshot',
|
||||
read(d, id) {
|
||||
@@ -177,9 +178,9 @@ const optionsChainHandler: KindHandler = {
|
||||
const right = r.right === 'put' ? 'put' : 'call';
|
||||
ins.run(
|
||||
symbol, expiry, strike, right,
|
||||
r.bid ?? null, r.ask ?? null, r.impliedVolatility ?? null,
|
||||
r.delta ?? null, r.gamma ?? null, r.theta ?? null, r.vega ?? null,
|
||||
r.openInterest ?? null, r.volume ?? null,
|
||||
numOrNull(r.bid), numOrNull(r.ask), numOrNull(r.impliedVolatility),
|
||||
numOrNull(r.delta), numOrNull(r.gamma), numOrNull(r.theta), numOrNull(r.vega),
|
||||
numOrNull(r.openInterest), numOrNull(r.volume),
|
||||
provenance.fetchedAt
|
||||
);
|
||||
}
|
||||
@@ -205,6 +206,87 @@ const optionsExpiryDatesHandler: KindHandler = {
|
||||
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.intraday; },
|
||||
};
|
||||
|
||||
/** Coerce a possibly-undefined/unknown value to number | null for SQL binding. */
|
||||
function numOrNull(v: unknown): number | null {
|
||||
return typeof v === 'number' ? v : null;
|
||||
}
|
||||
|
||||
const greeksHandler: KindHandler = {
|
||||
ttlClass: 'options_snapshot',
|
||||
read(d, id) {
|
||||
const { symbol, expiry, strike } = parseGreeksId(id);
|
||||
const r = d.prepare(
|
||||
'SELECT delta, gamma, theta, vega, strike, open_interest, iv, ts, type FROM options_chains WHERE symbol=? AND expiry=? AND strike=? LIMIT 1'
|
||||
).get(symbol, expiry, parseFloat(strike ?? '0')) as Record<string, unknown> | undefined;
|
||||
if (!r) return null;
|
||||
return {
|
||||
value: {
|
||||
delta: numOrNull(r.delta),
|
||||
gamma: numOrNull(r.gamma),
|
||||
theta: numOrNull(r.theta),
|
||||
vega: numOrNull(r.vega),
|
||||
strike: typeof r.strike === 'number' ? r.strike : 0,
|
||||
openInterest: numOrNull(r.open_interest),
|
||||
impliedVolatility: numOrNull(r.iv),
|
||||
right: r.type as 'call' | 'put',
|
||||
},
|
||||
stalenessTs: r.ts as string,
|
||||
};
|
||||
},
|
||||
write(d, id, value, provenance) {
|
||||
const { symbol, expiry, strike } = parseGreeksId(id);
|
||||
const v = value as Record<string, unknown>;
|
||||
const right = v.right === 'put' ? 'put' : 'call';
|
||||
d.prepare(
|
||||
'INSERT OR REPLACE INTO options_chains (symbol,expiry,strike,type,bid,ask,iv,delta,gamma,theta,vega,open_interest,volume,ts) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
|
||||
).run(
|
||||
symbol, expiry, parseFloat(strike ?? '0'), right,
|
||||
numOrNull(v.bid), numOrNull(v.ask), numOrNull(v.impliedVolatility),
|
||||
numOrNull(v.delta), numOrNull(v.gamma), numOrNull(v.theta), numOrNull(v.vega),
|
||||
numOrNull(v.openInterest), numOrNull(v.volume),
|
||||
provenance.fetchedAt
|
||||
);
|
||||
},
|
||||
isStale(ts, now) { return tsAgeMs(ts, now) > TTL_MS.options_snapshot; },
|
||||
};
|
||||
|
||||
/** Parse 'symbol:expiry:strike' from greeks cache key id. */
|
||||
function parseGreeksId(id: string): { symbol: string; expiry: string; strike: string } {
|
||||
const parts = id.split(':');
|
||||
return { symbol: parts[0] ?? '', expiry: parts[1] ?? '', strike: parts[2] ?? '0' };
|
||||
}
|
||||
|
||||
const fetchHandler: KindHandler = {
|
||||
ttlClass: 'daily_permanent',
|
||||
read(d, id) {
|
||||
const r = d.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(`sec-fetch:${id}`) as { value: string; observed_at: string } | undefined;
|
||||
if (!r) return null;
|
||||
try { return { value: JSON.parse(r.value), stalenessTs: r.observed_at }; } catch { return null; }
|
||||
},
|
||||
write(d, id, value, provenance) {
|
||||
d.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)').run(`sec-fetch:${id}`, JSON.stringify(value), provenance.fetchedAt);
|
||||
},
|
||||
isStale(ts) { return ts === null; }, // never stale once written
|
||||
};
|
||||
|
||||
const lintHoldersHandler: KindHandler = {
|
||||
ttlClass: 'daily_permanent',
|
||||
read() { return null; }, // never read — work happens in DB tables directly
|
||||
write(d, key, _value, provenance) {
|
||||
d.prepare('INSERT OR REPLACE INTO kv_cache (key,value,observed_at) VALUES (?,?,?)').run(`sec-lint-holders:${key}`, JSON.stringify({ ok: true }), provenance.fetchedAt);
|
||||
},
|
||||
isStale() { return false; }, // never stale once written (lint writes are permanent)
|
||||
};
|
||||
|
||||
const lintInsidersHandler: KindHandler = {
|
||||
ttlClass: 'daily_permanent',
|
||||
read() { return null; }, // never read — work happens in DB tables directly
|
||||
write(d, key, _value, provenance) {
|
||||
d.prepare('INSERT OR REPLACE INTO kv_cache (key,value,observed_at) VALUES (?,?,?)').run(`sec-lint-insiders:${key}`, JSON.stringify({ ok: true }), provenance.fetchedAt);
|
||||
},
|
||||
isStale() { return false; }, // never stale once written
|
||||
};
|
||||
|
||||
const HANDLERS = new Map<string, KindHandler>([
|
||||
['quote', quoteHandler],
|
||||
['candles', candlesHandler],
|
||||
@@ -212,6 +294,10 @@ const HANDLERS = new Map<string, KindHandler>([
|
||||
['adjustments', adjustmentsHandler],
|
||||
['chain', optionsChainHandler],
|
||||
['expiry_dates', optionsExpiryDatesHandler],
|
||||
['greeks', greeksHandler],
|
||||
['fetch', fetchHandler],
|
||||
['holders', lintHoldersHandler],
|
||||
['insiders', lintInsidersHandler],
|
||||
]);
|
||||
|
||||
export interface CacheRepository {
|
||||
|
||||
Reference in New Issue
Block a user