feat(sec-lint): lint+backfill system for SEC data gaps (B1-B4)
- SecLintAdapter implements SourceFetch, runs via shared queue/drain loop - Two new SourceKinds: sec-lint-holders, sec-lint-insiders (weekly schedules) - No-op cache handlers so drain->cache.set doesn't throw on lint keys - tRPC admin.queueLint(symbol, kind) — run lint for one symbol, returns LintResult - tRPC admin.queueLintAll(kind) — backfill ALL watched symbols at once - tRPC admin.dataQualityList() — query data_quality rows (filterable by symbol/kind) - InstitutionalDashboard: 'Lint holders' button + status badge in detail panel header - Admin queue page: Data Quality section with per-row status badges, 'Lint all' buttons - DEFAULT_RATE_MS includes 167ms (~6 req/s) for lint kinds matching EDGAR limiter
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
// Investor Flow — SecLintAdapter (B2). Admin-triggered or scheduled lint+backfill for SEC data.
|
||||
// Implements SourceFetch so the shared queue/drain loop can process it like any other source,
|
||||
// but actual work writes directly to institution_filings/insider_transactions/data_quality tables.
|
||||
// The kv_cache write from drain→cache.set is a harmless no-op side effect (never read).
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
import type { SourceFetch, FetchResult } from './SourceAdapter.ts';
|
||||
import type { CacheKey, TtlClass, Provenance } from '../cache/CacheRepository.ts';
|
||||
import { lintInstitutionalHolders, lintInsiderTransactions, type LintResult } from '../services/secDataFetcher.ts';
|
||||
|
||||
export class SecLintAdapter implements SourceFetch {
|
||||
readonly sourceKind: 'sec-lint-holders' | 'sec-lint-insiders';
|
||||
private db: () => DatabaseSync;
|
||||
|
||||
constructor(db: () => DatabaseSync, sourceKind: 'sec-lint-holders' | 'sec-lint-insiders') {
|
||||
this.db = db;
|
||||
this.sourceKind = sourceKind;
|
||||
}
|
||||
|
||||
async fetchOne(key: CacheKey): Promise<FetchResult> {
|
||||
// key format: sec-lint-holders:holders:{symbol} or sec-lint-insiders:insiders:{symbol}
|
||||
const parts = key.split(':');
|
||||
const symbol = (parts[2] ?? '').toUpperCase();
|
||||
let result: LintResult;
|
||||
if (this.sourceKind === 'sec-lint-holders') {
|
||||
result = await lintInstitutionalHolders(this.db(), symbol);
|
||||
} else {
|
||||
result = await lintInsiderTransactions(this.db(), symbol);
|
||||
}
|
||||
const value: FetchResult['value'] = { lintDone: true, ...result };
|
||||
return {
|
||||
value,
|
||||
ttlClass: 'daily_permanent' as TtlClass,
|
||||
provenance: { fetchedAt: new Date().toISOString(), sourceKind: this.sourceKind },
|
||||
};
|
||||
}
|
||||
|
||||
/** Run lint for ALL watched symbols at once (used by admin "run all" button). */
|
||||
async runAllSymbols(kind: 'sec-lint-holders' | 'sec-lint-insiders', db: DatabaseSync): Promise<LintResult[]> {
|
||||
const symbols = (db.prepare('SELECT symbol FROM symbol_demand WHERE in_demand=1 ORDER BY symbol').all() as Array<{ symbol: string }>).map((r) => r.symbol);
|
||||
const results: LintResult[] = [];
|
||||
for (const sym of symbols) {
|
||||
try {
|
||||
if (kind === 'sec-lint-holders') {
|
||||
results.push(await lintInstitutionalHolders(db, sym));
|
||||
} else {
|
||||
results.push(await lintInsiderTransactions(db, sym));
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
results.push({ symbol: sym.toUpperCase(), kind, discoveredCount: 0, storedCount: 0, missingCount: 0, backfilled: 0, stale: 1, status: 'error', detail: { reason: msg } });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user