feat(sec-lint): lint+backfill system for SEC data gaps (B1-B4)

- SecLintAdapter implements SourceFetch, runs via shared queue/drain loop
- Two new SourceKinds: sec-lint-holders, sec-lint-insiders (weekly schedules)
- No-op cache handlers so drain->cache.set doesn't throw on lint keys
- tRPC admin.queueLint(symbol, kind) — run lint for one symbol, returns LintResult
- tRPC admin.queueLintAll(kind) — backfill ALL watched symbols at once
- tRPC admin.dataQualityList() — query data_quality rows (filterable by symbol/kind)
- InstitutionalDashboard: 'Lint holders' button + status badge in detail panel header
- Admin queue page: Data Quality section with per-row status badges, 'Lint all' buttons
- DEFAULT_RATE_MS includes 167ms (~6 req/s) for lint kinds matching EDGAR limiter
This commit is contained in:
Investor Flow Build
2026-07-12 20:19:12 -04:00
parent ca385c1960
commit 5ef2b2f060
9 changed files with 1154 additions and 47 deletions
+55
View File
@@ -0,0 +1,55 @@
// Investor Flow — SecLintAdapter (B2). Admin-triggered or scheduled lint+backfill for SEC data.
// Implements SourceFetch so the shared queue/drain loop can process it like any other source,
// but actual work writes directly to institution_filings/insider_transactions/data_quality tables.
// The kv_cache write from drain→cache.set is a harmless no-op side effect (never read).
import type { DatabaseSync } from 'node:sqlite';
import type { SourceFetch, FetchResult } from './SourceAdapter.ts';
import type { CacheKey, TtlClass, Provenance } from '../cache/CacheRepository.ts';
import { lintInstitutionalHolders, lintInsiderTransactions, type LintResult } from '../services/secDataFetcher.ts';
export class SecLintAdapter implements SourceFetch {
readonly sourceKind: 'sec-lint-holders' | 'sec-lint-insiders';
private db: () => DatabaseSync;
constructor(db: () => DatabaseSync, sourceKind: 'sec-lint-holders' | 'sec-lint-insiders') {
this.db = db;
this.sourceKind = sourceKind;
}
async fetchOne(key: CacheKey): Promise<FetchResult> {
// key format: sec-lint-holders:holders:{symbol} or sec-lint-insiders:insiders:{symbol}
const parts = key.split(':');
const symbol = (parts[2] ?? '').toUpperCase();
let result: LintResult;
if (this.sourceKind === 'sec-lint-holders') {
result = await lintInstitutionalHolders(this.db(), symbol);
} else {
result = await lintInsiderTransactions(this.db(), symbol);
}
const value: FetchResult['value'] = { lintDone: true, ...result };
return {
value,
ttlClass: 'daily_permanent' as TtlClass,
provenance: { fetchedAt: new Date().toISOString(), sourceKind: this.sourceKind },
};
}
/** Run lint for ALL watched symbols at once (used by admin "run all" button). */
async runAllSymbols(kind: 'sec-lint-holders' | 'sec-lint-insiders', db: DatabaseSync): Promise<LintResult[]> {
const symbols = (db.prepare('SELECT symbol FROM symbol_demand WHERE in_demand=1 ORDER BY symbol').all() as Array<{ symbol: string }>).map((r) => r.symbol);
const results: LintResult[] = [];
for (const sym of symbols) {
try {
if (kind === 'sec-lint-holders') {
results.push(await lintInstitutionalHolders(db, sym));
} else {
results.push(await lintInsiderTransactions(db, sym));
}
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
results.push({ symbol: sym.toUpperCase(), kind, discoveredCount: 0, storedCount: 0, missingCount: 0, backfilled: 0, stale: 1, status: 'error', detail: { reason: msg } });
}
}
return results;
}
}