feat(sec-lint): lint+backfill system for SEC data gaps (B1-B4)

- SecLintAdapter implements SourceFetch, runs via shared queue/drain loop
- Two new SourceKinds: sec-lint-holders, sec-lint-insiders (weekly schedules)
- No-op cache handlers so drain->cache.set doesn't throw on lint keys
- tRPC admin.queueLint(symbol, kind) — run lint for one symbol, returns LintResult
- tRPC admin.queueLintAll(kind) — backfill ALL watched symbols at once
- tRPC admin.dataQualityList() — query data_quality rows (filterable by symbol/kind)
- InstitutionalDashboard: 'Lint holders' button + status badge in detail panel header
- Admin queue page: Data Quality section with per-row status badges, 'Lint all' buttons
- DEFAULT_RATE_MS includes 167ms (~6 req/s) for lint kinds matching EDGAR limiter
This commit is contained in:
Investor Flow Build
2026-07-12 20:19:12 -04:00
parent ca385c1960
commit 5ef2b2f060
9 changed files with 1154 additions and 47 deletions
+62 -4
View File
@@ -10,6 +10,7 @@ import { STARTER_WATCHLIST, defaultDrawdownTolerancePct, defaultRiskTolerance, O
import type { Quote, PriceCandle, SymbolMeta } from '../cache/CacheRepository.ts';
import { emaFromCandles, rsi as rsiFn, relativeVolume } from '../analysis/indicators.ts';
import { listUsers, resetPassword, gdprExport, queueHealth, resetQueueBackoff, NotOwnerError, listUserSessions, listAuditLog, queueSecFetch } from '../admin/admin.ts';
import type { LintResult } from '../services/secDataFetcher.ts';
import { EdgarAdapter } from '../adapters/EdgarAdapter.ts';
import { OptionsAdapter, parseOptionChainRows } from '../adapters/OptionsAdapter.ts';
import type { OptionChainRow, OptionGreeks } from '../adapters/OptionsAdapter.ts';
@@ -328,6 +329,53 @@ const adminRouter = router({
ctx.queue.deleteSchedule(input.sourceKind);
return { ok: true };
}),
queueLint: adminProcedure
.input(z.object({ symbol: z.string().min(1).max(10), kind: z.enum(['sec-lint-holders', 'sec-lint-insiders']) }))
.mutation(async ({ ctx, input }) => {
const { lintInstitutionalHolders, lintInsiderTransactions } = await import('../services/secDataFetcher.ts');
let result: LintResult;
if (input.kind === 'sec-lint-holders') {
result = await lintInstitutionalHolders(ctx.db, input.symbol);
} else {
result = await lintInsiderTransactions(ctx.db, input.symbol);
}
// Also enqueue so the weekly schedule picks it up too.
try { ctx.queue.queue(`${input.kind}:${input.kind === 'sec-lint-holders' ? 'holders' : 'insiders'}:${input.symbol}`); } catch { /* non-fatal */ }
return result;
}),
dataQualityList: adminProcedure
.input(z.object({ symbol: z.string().min(1).max(20).optional(), kind: z.enum(['institution_filings', 'insider_transactions']).nullish() }))
.query(({ ctx, input }) => {
let sql = 'SELECT symbol, kind, last_checked_at, stored_count, discovered_count, missing_count, stale, status, detail FROM data_quality WHERE 1=1';
const params: unknown[] = [];
if (input.symbol) { sql += ' AND symbol = ?'; params.push(input.symbol); }
if (input.kind) { sql += ' AND kind = ?'; params.push(input.kind); }
sql += ' ORDER BY last_checked_at DESC LIMIT 200';
return ctx.db.prepare(sql).all(...params) as Array<Record<string, unknown>>;
}),
queueLintAll: adminProcedure
.input(z.object({ kind: z.enum(['sec-lint-holders', 'sec-lint-insiders']) }))
.mutation(async ({ ctx, input }) => {
const { lintInstitutionalHolders, lintInsiderTransactions } = await import('../services/secDataFetcher.ts');
const symbols = (ctx.db.prepare('SELECT symbol FROM symbol_demand WHERE in_demand=1 ORDER BY symbol').all() as Array<{ symbol: string }>).map((r) => r.symbol);
const results: LintResult[] = [];
for (const sym of symbols) {
try {
if (input.kind === 'sec-lint-holders') {
results.push(await lintInstitutionalHolders(ctx.db, sym));
} else {
results.push(await lintInsiderTransactions(ctx.db, sym));
}
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
results.push({ symbol: sym.toUpperCase(), kind: input.kind === 'sec-lint-holders' ? 'institution_filings' : 'insider_transactions', discoveredCount: 0, storedCount: 0, missingCount: 0, backfilled: 0, stale: 1, status: 'error', detail: { reason: msg } });
}
}
return { total: symbols.length, results };
}),
});
@@ -646,6 +694,7 @@ interface Form13fHoldingsResult {
value: number;
sshPrnamt: number;
}>;
total: number;
accession: string;
}
@@ -722,14 +771,23 @@ const edgarRouter = router({
}
}),
/** 13F-HR holdings for a CIK + accession. */
/** 13F-HR holdings for a CIK + accession (server-side paginated). */
form13f_holdings: publicProcedure
.input(z.object({ cik: z.string().min(1), accession: z.string().min(1) }))
.input(z.object({
cik: z.string().min(1),
accession: z.string().min(1),
limit: z.number().int().positive().optional(),
offset: z.number().int().min(0).optional(),
}))
.query(async ({ ctx, input }) => {
const edgar = new EdgarAdapter();
try {
const result = await edgar.form13f_holdings(input.cik, input.accession);
return { holdings: result.value as Form13fHoldingsResult };
const result = await edgar.form13f_holdings(input.cik, input.accession, {
limit: input.limit,
offset: input.offset,
});
const v = result.value as Form13fHoldingsResult;
return { holdings: v.holdings, total: v.total, accession: v.accession };
} catch (e) {
throw new TRPCError({ code: 'NOT_FOUND', message: (e as Error).message });
}