feat(sec-lint): lint+backfill system for SEC data gaps (B1-B4)
- SecLintAdapter implements SourceFetch, runs via shared queue/drain loop - Two new SourceKinds: sec-lint-holders, sec-lint-insiders (weekly schedules) - No-op cache handlers so drain->cache.set doesn't throw on lint keys - tRPC admin.queueLint(symbol, kind) — run lint for one symbol, returns LintResult - tRPC admin.queueLintAll(kind) — backfill ALL watched symbols at once - tRPC admin.dataQualityList() — query data_quality rows (filterable by symbol/kind) - InstitutionalDashboard: 'Lint holders' button + status badge in detail panel header - Admin queue page: Data Quality section with per-row status badges, 'Lint all' buttons - DEFAULT_RATE_MS includes 167ms (~6 req/s) for lint kinds matching EDGAR limiter
This commit is contained in:
@@ -10,6 +10,7 @@ import { STARTER_WATCHLIST, defaultDrawdownTolerancePct, defaultRiskTolerance, O
|
||||
import type { Quote, PriceCandle, SymbolMeta } from '../cache/CacheRepository.ts';
|
||||
import { emaFromCandles, rsi as rsiFn, relativeVolume } from '../analysis/indicators.ts';
|
||||
import { listUsers, resetPassword, gdprExport, queueHealth, resetQueueBackoff, NotOwnerError, listUserSessions, listAuditLog, queueSecFetch } from '../admin/admin.ts';
|
||||
import type { LintResult } from '../services/secDataFetcher.ts';
|
||||
import { EdgarAdapter } from '../adapters/EdgarAdapter.ts';
|
||||
import { OptionsAdapter, parseOptionChainRows } from '../adapters/OptionsAdapter.ts';
|
||||
import type { OptionChainRow, OptionGreeks } from '../adapters/OptionsAdapter.ts';
|
||||
@@ -328,6 +329,53 @@ const adminRouter = router({
|
||||
ctx.queue.deleteSchedule(input.sourceKind);
|
||||
return { ok: true };
|
||||
}),
|
||||
|
||||
queueLint: adminProcedure
|
||||
.input(z.object({ symbol: z.string().min(1).max(10), kind: z.enum(['sec-lint-holders', 'sec-lint-insiders']) }))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const { lintInstitutionalHolders, lintInsiderTransactions } = await import('../services/secDataFetcher.ts');
|
||||
let result: LintResult;
|
||||
if (input.kind === 'sec-lint-holders') {
|
||||
result = await lintInstitutionalHolders(ctx.db, input.symbol);
|
||||
} else {
|
||||
result = await lintInsiderTransactions(ctx.db, input.symbol);
|
||||
}
|
||||
// Also enqueue so the weekly schedule picks it up too.
|
||||
try { ctx.queue.queue(`${input.kind}:${input.kind === 'sec-lint-holders' ? 'holders' : 'insiders'}:${input.symbol}`); } catch { /* non-fatal */ }
|
||||
return result;
|
||||
}),
|
||||
|
||||
dataQualityList: adminProcedure
|
||||
.input(z.object({ symbol: z.string().min(1).max(20).optional(), kind: z.enum(['institution_filings', 'insider_transactions']).nullish() }))
|
||||
.query(({ ctx, input }) => {
|
||||
let sql = 'SELECT symbol, kind, last_checked_at, stored_count, discovered_count, missing_count, stale, status, detail FROM data_quality WHERE 1=1';
|
||||
const params: unknown[] = [];
|
||||
if (input.symbol) { sql += ' AND symbol = ?'; params.push(input.symbol); }
|
||||
if (input.kind) { sql += ' AND kind = ?'; params.push(input.kind); }
|
||||
sql += ' ORDER BY last_checked_at DESC LIMIT 200';
|
||||
return ctx.db.prepare(sql).all(...params) as Array<Record<string, unknown>>;
|
||||
}),
|
||||
|
||||
queueLintAll: adminProcedure
|
||||
.input(z.object({ kind: z.enum(['sec-lint-holders', 'sec-lint-insiders']) }))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const { lintInstitutionalHolders, lintInsiderTransactions } = await import('../services/secDataFetcher.ts');
|
||||
const symbols = (ctx.db.prepare('SELECT symbol FROM symbol_demand WHERE in_demand=1 ORDER BY symbol').all() as Array<{ symbol: string }>).map((r) => r.symbol);
|
||||
const results: LintResult[] = [];
|
||||
for (const sym of symbols) {
|
||||
try {
|
||||
if (input.kind === 'sec-lint-holders') {
|
||||
results.push(await lintInstitutionalHolders(ctx.db, sym));
|
||||
} else {
|
||||
results.push(await lintInsiderTransactions(ctx.db, sym));
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
results.push({ symbol: sym.toUpperCase(), kind: input.kind === 'sec-lint-holders' ? 'institution_filings' : 'insider_transactions', discoveredCount: 0, storedCount: 0, missingCount: 0, backfilled: 0, stale: 1, status: 'error', detail: { reason: msg } });
|
||||
}
|
||||
}
|
||||
return { total: symbols.length, results };
|
||||
}),
|
||||
});
|
||||
|
||||
|
||||
@@ -646,6 +694,7 @@ interface Form13fHoldingsResult {
|
||||
value: number;
|
||||
sshPrnamt: number;
|
||||
}>;
|
||||
total: number;
|
||||
accession: string;
|
||||
}
|
||||
|
||||
@@ -722,14 +771,23 @@ const edgarRouter = router({
|
||||
}
|
||||
}),
|
||||
|
||||
/** 13F-HR holdings for a CIK + accession. */
|
||||
/** 13F-HR holdings for a CIK + accession (server-side paginated). */
|
||||
form13f_holdings: publicProcedure
|
||||
.input(z.object({ cik: z.string().min(1), accession: z.string().min(1) }))
|
||||
.input(z.object({
|
||||
cik: z.string().min(1),
|
||||
accession: z.string().min(1),
|
||||
limit: z.number().int().positive().optional(),
|
||||
offset: z.number().int().min(0).optional(),
|
||||
}))
|
||||
.query(async ({ ctx, input }) => {
|
||||
const edgar = new EdgarAdapter();
|
||||
try {
|
||||
const result = await edgar.form13f_holdings(input.cik, input.accession);
|
||||
return { holdings: result.value as Form13fHoldingsResult };
|
||||
const result = await edgar.form13f_holdings(input.cik, input.accession, {
|
||||
limit: input.limit,
|
||||
offset: input.offset,
|
||||
});
|
||||
const v = result.value as Form13fHoldingsResult;
|
||||
return { holdings: v.holdings, total: v.total, accession: v.accession };
|
||||
} catch (e) {
|
||||
throw new TRPCError({ code: 'NOT_FOUND', message: (e as Error).message });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user