feat: admin-configurable FINRA download URL

- Add finra_config singleton table (base_url + updated_at)
- Add getFinraBaseUrl() and setFinraBaseUrl() to admin.ts
  Resolution order: DB config > env FINRA_BASE_URL > built-in default
- Add finraConfig query + finraConfigUpdate mutation to admin tRPC router
- Update FinraIngestService.downloadAndIngestFinra to accept optional baseUrl
  param; falls back to getFinraBaseUrl(db) when omitted
- FinraBulkAdapter resolves URL implicitly via the service (no adapter change)
- Add migration #10 for existing DBs
This commit is contained in:
Investor Flow Build
2026-07-25 13:11:50 -04:00
parent 0b9df1e9d3
commit 8a00363dd0
5 changed files with 48 additions and 5 deletions
+15
View File
@@ -288,4 +288,19 @@ export function resetQueueBackoff(
.run(sourceKind);
recordAudit(db, actorId ?? 'cli', 'queue.reset-backoff', null, { source: sourceKind, cleared: Number(res.changes) });
return { cleared: Number(res.changes) };
}
/** Resolve the effective FINRA download URL: DB config > env var > built-in default. */
export function getFinraBaseUrl(db: DatabaseSync): string {
const row = db.prepare("SELECT base_url FROM finra_config WHERE id='singleton'").get() as { base_url: string } | undefined;
return row?.base_url ?? process.env.FINRA_BASE_URL ?? 'https://www.finra.org/sites/default/files';
}
/** Update FINRA download URL in DB config. Returns the new URL. */
export function setFinraBaseUrl(db: DatabaseSync, baseUrl: string): { baseUrl: string } {
const now = new Date().toISOString();
db.prepare(
"INSERT OR REPLACE INTO finra_config (id, base_url, updated_at) VALUES ('singleton', ?, ?)"
).run(baseUrl, now);
return { baseUrl };
}
+11
View File
@@ -232,6 +232,17 @@ function runMigrations(db: DatabaseSync): void {
)
`);
} catch { /* ignore */ }
// 10. FINRA config table for admin-configurable download URL.
try {
db.exec(`
CREATE TABLE IF NOT EXISTS finra_config (
id TEXT PRIMARY KEY DEFAULT 'singleton',
base_url TEXT NOT NULL DEFAULT 'https://www.finra.org/sites/default/files',
updated_at TEXT NOT NULL
)
`);
} catch { /* ignore */ }
}
let _db: DatabaseSync | null = null;
+7
View File
@@ -694,3 +694,10 @@ CREATE TABLE IF NOT EXISTS finra_short_interest (
CREATE INDEX IF NOT EXISTS idx_finra_symbol ON finra_short_interest(symbol);
CREATE INDEX IF NOT EXISTS idx_finra_settlement ON finra_short_interest(settlement_date);
-- FINRA download configuration (admin-configurable URL).
CREATE TABLE IF NOT EXISTS finra_config (
id TEXT PRIMARY KEY DEFAULT 'singleton',
base_url TEXT NOT NULL DEFAULT 'https://www.finra.org/sites/default/files',
updated_at TEXT NOT NULL
);
@@ -1,6 +1,5 @@
import type { DatabaseSync } from 'node:sqlite';
const FINRA_BASE_URL = process.env.FINRA_BASE_URL ?? 'https://www.finra.org/sites/default/files';
import { getFinraBaseUrl } from '../admin/admin.ts';
/** Format a FINRA consolidated-short-interest filename: CAshvol{YYYYMMDD}.txt */
function finraFilename(settlementDate: string): string {
@@ -64,13 +63,14 @@ function parseFinraFile(
}
/** Download and ingest a FINRA consolidated-short-interest file.
* Returns count of symbols stored. */
* baseUrl resolves from DB config > env FINRA_BASE_URL > built-in default when omitted. */
export async function downloadAndIngestFinra(
db: DatabaseSync,
settlementDate: string,
baseUrl?: string,
): Promise<{ symbolsStored: number; sourceFile: string; exchanges: string[] }> {
const url = baseUrl ?? getFinraBaseUrl(db);
const filename = finraFilename(settlementDate);
const url = `${FINRA_BASE_URL}/${filename}`;
const ingestedAt = new Date().toISOString();
console.log(`[finra] downloading ${url}`);
+11 -1
View File
@@ -14,7 +14,7 @@ import type { ThesisEvent } from '../thesis/ThesisMonitor.ts';
import type { OptionsUnlockState } from '../options/ConvexityGate.ts';
import type { XCookieHealth } from '../adapters/XCookieAdapter.ts';
import { emaFromCandles, rsi as rsiFn, relativeVolume, macd as macdFn } from '../analysis/indicators.ts';
import { listUsers, resetPassword, gdprExport, queueHealth, resetQueueBackoff, NotOwnerError, listUserSessions, listAuditLog, queueSecFetch, setUserModules, disableUser, enableUser, deleteUser } from '../admin/admin.ts';
import { listUsers, resetPassword, gdprExport, queueHealth, resetQueueBackoff, NotOwnerError, listUserSessions, listAuditLog, queueSecFetch, setUserModules, disableUser, enableUser, deleteUser, getFinraBaseUrl, setFinraBaseUrl } from '../admin/admin.ts';
import { restartServers, type RestartTarget } from '../lib/restart.ts';
import type { LintResult } from '../services/secDataFetcher.ts';
import { EdgarAdapter } from '../adapters/EdgarAdapter.ts';
@@ -1466,6 +1466,16 @@ const adminRouter = router({
if (!sent) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to send test email.' });
return { ok: true };
}),
/** Get FINRA config (admin-only). */
finraConfig: adminProcedure.query(({ ctx }) => ({
baseUrl: getFinraBaseUrl(ctx.db),
})),
/** Update FINRA download URL (admin-only). */
finraConfigUpdate: adminProcedure
.input(z.object({ baseUrl: z.string().url() }))
.mutation(({ ctx, input }) => setFinraBaseUrl(ctx.db, input.baseUrl)),
});