feat: admin-configurable FINRA download URL
- Add finra_config singleton table (base_url + updated_at) - Add getFinraBaseUrl() and setFinraBaseUrl() to admin.ts Resolution order: DB config > env FINRA_BASE_URL > built-in default - Add finraConfig query + finraConfigUpdate mutation to admin tRPC router - Update FinraIngestService.downloadAndIngestFinra to accept optional baseUrl param; falls back to getFinraBaseUrl(db) when omitted - FinraBulkAdapter resolves URL implicitly via the service (no adapter change) - Add migration #10 for existing DBs
This commit is contained in:
@@ -288,4 +288,19 @@ export function resetQueueBackoff(
|
|||||||
.run(sourceKind);
|
.run(sourceKind);
|
||||||
recordAudit(db, actorId ?? 'cli', 'queue.reset-backoff', null, { source: sourceKind, cleared: Number(res.changes) });
|
recordAudit(db, actorId ?? 'cli', 'queue.reset-backoff', null, { source: sourceKind, cleared: Number(res.changes) });
|
||||||
return { cleared: Number(res.changes) };
|
return { cleared: Number(res.changes) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Resolve the effective FINRA download URL: DB config > env var > built-in default. */
|
||||||
|
export function getFinraBaseUrl(db: DatabaseSync): string {
|
||||||
|
const row = db.prepare("SELECT base_url FROM finra_config WHERE id='singleton'").get() as { base_url: string } | undefined;
|
||||||
|
return row?.base_url ?? process.env.FINRA_BASE_URL ?? 'https://www.finra.org/sites/default/files';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Update FINRA download URL in DB config. Returns the new URL. */
|
||||||
|
export function setFinraBaseUrl(db: DatabaseSync, baseUrl: string): { baseUrl: string } {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
db.prepare(
|
||||||
|
"INSERT OR REPLACE INTO finra_config (id, base_url, updated_at) VALUES ('singleton', ?, ?)"
|
||||||
|
).run(baseUrl, now);
|
||||||
|
return { baseUrl };
|
||||||
}
|
}
|
||||||
@@ -232,6 +232,17 @@ function runMigrations(db: DatabaseSync): void {
|
|||||||
)
|
)
|
||||||
`);
|
`);
|
||||||
} catch { /* ignore */ }
|
} catch { /* ignore */ }
|
||||||
|
|
||||||
|
// 10. FINRA config table for admin-configurable download URL.
|
||||||
|
try {
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS finra_config (
|
||||||
|
id TEXT PRIMARY KEY DEFAULT 'singleton',
|
||||||
|
base_url TEXT NOT NULL DEFAULT 'https://www.finra.org/sites/default/files',
|
||||||
|
updated_at TEXT NOT NULL
|
||||||
|
)
|
||||||
|
`);
|
||||||
|
} catch { /* ignore */ }
|
||||||
}
|
}
|
||||||
|
|
||||||
let _db: DatabaseSync | null = null;
|
let _db: DatabaseSync | null = null;
|
||||||
|
|||||||
@@ -694,3 +694,10 @@ CREATE TABLE IF NOT EXISTS finra_short_interest (
|
|||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_finra_symbol ON finra_short_interest(symbol);
|
CREATE INDEX IF NOT EXISTS idx_finra_symbol ON finra_short_interest(symbol);
|
||||||
CREATE INDEX IF NOT EXISTS idx_finra_settlement ON finra_short_interest(settlement_date);
|
CREATE INDEX IF NOT EXISTS idx_finra_settlement ON finra_short_interest(settlement_date);
|
||||||
|
|
||||||
|
-- FINRA download configuration (admin-configurable URL).
|
||||||
|
CREATE TABLE IF NOT EXISTS finra_config (
|
||||||
|
id TEXT PRIMARY KEY DEFAULT 'singleton',
|
||||||
|
base_url TEXT NOT NULL DEFAULT 'https://www.finra.org/sites/default/files',
|
||||||
|
updated_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
import type { DatabaseSync } from 'node:sqlite';
|
import type { DatabaseSync } from 'node:sqlite';
|
||||||
|
import { getFinraBaseUrl } from '../admin/admin.ts';
|
||||||
const FINRA_BASE_URL = process.env.FINRA_BASE_URL ?? 'https://www.finra.org/sites/default/files';
|
|
||||||
|
|
||||||
/** Format a FINRA consolidated-short-interest filename: CAshvol{YYYYMMDD}.txt */
|
/** Format a FINRA consolidated-short-interest filename: CAshvol{YYYYMMDD}.txt */
|
||||||
function finraFilename(settlementDate: string): string {
|
function finraFilename(settlementDate: string): string {
|
||||||
@@ -64,13 +63,14 @@ function parseFinraFile(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Download and ingest a FINRA consolidated-short-interest file.
|
/** Download and ingest a FINRA consolidated-short-interest file.
|
||||||
* Returns count of symbols stored. */
|
* baseUrl resolves from DB config > env FINRA_BASE_URL > built-in default when omitted. */
|
||||||
export async function downloadAndIngestFinra(
|
export async function downloadAndIngestFinra(
|
||||||
db: DatabaseSync,
|
db: DatabaseSync,
|
||||||
settlementDate: string,
|
settlementDate: string,
|
||||||
|
baseUrl?: string,
|
||||||
): Promise<{ symbolsStored: number; sourceFile: string; exchanges: string[] }> {
|
): Promise<{ symbolsStored: number; sourceFile: string; exchanges: string[] }> {
|
||||||
|
const url = baseUrl ?? getFinraBaseUrl(db);
|
||||||
const filename = finraFilename(settlementDate);
|
const filename = finraFilename(settlementDate);
|
||||||
const url = `${FINRA_BASE_URL}/${filename}`;
|
|
||||||
const ingestedAt = new Date().toISOString();
|
const ingestedAt = new Date().toISOString();
|
||||||
|
|
||||||
console.log(`[finra] downloading ${url}`);
|
console.log(`[finra] downloading ${url}`);
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import type { ThesisEvent } from '../thesis/ThesisMonitor.ts';
|
|||||||
import type { OptionsUnlockState } from '../options/ConvexityGate.ts';
|
import type { OptionsUnlockState } from '../options/ConvexityGate.ts';
|
||||||
import type { XCookieHealth } from '../adapters/XCookieAdapter.ts';
|
import type { XCookieHealth } from '../adapters/XCookieAdapter.ts';
|
||||||
import { emaFromCandles, rsi as rsiFn, relativeVolume, macd as macdFn } from '../analysis/indicators.ts';
|
import { emaFromCandles, rsi as rsiFn, relativeVolume, macd as macdFn } from '../analysis/indicators.ts';
|
||||||
import { listUsers, resetPassword, gdprExport, queueHealth, resetQueueBackoff, NotOwnerError, listUserSessions, listAuditLog, queueSecFetch, setUserModules, disableUser, enableUser, deleteUser } from '../admin/admin.ts';
|
import { listUsers, resetPassword, gdprExport, queueHealth, resetQueueBackoff, NotOwnerError, listUserSessions, listAuditLog, queueSecFetch, setUserModules, disableUser, enableUser, deleteUser, getFinraBaseUrl, setFinraBaseUrl } from '../admin/admin.ts';
|
||||||
import { restartServers, type RestartTarget } from '../lib/restart.ts';
|
import { restartServers, type RestartTarget } from '../lib/restart.ts';
|
||||||
import type { LintResult } from '../services/secDataFetcher.ts';
|
import type { LintResult } from '../services/secDataFetcher.ts';
|
||||||
import { EdgarAdapter } from '../adapters/EdgarAdapter.ts';
|
import { EdgarAdapter } from '../adapters/EdgarAdapter.ts';
|
||||||
@@ -1466,6 +1466,16 @@ const adminRouter = router({
|
|||||||
if (!sent) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to send test email.' });
|
if (!sent) throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Failed to send test email.' });
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
/** Get FINRA config (admin-only). */
|
||||||
|
finraConfig: adminProcedure.query(({ ctx }) => ({
|
||||||
|
baseUrl: getFinraBaseUrl(ctx.db),
|
||||||
|
})),
|
||||||
|
|
||||||
|
/** Update FINRA download URL (admin-only). */
|
||||||
|
finraConfigUpdate: adminProcedure
|
||||||
|
.input(z.object({ baseUrl: z.string().url() }))
|
||||||
|
.mutation(({ ctx, input }) => setFinraBaseUrl(ctx.db, input.baseUrl)),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user