transnet/investor-flow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correct FINRA URL, filename format, parser column order, and default base URL

Browse Source 
- Fix default base URL: https://cdn.finra.org/equity/regsho/daily (was www.finra.org/sites/default/files)
- Fix filename pattern: CNMSshvol{YYYYMMDD}.txt (was CAshvol with year/month subdir)
- Fix parser: actual file columns are Date|Symbol|ShortVolume|ShortExemptVolume|TotalVolume|Market
  (was incorrectly expecting Market|Symbol|... — first col is Date, last col is Market)
- Dedupe by symbol (volume is total across all listed exchanges, not per-exchange)
- Store combined market codes (e.g. 'B,Q,N') as exchange string
- Use BEGIN/COMMIT instead of db.transaction() (node:sqlite DatabaseSync doesn't have transaction())
- Update finra_config defaults in schema.sql + migration + admin fallback
This commit is contained in:
Investor Flow Build
2026-07-25 13:14:05 -04:00
parent 8a00363dd0
commit 48872b7805
4 changed files with 50 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/server/src/admin/admin.ts
+1 -1
View File
@@ -293,7 +293,7 @@ export function resetQueueBackoff(
/** Resolve the effective FINRA download URL: DB config > env var > built-in default. */
export function getFinraBaseUrl(db: DatabaseSync): string {
const row = db.prepare("SELECT base_url FROM finra_config WHERE id='singleton'").get() as { base_url: string } | undefined;
return row?.base_url ?? process.env.FINRA_BASE_URL ?? 'https://www.finra.org/sites/default/files';
return row?.base_url ?? process.env.FINRA_BASE_URL ?? 'https://cdn.finra.org/equity/regsho/daily';
}
/** Update FINRA download URL in DB config. Returns the new URL. */
app/server/src/db/client.ts
+1 -1
View File
@@ -238,7 +238,7 @@ function runMigrations(db: DatabaseSync): void {
db.exec(`
CREATE TABLE IF NOT EXISTS finra_config (
id TEXT PRIMARY KEY DEFAULT 'singleton',
base_url TEXT NOT NULL DEFAULT 'https://www.finra.org/sites/default/files',
base_url TEXT NOT NULL DEFAULT 'https://cdn.finra.org/equity/regsho/daily',
updated_at TEXT NOT NULL
)
`);
app/server/src/db/schema.sql
+1 -1
View File
@@ -698,6 +698,6 @@ CREATE INDEX IF NOT EXISTS idx_finra_settlement ON finra_short_interest(settleme
-- FINRA download configuration (admin-configurable URL).
CREATE TABLE IF NOT EXISTS finra_config (
id TEXT PRIMARY KEY DEFAULT 'singleton',
base_url TEXT NOT NULL DEFAULT 'https://www.finra.org/sites/default/files',
base_url TEXT NOT NULL DEFAULT 'https://cdn.finra.org/equity/regsho/daily',
updated_at TEXT NOT NULL
);
app/server/src/services/FinraIngestService.ts
+47 -29
View File
@@ -1,16 +1,17 @@
import type { DatabaseSync } from 'node:sqlite';
import { getFinraBaseUrl } from '../admin/admin.ts';
/** Format a FINRA consolidated-short-interest filename: CAshvol{YYYYMMDD}.txt */
/** Format a FINRA consolidated-short-interest filename: CNMSshvol{YYYYMMDD}.txt */
function finraFilename(settlementDate: string): string {
const d = settlementDate.replace(/-/g, '');
const ym = settlementDate.slice(0, 7).replace(/-/, '-');
return `${ym}/CAshvol${d}.txt`;
return `CNMSshvol${d}.txt`;
}
/** Parse a FINRA consolidated-short-interest file body (pipe-delimited).
* Expected columns: Market|Symbol|ShortVolume|ShortExemptVolume|TotalVolume
* Returns per-symbol rows aggregated across all exchanges. */
/** Parse a FINRA CNMS short-vol file body (pipe-delimited).
* Actual columns (verified against live file):
* Date|Symbol|ShortVolume|ShortExemptVolume|TotalVolume|Market
* Where Market is comma-separated exchange codes (B=NYSE, Q=NASDAQ, N=NYSE Amer/ARCA).
* Volumes are TOTAL across all listed exchanges — one row per symbol per settlement date. */
function parseFinraFile(
body: string,
settlementDate: string,
@@ -36,41 +37,56 @@ function parseFinraFile(
for (const raw of lines) {
const line = raw.trim();
if (!line || line.startsWith('#')) continue;
if (line.startsWith('Date Range') || line.startsWith('Period')) continue;
if (line.includes('Market|Symbol|')) { headerFound = true; continue; }
if (line.includes('Date|Symbol|Short') || line.includes('Date|Symbol|')) {
headerFound = true;
continue;
}
if (!headerFound) continue;
const cols = line.split('|').map((c) => c.trim());
if (cols.length < 4) continue;
if (cols.length < 5) continue;
const market = cols[0];
const symbol = cols[1];
const shortVolume = parseFloat(cols[2]?.replace(/,/g, ''));
const shortExempt = cols[3] ? parseFloat(cols[3].replace(/,/g, '')) : 0;
const totalVolume = cols[4] ? parseFloat(cols[4].replace(/,/g, '')) : shortVolume + shortExempt;
// cols[0] = Date (YYYYMMDD), cols[1] = Symbol, cols[4] = TotalVolume, cols[5] = Market
const symbol = cols[1]?.replace(/\/.*$/, ''); // strip warrant suffix like /WS
const rawShort = cols[2]?.replace(/,/g, '') ?? '0';
const rawExempt = cols[3]?.replace(/,/g, '') ?? '0';
const rawTotal = cols[4]?.replace(/,/g, '') ?? rawShort;
const markets = cols[5] ?? '';
const shortVolume = parseFloat(rawShort);
const shortExempt = parseFloat(rawExempt);
const totalVolume = parseFloat(rawTotal);
if (!symbol || Number.isNaN(shortVolume)) continue;
rows.push({
symbol: symbol.toUpperCase(),
exchange: market.toUpperCase(),
shortVolume,
shortExempt,
totalVolume,
});
// Store market codes as combined string (e.g. "B,Q,N")
const normSymbol = symbol.toUpperCase();
const exchangeCode = markets.replace(/\s/g, '').replace(/^,+|,+$/g, '') || 'ALL';
// Dedupe by symbol — the volume is total across all listed exchanges
if (!rows.some((r) => r.symbol === normSymbol)) {
rows.push({
symbol: normSymbol,
exchange: exchangeCode,
shortVolume,
shortExempt,
totalVolume,
});
}
}
return rows;
}
/** Download and ingest a FINRA consolidated-short-interest file.
/** Download and ingest a FINRA consolidated-short-interest file (CNMSshvol{YYYYMMDD}.txt).
* baseUrl resolves from DB config > env FINRA_BASE_URL > built-in default when omitted. */
export async function downloadAndIngestFinra(
db: DatabaseSync,
settlementDate: string,
baseUrl?: string,
): Promise<{ symbolsStored: number; sourceFile: string; exchanges: string[] }> {
const url = baseUrl ?? getFinraBaseUrl(db);
const base = (baseUrl ?? getFinraBaseUrl(db)).replace(/\/+$/, '');
const filename = finraFilename(settlementDate);
const url = `${base}/${filename}`;
const ingestedAt = new Date().toISOString();
console.log(`[finra] downloading ${url}`);
@@ -86,8 +102,6 @@ export async function downloadAndIngestFinra(
if (!rows.length) throw new Error('No FINRA short interest rows parsed');
const exchanges = [...new Set(rows.map((r) => r.exchange))];
const exchangeMap: Record<string, string> = {};
exchanges.forEach((e) => { exchangeMap[e] = e; });
const upsert = db.prepare(
`INSERT OR REPLACE INTO finra_short_interest
@@ -95,7 +109,8 @@ export async function downloadAndIngestFinra(
VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)`
);
const tx = db.transaction(() => {
db.exec('BEGIN TRANSACTION');
try {
for (const r of rows) {
upsert.run(
r.symbol,
@@ -108,10 +123,13 @@ export async function downloadAndIngestFinra(
ingestedAt,
);
}
});
tx();
db.exec('COMMIT');
} catch (e) {
db.exec('ROLLBACK');
throw e;
}
console.log(`[finra] ingested ${rows.length} symbols from ${filename} (exchanges: ${exchanges.join(', ')})`);
console.log(`[finra] ingested ${rows.length} rows from ${filename} (${exchanges.length} exchanges)`);
return { symbolsStored: rows.length, sourceFile: filename, exchanges };
}