From 48872b780562187ddf8e1db904b159a356fa5192 Mon Sep 17 00:00:00 2001 From: Investor Flow Build Date: Sat, 25 Jul 2026 13:14:05 -0400 Subject: [PATCH] fix: correct FINRA URL, filename format, parser column order, and default base URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix default base URL: https://cdn.finra.org/equity/regsho/daily (was www.finra.org/sites/default/files) - Fix filename pattern: CNMSshvol{YYYYMMDD}.txt (was CAshvol with year/month subdir) - Fix parser: actual file columns are Date|Symbol|ShortVolume|ShortExemptVolume|TotalVolume|Market (was incorrectly expecting Market|Symbol|... — first col is Date, last col is Market) - Dedupe by symbol (volume is total across all listed exchanges, not per-exchange) - Store combined market codes (e.g. 'B,Q,N') as exchange string - Use BEGIN/COMMIT instead of db.transaction() (node:sqlite DatabaseSync doesn't have transaction()) - Update finra_config defaults in schema.sql + migration + admin fallback --- app/server/src/admin/admin.ts | 2 +- app/server/src/db/client.ts | 2 +- app/server/src/db/schema.sql | 2 +- app/server/src/services/FinraIngestService.ts | 76 ++++++++++++------- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/app/server/src/admin/admin.ts b/app/server/src/admin/admin.ts index 3914996..a134f9b 100644 --- a/app/server/src/admin/admin.ts +++ b/app/server/src/admin/admin.ts @@ -293,7 +293,7 @@ export function resetQueueBackoff( /** Resolve the effective FINRA download URL: DB config > env var > built-in default. */ export function getFinraBaseUrl(db: DatabaseSync): string { const row = db.prepare("SELECT base_url FROM finra_config WHERE id='singleton'").get() as { base_url: string } | undefined; - return row?.base_url ?? process.env.FINRA_BASE_URL ?? 'https://www.finra.org/sites/default/files'; + return row?.base_url ?? process.env.FINRA_BASE_URL ?? 'https://cdn.finra.org/equity/regsho/daily'; } /** Update FINRA download URL in DB config. Returns the new URL. */ diff --git a/app/server/src/db/client.ts b/app/server/src/db/client.ts index 2ba7c74..133381a 100644 --- a/app/server/src/db/client.ts +++ b/app/server/src/db/client.ts @@ -238,7 +238,7 @@ function runMigrations(db: DatabaseSync): void { db.exec(` CREATE TABLE IF NOT EXISTS finra_config ( id TEXT PRIMARY KEY DEFAULT 'singleton', - base_url TEXT NOT NULL DEFAULT 'https://www.finra.org/sites/default/files', + base_url TEXT NOT NULL DEFAULT 'https://cdn.finra.org/equity/regsho/daily', updated_at TEXT NOT NULL ) `); diff --git a/app/server/src/db/schema.sql b/app/server/src/db/schema.sql index 0e350a5..2f30627 100644 --- a/app/server/src/db/schema.sql +++ b/app/server/src/db/schema.sql @@ -698,6 +698,6 @@ CREATE INDEX IF NOT EXISTS idx_finra_settlement ON finra_short_interest(settleme -- FINRA download configuration (admin-configurable URL). CREATE TABLE IF NOT EXISTS finra_config ( id TEXT PRIMARY KEY DEFAULT 'singleton', - base_url TEXT NOT NULL DEFAULT 'https://www.finra.org/sites/default/files', + base_url TEXT NOT NULL DEFAULT 'https://cdn.finra.org/equity/regsho/daily', updated_at TEXT NOT NULL ); diff --git a/app/server/src/services/FinraIngestService.ts b/app/server/src/services/FinraIngestService.ts index bf062fc..637f2d6 100644 --- a/app/server/src/services/FinraIngestService.ts +++ b/app/server/src/services/FinraIngestService.ts @@ -1,16 +1,17 @@ import type { DatabaseSync } from 'node:sqlite'; import { getFinraBaseUrl } from '../admin/admin.ts'; -/** Format a FINRA consolidated-short-interest filename: CAshvol{YYYYMMDD}.txt */ +/** Format a FINRA consolidated-short-interest filename: CNMSshvol{YYYYMMDD}.txt */ function finraFilename(settlementDate: string): string { const d = settlementDate.replace(/-/g, ''); - const ym = settlementDate.slice(0, 7).replace(/-/, '-'); - return `${ym}/CAshvol${d}.txt`; + return `CNMSshvol${d}.txt`; } -/** Parse a FINRA consolidated-short-interest file body (pipe-delimited). - * Expected columns: Market|Symbol|ShortVolume|ShortExemptVolume|TotalVolume - * Returns per-symbol rows aggregated across all exchanges. */ +/** Parse a FINRA CNMS short-vol file body (pipe-delimited). + * Actual columns (verified against live file): + * Date|Symbol|ShortVolume|ShortExemptVolume|TotalVolume|Market + * Where Market is comma-separated exchange codes (B=NYSE, Q=NASDAQ, N=NYSE Amer/ARCA). + * Volumes are TOTAL across all listed exchanges — one row per symbol per settlement date. */ function parseFinraFile( body: string, settlementDate: string, @@ -36,41 +37,56 @@ function parseFinraFile( for (const raw of lines) { const line = raw.trim(); if (!line || line.startsWith('#')) continue; - if (line.startsWith('Date Range') || line.startsWith('Period')) continue; - if (line.includes('Market|Symbol|')) { headerFound = true; continue; } + if (line.includes('Date|Symbol|Short') || line.includes('Date|Symbol|')) { + headerFound = true; + continue; + } if (!headerFound) continue; const cols = line.split('|').map((c) => c.trim()); - if (cols.length < 4) continue; + if (cols.length < 5) continue; - const market = cols[0]; - const symbol = cols[1]; - const shortVolume = parseFloat(cols[2]?.replace(/,/g, '')); - const shortExempt = cols[3] ? parseFloat(cols[3].replace(/,/g, '')) : 0; - const totalVolume = cols[4] ? parseFloat(cols[4].replace(/,/g, '')) : shortVolume + shortExempt; + // cols[0] = Date (YYYYMMDD), cols[1] = Symbol, cols[4] = TotalVolume, cols[5] = Market + const symbol = cols[1]?.replace(/\/.*$/, ''); // strip warrant suffix like /WS + const rawShort = cols[2]?.replace(/,/g, '') ?? '0'; + const rawExempt = cols[3]?.replace(/,/g, '') ?? '0'; + const rawTotal = cols[4]?.replace(/,/g, '') ?? rawShort; + const markets = cols[5] ?? ''; + + const shortVolume = parseFloat(rawShort); + const shortExempt = parseFloat(rawExempt); + const totalVolume = parseFloat(rawTotal); if (!symbol || Number.isNaN(shortVolume)) continue; - rows.push({ - symbol: symbol.toUpperCase(), - exchange: market.toUpperCase(), - shortVolume, - shortExempt, - totalVolume, - }); + // Store market codes as combined string (e.g. "B,Q,N") + const normSymbol = symbol.toUpperCase(); + const exchangeCode = markets.replace(/\s/g, '').replace(/^,+|,+$/g, '') || 'ALL'; + + // Dedupe by symbol — the volume is total across all listed exchanges + if (!rows.some((r) => r.symbol === normSymbol)) { + rows.push({ + symbol: normSymbol, + exchange: exchangeCode, + shortVolume, + shortExempt, + totalVolume, + }); + } } return rows; } -/** Download and ingest a FINRA consolidated-short-interest file. +/** Download and ingest a FINRA consolidated-short-interest file (CNMSshvol{YYYYMMDD}.txt). * baseUrl resolves from DB config > env FINRA_BASE_URL > built-in default when omitted. */ export async function downloadAndIngestFinra( db: DatabaseSync, settlementDate: string, baseUrl?: string, ): Promise<{ symbolsStored: number; sourceFile: string; exchanges: string[] }> { - const url = baseUrl ?? getFinraBaseUrl(db); + const base = (baseUrl ?? getFinraBaseUrl(db)).replace(/\/+$/, ''); const filename = finraFilename(settlementDate); + const url = `${base}/${filename}`; const ingestedAt = new Date().toISOString(); console.log(`[finra] downloading ${url}`); @@ -86,8 +102,6 @@ export async function downloadAndIngestFinra( if (!rows.length) throw new Error('No FINRA short interest rows parsed'); const exchanges = [...new Set(rows.map((r) => r.exchange))]; - const exchangeMap: Record = {}; - exchanges.forEach((e) => { exchangeMap[e] = e; }); const upsert = db.prepare( `INSERT OR REPLACE INTO finra_short_interest @@ -95,7 +109,8 @@ export async function downloadAndIngestFinra( VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)` ); - const tx = db.transaction(() => { + db.exec('BEGIN TRANSACTION'); + try { for (const r of rows) { upsert.run( r.symbol, @@ -108,10 +123,13 @@ export async function downloadAndIngestFinra( ingestedAt, ); } - }); - tx(); + db.exec('COMMIT'); + } catch (e) { + db.exec('ROLLBACK'); + throw e; + } - console.log(`[finra] ingested ${rows.length} symbols from ${filename} (exchanges: ${exchanges.join(', ')})`); + console.log(`[finra] ingested ${rows.length} rows from ${filename} (${exchanges.length} exchanges)`); return { symbolsStored: rows.length, sourceFile: filename, exchanges }; }