fix: correct FINRA URL, filename format, parser column order, and default base URL

- Fix default base URL: https://cdn.finra.org/equity/regsho/daily (was www.finra.org/sites/default/files)
- Fix filename pattern: CNMSshvol{YYYYMMDD}.txt (was CAshvol with year/month subdir)
- Fix parser: actual file columns are Date|Symbol|ShortVolume|ShortExemptVolume|TotalVolume|Market
  (was incorrectly expecting Market|Symbol|... — first col is Date, last col is Market)
- Dedupe by symbol (volume is total across all listed exchanges, not per-exchange)
- Store combined market codes (e.g. 'B,Q,N') as exchange string
- Use BEGIN/COMMIT instead of db.transaction() (node:sqlite DatabaseSync doesn't have transaction())
- Update finra_config defaults in schema.sql + migration + admin fallback

app/server/src/services/FinraIngestService.ts

@@ -1,16 +1,17 @@
 import type { DatabaseSync } from 'node:sqlite';
 import { getFinraBaseUrl } from '../admin/admin.ts';
 
-/** Format a FINRA consolidated-short-interest filename: CAshvol{YYYYMMDD}.txt */
+/** Format a FINRA consolidated-short-interest filename: CNMSshvol{YYYYMMDD}.txt */
 function finraFilename(settlementDate: string): string {
   const d = settlementDate.replace(/-/g, '');
-  const ym = settlementDate.slice(0, 7).replace(/-/, '-');
-  return `${ym}/CAshvol${d}.txt`;
+  return `CNMSshvol${d}.txt`;
 }
 
-/** Parse a FINRA consolidated-short-interest file body (pipe-delimited).
- *  Expected columns: Market|Symbol|ShortVolume|ShortExemptVolume|TotalVolume
- *  Returns per-symbol rows aggregated across all exchanges. */
+/** Parse a FINRA CNMS short-vol file body (pipe-delimited).
+ *  Actual columns (verified against live file):
+ *    Date|Symbol|ShortVolume|ShortExemptVolume|TotalVolume|Market
+ *  Where Market is comma-separated exchange codes (B=NYSE, Q=NASDAQ, N=NYSE Amer/ARCA).
+ *  Volumes are TOTAL across all listed exchanges — one row per symbol per settlement date. */
 function parseFinraFile(
   body: string,
   settlementDate: string,
@@ -36,41 +37,56 @@ function parseFinraFile(
   for (const raw of lines) {
     const line = raw.trim();
     if (!line || line.startsWith('#')) continue;
-    if (line.startsWith('Date Range') || line.startsWith('Period')) continue;
-    if (line.includes('Market|Symbol|')) { headerFound = true; continue; }
+    if (line.includes('Date|Symbol|Short') || line.includes('Date|Symbol|')) {
+      headerFound = true;
+      continue;
+    }
     if (!headerFound) continue;
 
     const cols = line.split('|').map((c) => c.trim());
-    if (cols.length < 4) continue;
+    if (cols.length < 5) continue;
 
-    const market = cols[0];
-    const symbol = cols[1];
-    const shortVolume = parseFloat(cols[2]?.replace(/,/g, ''));
-    const shortExempt = cols[3] ? parseFloat(cols[3].replace(/,/g, '')) : 0;
-    const totalVolume = cols[4] ? parseFloat(cols[4].replace(/,/g, '')) : shortVolume + shortExempt;
+    // cols[0] = Date (YYYYMMDD), cols[1] = Symbol, cols[4] = TotalVolume, cols[5] = Market
+    const symbol = cols[1]?.replace(/\/.*$/, ''); // strip warrant suffix like /WS
+    const rawShort = cols[2]?.replace(/,/g, '') ?? '0';
+    const rawExempt = cols[3]?.replace(/,/g, '') ?? '0';
+    const rawTotal = cols[4]?.replace(/,/g, '') ?? rawShort;
+    const markets = cols[5] ?? '';
+
+    const shortVolume = parseFloat(rawShort);
+    const shortExempt = parseFloat(rawExempt);
+    const totalVolume = parseFloat(rawTotal);
 
     if (!symbol || Number.isNaN(shortVolume)) continue;
 
-    rows.push({
-      symbol: symbol.toUpperCase(),
-      exchange: market.toUpperCase(),
-      shortVolume,
-      shortExempt,
-      totalVolume,
-    });
+    // Store market codes as combined string (e.g. "B,Q,N")
+    const normSymbol = symbol.toUpperCase();
+    const exchangeCode = markets.replace(/\s/g, '').replace(/^,+|,+$/g, '') || 'ALL';
+
+    // Dedupe by symbol — the volume is total across all listed exchanges
+    if (!rows.some((r) => r.symbol === normSymbol)) {
+      rows.push({
+        symbol: normSymbol,
+        exchange: exchangeCode,
+        shortVolume,
+        shortExempt,
+        totalVolume,
+      });
+    }
   }
   return rows;
 }
 
-/** Download and ingest a FINRA consolidated-short-interest file.
+/** Download and ingest a FINRA consolidated-short-interest file (CNMSshvol{YYYYMMDD}.txt).
  *  baseUrl resolves from DB config > env FINRA_BASE_URL > built-in default when omitted. */
 export async function downloadAndIngestFinra(
   db: DatabaseSync,
   settlementDate: string,
   baseUrl?: string,
 ): Promise<{ symbolsStored: number; sourceFile: string; exchanges: string[] }> {
-  const url = baseUrl ?? getFinraBaseUrl(db);
+  const base = (baseUrl ?? getFinraBaseUrl(db)).replace(/\/+$/, '');
   const filename = finraFilename(settlementDate);
+  const url = `${base}/${filename}`;
   const ingestedAt = new Date().toISOString();
 
   console.log(`[finra] downloading ${url}`);
@@ -86,8 +102,6 @@ export async function downloadAndIngestFinra(
   if (!rows.length) throw new Error('No FINRA short interest rows parsed');
 
   const exchanges = [...new Set(rows.map((r) => r.exchange))];
-  const exchangeMap: Record<string, string> = {};
-  exchanges.forEach((e) => { exchangeMap[e] = e; });
 
   const upsert = db.prepare(
     `INSERT OR REPLACE INTO finra_short_interest
@@ -95,7 +109,8 @@ export async function downloadAndIngestFinra(
      VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)`
   );
 
-  const tx = db.transaction(() => {
+  db.exec('BEGIN TRANSACTION');
+  try {
     for (const r of rows) {
       upsert.run(
         r.symbol,
@@ -108,10 +123,13 @@ export async function downloadAndIngestFinra(
         ingestedAt,
       );
     }
-  });
-  tx();
+    db.exec('COMMIT');
+  } catch (e) {
+    db.exec('ROLLBACK');
+    throw e;
+  }
 
-  console.log(`[finra] ingested ${rows.length} symbols from ${filename} (exchanges: ${exchanges.join(', ')})`);
+  console.log(`[finra] ingested ${rows.length} rows from ${filename} (${exchanges.length} exchanges)`);
   return { symbolsStored: rows.length, sourceFile: filename, exchanges };
 }