fix: populate short interest cache directly instead of relying on clogged drain queue

The stale-while-revalidate pattern queues background fetches via the AdapterQueue,
but with 227+ pending jobs, short interest keys take too long to process.
Users saw empty panels on every symbol because the fetch never completed before
the response returned.

Fix:
- shortInterest procedure: when Yahoo + NASDAQ caches are both empty, fetch
  directly via the adapters, store in cache, and return — all within the request.
  Subsequent requests hit the cache. Bypasses the clogged queue entirely.
- FinraBulkAdapter: guard against per-symbol keys (non-'latest' and non-date IDs)
  by returning a skipped result instead of trying to download a file named
  after the ticker.
- finraShortInterest handler: return isStale=false when no data is in the table,
  so per-symbol reads never trigger queue fetches (FINRA is schedule-only).
  Router stale computation now excludes FINRA from the combined stale flag.
This commit is contained in:
Investor Flow Build
2026-07-25 13:23:07 -04:00
parent 3d12f50418
commit 04fc11b2fd
3 changed files with 41 additions and 4 deletions
+26 -3
View File
@@ -1928,10 +1928,33 @@ const institutionalRouter = router({
ctx.cache.get<Record<string, unknown>>(`nasdaq:nasdaqShortinterest:${symbol}`),
ctx.cache.get<Record<string, unknown>>(`finra-bulk:finraShortinterest:${symbol}`),
]);
const yf = yfEntry.value;
const ndq = ndqEntry.value;
let yf = yfEntry.value;
let ndq = ndqEntry.value;
const fr = frEntry.value;
// Direct fetch when cache is empty — the drain queue can be clogged with
// hundreds of pending jobs, making stale-while-revalidate too slow.
if (!yf && !ndq) {
const { YFinanceAdapter } = await import('../adapters/YFinanceAdapter.ts');
const { NasdaqAdapter } = await import('../adapters/NasdaqAdapter.ts');
const results = await Promise.allSettled([
(async () => {
const ya = new YFinanceAdapter();
const r = await ya.fetchOne(`yfinance:shortinterest:${symbol}`);
await ctx.cache.set(`yfinance:shortinterest:${symbol}`, r.value, r.ttlClass, r.provenance);
return r.value as Record<string, unknown>;
})(),
(async () => {
const na = new NasdaqAdapter();
const r = await na.fetchOne(`nasdaq:nasdaqShortinterest:${symbol}`);
await ctx.cache.set(`nasdaq:nasdaqShortinterest:${symbol}`, r.value, r.ttlClass, r.provenance);
return r.value as Record<string, unknown>;
})(),
]);
if (results[0].status === 'fulfilled') yf = results[0].value;
if (results[1].status === 'fulfilled') ndq = results[1].value;
}
// Collect sharesShort from all sources for discrepancy detection
const sharesSources: Array<{ source: string; sharesShort: number | null }> = [
{ source: 'finra', sharesShort: (fr?.shortVolume as number | undefined) ?? null },
@@ -1980,7 +2003,7 @@ const institutionalRouter = router({
nasdaq: { available: !!ndq, stale: ndqEntry.isStale },
finra: { available: !!fr, stale: frEntry.isStale },
},
stale: yfEntry.isStale && ndqEntry.isStale && frEntry.isStale,
stale: yfEntry.isStale && ndqEntry.isStale, // FINRA is schedule-only, not per-symbol
};
}),
});