From 351c104ebaeb0e1f098824872a3f33646fa02d46 Mon Sep 17 00:00:00 2001 From: Investor Flow Build Date: Sat, 25 Jul 2026 11:52:48 -0400 Subject: [PATCH] fix: analyst ratings timeout + add accession column to institution_filings - Add 6s server-side timeout to Yahoo Finance quoteSummary (Promise.race) so the backend responds with stale data instead of hanging indefinitely. Stale cache is served on error (router.ts:1895-1898), matching ADR-0009. - Hoist YahooFinance to a module-level singleton (getYf()) so the crumb/cookie is fetched once, not per call. Matches YFinanceAdapter pattern. - Add suppressNotices for yahooSurvey to reduce log noise. - Add accession column to institution_filings (schema.sql + migration in client.ts) and thread it through both secDataFetcher.ts INSERT paths (13D/G and 13F-HR). Fixes recurring `no such column: accession` error in new13daProducer alert tick. - new13daProducer.ts needs no changes - its SELECT accession query now works. --- app/server/src/db/client.ts | 1 + app/server/src/db/schema.sql | 3 +- .../src/services/analystRatingsService.ts | 28 +++++++++++++------ app/server/src/services/secDataFetcher.ts | 12 ++++---- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/app/server/src/db/client.ts b/app/server/src/db/client.ts index 4b4a4dd..21c58cf 100644 --- a/app/server/src/db/client.ts +++ b/app/server/src/db/client.ts @@ -57,6 +57,7 @@ function runMigrations(db: DatabaseSync): void { `ALTER TABLE adapter_queue ADD COLUMN error TEXT`, `ALTER TABLE adapter_queue ADD COLUMN scheduled_for TEXT`, `ALTER TABLE insider_transactions ADD COLUMN accession TEXT`, + `ALTER TABLE institution_filings ADD COLUMN accession TEXT`, `ALTER TABLE x_credentials ADD COLUMN fred_api_key_enc TEXT`, `ALTER TABLE users ADD COLUMN status TEXT NOT NULL DEFAULT 'active'`, // Onboarding / risk posture fields added after early installs (CREATE IF NOT EXISTS does not alter). diff --git a/app/server/src/db/schema.sql b/app/server/src/db/schema.sql index 492d268..5e204ce 100644 --- a/app/server/src/db/schema.sql +++ b/app/server/src/db/schema.sql @@ -113,7 +113,8 @@ CREATE TABLE IF NOT EXISTS institution_filings ( shares REAL, -- shares reported value_usd REAL, reported_quarter TEXT NOT NULL, -- calendar quarter positions are "as of" - filed_at TEXT NOT NULL, + filed_at TEXT NOT NULL, -- when the filing was submitted to SEC + accession TEXT, -- SEC accession number (enables 13D/G dedup) fetched_at TEXT NOT NULL, put_call TEXT -- Put|Call|null for common stock (13F only) ); diff --git a/app/server/src/services/analystRatingsService.ts b/app/server/src/services/analystRatingsService.ts index 21433d0..4b0e30c 100644 --- a/app/server/src/services/analystRatingsService.ts +++ b/app/server/src/services/analystRatingsService.ts @@ -21,11 +21,23 @@ export interface AnalystConsensus { const RATINGS_TTL_MS = 86_400_000; // 24h const MIN_FETCH_INTERVAL_MS = 5_000; // 5s between fetches per symbol const RATE_LIMIT_BACKOFF_MS = 120_000; // 2min backoff after rate-limit +const YAHOO_TIMEOUT_MS = 6_000; // 6s — shorter than frontend 8s TRPC timeout // In-memory per-symbol throttle to prevent stampeding in the absence of queue integration const lastFetchBySymbol = new Map(); const rateLimitUntil = new Map(); +// Singleton YahooFinance instance (crumb/cookie is fetched once, reused across calls). +let yfInstance: { quoteSummary(...args: unknown[]): Promise> } | null = null; + +async function getYf(): Promise<{ quoteSummary(...args: unknown[]): Promise> }> { + if (!yfInstance) { + const mod = await import('yahoo-finance2'); + yfInstance = new mod.default({ suppressNotices: ['yahooSurvey'] }) as typeof yfInstance; + } + return yfInstance; +} + async function fetchFromYahoo(symbol: string): Promise<{ ratings: AnalystRating[]; consensus: AnalystConsensus } | { error: string }> { const now = Date.now(); @@ -41,17 +53,17 @@ async function fetchFromYahoo(symbol: string): Promise<{ ratings: AnalystRating[ } lastFetchBySymbol.set(symbol, now); - let yfMod: { default: new () => { quoteSummary(...args: unknown[]): Promise> } }; - try { - yfMod = await import('yahoo-finance2'); - } catch { - return { error: 'yahoo-finance2 not available' }; - } - const yf = new yfMod.default(); + const yf = await getYf(); let raw: Record; + let timeoutId: ReturnType | null = null; try { - raw = await yf.quoteSummary(symbol, { modules: ['upgradeDowngradeHistory', 'recommendationTrend'] }, { validateResult: false }); + const yfPromise = yf.quoteSummary(symbol, { modules: ['upgradeDowngradeHistory', 'recommendationTrend'] }, { validateResult: false }); + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => reject(new Error('Yahoo Finance timed out')), YAHOO_TIMEOUT_MS); + }); + raw = await Promise.race([yfPromise, timeoutPromise]) as Record; } catch (e) { + if (timeoutId) { clearTimeout(timeoutId); timeoutId = null; } const msg = (e as Error).message; if (/too many requests|rate[- ]?limit|429|edge:\s*too many/i.test(msg)) { rateLimitUntil.set(symbol, Date.now() + RATE_LIMIT_BACKOFF_MS); diff --git a/app/server/src/services/secDataFetcher.ts b/app/server/src/services/secDataFetcher.ts index b88f398..7183ee4 100644 --- a/app/server/src/services/secDataFetcher.ts +++ b/app/server/src/services/secDataFetcher.ts @@ -402,13 +402,13 @@ async function fetchAndStoreSCFilings( const insert = db.prepare(` INSERT OR REPLACE INTO institution_filings - (filer_cik, filer_name, filer_sic, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (filer_cik, filer_name, filer_sic, symbol, form, shares, value_usd, reported_quarter, filed_at, accession, fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); insert.run( parsed.filerCik, parsed.filerName, filerSic, symbol, - parsed.formType, parsed.shares, parsed.value, quarter, filedAt, now, + parsed.formType, parsed.shares, parsed.value, quarter, filedAt, filing.accession, now, ); storedCount++; } catch { @@ -469,13 +469,13 @@ async function storeMatchedHoldings( const now = new Date().toISOString(); const insert = db.prepare(` INSERT OR REPLACE INTO institution_filings - (filer_cik, filer_name, filer_sic, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at, put_call) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (filer_cik, filer_name, filer_sic, symbol, form, shares, value_usd, reported_quarter, filed_at, accession, fetched_at, put_call) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); let n = 0; for (const h of matching) { - insert.run(filing.cik, filing.filerName, filerSic, symbol, formType, h.sshPrnamt, h.value, quarter, filing.filedAt, now, h.putCall || null); + insert.run(filing.cik, filing.filerName, filerSic, symbol, formType, h.sshPrnamt, h.value, quarter, filing.filedAt, filing.accession, now, h.putCall || null); n++; } return n;