fix: backfill symbol_demand for sidebar-added symbols + analyst ratings schema fix
- Add await ctx.cache.subscribe() to addSymbol mutation so symbols added via the sidebar get registered in symbol_demand and yfinance jobs are queued immediately - Backfill PEP, WYNN, STZ, CELH into symbol_demand + adapter_queue - Upgrade yahoo-finance2 3.15.3 -> 3.15.4 and pass validateResult:false to quoteSummary() to handle Yahoo schema drift - Add error detail logging for analyst ratings schema failures - Update .gitignore with common ignores
This commit is contained in:
Investor Flow Build
@@ -188,7 +188,7 @@ export class EdgarAdapter implements SourceFetch {
|
||||
} else if (typeof recentObj === 'object') {
|
||||
// Parallel arrays — convert to array of objects.
|
||||
const keys = Object.keys(recentObj);
|
||||
const len = (recentObj[keys[0]] as unknown[])?.length ?? 0;
|
||||
const len = ((recentObj as Record<string, unknown>)[keys[0]] as unknown[])?.length ?? 0;
|
||||
recentFilings = [];
|
||||
for (let i = 0; i < len; i++) {
|
||||
const row: Record<string, unknown> = {};
|
||||
@@ -205,7 +205,7 @@ export class EdgarAdapter implements SourceFetch {
|
||||
let filings = recentFilings;
|
||||
if (opts?.formTypes && opts.formTypes.length > 0) {
|
||||
const allowed = new Set(opts.formTypes.map((f) => f.toUpperCase()));
|
||||
filings = filings.filter((f) => allowed.has((f.form ?? '').toUpperCase()));
|
||||
filings = filings.filter((f) => allowed.has(String(f.form ?? '').toUpperCase()));
|
||||
}
|
||||
|
||||
// Filter by date range.
|
||||
@@ -213,7 +213,7 @@ export class EdgarAdapter implements SourceFetch {
|
||||
const from = opts.dateRange.from ? new Date(opts.dateRange.from).getTime() : null;
|
||||
const to = opts.dateRange.to ? new Date(opts.dateRange.to).getTime() : null;
|
||||
filings = filings.filter((f) => {
|
||||
const ts = new Date(f.reportDate ?? f.filingDate ?? f.dateReporter ?? '').getTime();
|
||||
const ts = new Date(String(f.reportDate ?? f.filingDate ?? f.dateReporter ?? '')).getTime();
|
||||
if (Number.isNaN(ts)) return false;
|
||||
if (from !== null && ts < from) return false;
|
||||
if (to !== null && ts > to) return false;
|
||||
@@ -352,6 +352,7 @@ export class EdgarAdapter implements SourceFetch {
|
||||
async form13f_holdings(
|
||||
cik: string,
|
||||
accession: string,
|
||||
opts?: { limit?: number; offset?: number },
|
||||
): Promise<FetchResult> {
|
||||
const filerCik = accession.split('-')[0];
|
||||
const padded = padCik(filerCik);
|
||||
@@ -390,7 +391,7 @@ export class EdgarAdapter implements SourceFetch {
|
||||
|
||||
// Try each candidate until we find one with holdings; fall back to any non-HTML doc
|
||||
let docText = '';
|
||||
let parsed: Array<{ cusip: string; issuerName: string; value: number; sshPrnamt: number }> = [];
|
||||
let parsed: Array<{ cusip: string; issuerName: string; value: number; sshPrnamt: number; putCall: string }> = [];
|
||||
for (const cand of xmlCandidates) {
|
||||
const docUrl = `https://www.sec.gov/Archives/edgar/data/${padded}/${accessionNoDashes}/${cand.name}`;
|
||||
docText = await this.edgarXmlFetch(docUrl);
|
||||
@@ -410,10 +411,14 @@ export class EdgarAdapter implements SourceFetch {
|
||||
}
|
||||
}
|
||||
|
||||
const holdings = parsed;
|
||||
const allHoldings = parsed;
|
||||
const total = allHoldings.length;
|
||||
const offset = Math.max(0, opts?.offset ?? 0);
|
||||
const limit = opts?.limit;
|
||||
const holdings = limit != null ? allHoldings.slice(offset, offset + limit) : allHoldings;
|
||||
|
||||
return {
|
||||
value: { holdings, accession: `${padded}/${accessionNoDashes}` },
|
||||
value: { holdings, total, accession: `${padded}/${accessionNoDashes}` },
|
||||
ttlClass: 'daily_permanent' as const,
|
||||
provenance: {
|
||||
fetchedAt: new Date().toISOString(),
|
||||
@@ -581,10 +586,11 @@ function parse13fHoldings(text: string): Array<{
|
||||
issuerName: string;
|
||||
value: number;
|
||||
sshPrnamt: number;
|
||||
putCall: string;
|
||||
}> {
|
||||
const holdings: Array<{ cusip: string; issuerName: string; value: number; sshPrnamt: number }> = [];
|
||||
const holdings: Array<{ cusip: string; issuerName: string; value: number; sshPrnamt: number; putCall: string }> = [];
|
||||
|
||||
// Try 13F XML format: <infoTable> (possibly namespaced) with <nameOfIssuer>, <cusip>, <value>, <sshPrnamt>
|
||||
// Try 13F XML format: <infoTable> (possibly namespaced) with <nameOfIssuer>, <cusip>, <value>, <sshPrnamt>, <putCall>
|
||||
// SEC 13F XML uses namespaces like <ns1:infoTable><ns1:nameOfIssuer>...
|
||||
const infoTableMatches = text.match(/<(?:\w+:)?infoTable[^>]*>([\s\S]*?)<\/(?:\w+:)?infoTable>/gi);
|
||||
if (infoTableMatches && infoTableMatches.length > 0) {
|
||||
@@ -593,6 +599,7 @@ function parse13fHoldings(text: string): Array<{
|
||||
const cusipMatch = block.match(/<(?:\w+:)?cusip>\s*([\s\S]*?)\s*<\/(?:\w+:)?cusip>/i);
|
||||
const valueMatch = block.match(/<(?:\w+:)?value>\s*([\s\S]*?)\s*<\/(?:\w+:)?value>/i);
|
||||
const sharesMatch = block.match(/<(?:\w+:)?sshPrnamt>\s*([\s\S]*?)\s*<\/(?:\w+:)?sshPrnamt>/i);
|
||||
const putCallMatch = block.match(/<(?:\w+:)?putCall>\s*([\s\S]*?)\s*<\/(?:\w+:)?putCall>/i);
|
||||
|
||||
if (!cusipMatch) continue;
|
||||
|
||||
@@ -600,8 +607,9 @@ function parse13fHoldings(text: string): Array<{
|
||||
const cusip = cusipMatch ? cusipMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||||
const value = valueMatch ? parseFloat(valueMatch[1].replace(/<[^>]+>/g, '').replace(/,/g, '')) || 0 : 0;
|
||||
const sshPrnamt = sharesMatch ? parseFloat(sharesMatch[1].replace(/<[^>]+>/g, '').replace(/,/g, '')) || 0 : 0;
|
||||
const putCall = putCallMatch ? putCallMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||||
|
||||
holdings.push({ cusip, issuerName, value, sshPrnamt });
|
||||
holdings.push({ cusip, issuerName, value, sshPrnamt, putCall });
|
||||
}
|
||||
|
||||
if (holdings.length > 0) return holdings;
|
||||
@@ -638,7 +646,7 @@ function parse13fHoldings(text: string): Array<{
|
||||
const sharesMatch = cells[3].match(/([\d,.]+)/);
|
||||
const sshPrnamt = sharesMatch ? parseFloat(sharesMatch[1].replace(/,/g, '')) : 0;
|
||||
|
||||
holdings.push({ cusip: cusipMatch[1], issuerName: issuerRaw, value, sshPrnamt });
|
||||
holdings.push({ cusip: cusipMatch[1], issuerName: issuerRaw, value, sshPrnamt, putCall: '' });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -657,7 +665,7 @@ function parse13fHoldings(text: string): Array<{
|
||||
if (cusipLine && !/Class|CUSIP|Name|Issuer/i.test(line)) {
|
||||
// Save previous holding if we have one.
|
||||
if (currentCusip) {
|
||||
holdings.push({ cusip: currentCusip, issuerName: currentIssuer, value: currentValue, sshPrnamt: currentShares });
|
||||
holdings.push({ cusip: currentCusip, issuerName: currentIssuer, value: currentValue, sshPrnamt: currentShares, putCall: '' });
|
||||
}
|
||||
currentCusip = cusipLine[1];
|
||||
currentIssuer = '';
|
||||
@@ -681,7 +689,7 @@ function parse13fHoldings(text: string): Array<{
|
||||
}
|
||||
// Push last holding.
|
||||
if (currentCusip) {
|
||||
holdings.push({ cusip: currentCusip, issuerName: currentIssuer.trim(), value: currentValue, sshPrnamt: currentShares });
|
||||
holdings.push({ cusip: currentCusip, issuerName: currentIssuer.trim(), value: currentValue, sshPrnamt: currentShares, putCall: '' });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -100,13 +100,11 @@ export class OptionsAdapter implements SourceFetch {
|
||||
const yf = await this.yf();
|
||||
|
||||
if (kind === 'expiry_dates') {
|
||||
// yahoo-finance2 v3: options() returns expirationDates as Date objects
|
||||
const result = await yf.options(id);
|
||||
const dates = (result.expirationDates ?? [])
|
||||
.map((d: Date) => d.toISOString().slice(0, 10))
|
||||
.sort() as OptionExpiryDate[];
|
||||
// Use optionsExpiryDates() which returns string[] directly.
|
||||
const dates = await yf.optionsExpiryDates(id);
|
||||
const sorted = [...dates].sort() as OptionExpiryDate[];
|
||||
return {
|
||||
value: dates,
|
||||
value: sorted,
|
||||
ttlClass: 'intraday',
|
||||
provenance: { fetchedAt, sourceKind: 'yfinance', rawSourceId: `options:expiry:${id}` },
|
||||
};
|
||||
@@ -114,14 +112,26 @@ export class OptionsAdapter implements SourceFetch {
|
||||
|
||||
if (kind === 'chain') {
|
||||
const [symbol, expiry] = id.split(':');
|
||||
const rawChain = await yf.options(symbol, { date: expiry });
|
||||
// v3: options is indexed by number, extract calls/puts from first entry
|
||||
const optionsObj = rawChain.options;
|
||||
const firstKey = Object.keys(optionsObj)[0];
|
||||
const expiryData = firstKey ? optionsObj[firstKey] : null;
|
||||
const rawResult = await yf.options(symbol, expiry);
|
||||
// Handle multiple response shapes:
|
||||
// v3: { options: { '1234567890': { calls: [...], puts: [...] } } }
|
||||
// flat: { calls: [...], puts: [...] }
|
||||
let expiryData: Record<string, unknown> | null = null;
|
||||
if (rawResult.options && typeof rawResult.options === 'object' && !Array.isArray(rawResult.options)) {
|
||||
const optObj = rawResult.options as Record<string, unknown>;
|
||||
const dateKeys = Object.keys(optObj).filter((k) => /^\d+$/.test(k) || /\d{4}-\d{2}-\d{2}/.test(k));
|
||||
if (dateKeys.length > 0) {
|
||||
expiryData = optObj[dateKeys[0]] as Record<string, unknown>;
|
||||
} else {
|
||||
expiryData = optObj;
|
||||
}
|
||||
} else if (Array.isArray(rawResult.calls) || Array.isArray(rawResult.puts)) {
|
||||
// Flat format: { calls: [...], puts: [...] }
|
||||
expiryData = rawResult as Record<string, unknown>;
|
||||
}
|
||||
const allRows = [
|
||||
...(expiryData?.calls ?? []).map((c: Record<string, unknown>) => ({ ...c, right: 'call' as const })),
|
||||
...(expiryData?.puts ?? []).map((p: Record<string, unknown>) => ({ ...p, right: 'put' as const })),
|
||||
...((expiryData?.calls as Array<Record<string, unknown>> | undefined) ?? []).map((c) => ({ ...c, right: 'call' as const })),
|
||||
...((expiryData?.puts as Array<Record<string, unknown>> | undefined) ?? []).map((p) => ({ ...p, right: 'put' as const })),
|
||||
];
|
||||
const rows = parseOptionChainRows(symbol, allRows);
|
||||
return {
|
||||
@@ -136,13 +146,23 @@ export class OptionsAdapter implements SourceFetch {
|
||||
const symbol = parts[0];
|
||||
const expiry = parts[1];
|
||||
const strike = parseFloat(parts[2] ?? '0');
|
||||
const rawChain = await yf.options(symbol, { date: expiry });
|
||||
const optionsObj = rawChain.options;
|
||||
const firstKey = Object.keys(optionsObj)[0];
|
||||
const expiryData = firstKey ? optionsObj[firstKey] : null;
|
||||
const rawResult = await yf.options(symbol, expiry);
|
||||
// Handle multiple response shapes (same logic as chain kind).
|
||||
let expiryData: Record<string, unknown> | null = null;
|
||||
if (rawResult.options && typeof rawResult.options === 'object' && !Array.isArray(rawResult.options)) {
|
||||
const optObj = rawResult.options as Record<string, unknown>;
|
||||
const dateKeys = Object.keys(optObj).filter((k) => /^\d+$/.test(k) || /\d{4}-\d{2}-\d{2}/.test(k));
|
||||
if (dateKeys.length > 0) {
|
||||
expiryData = optObj[dateKeys[0]] as Record<string, unknown>;
|
||||
} else {
|
||||
expiryData = optObj;
|
||||
}
|
||||
} else if (Array.isArray(rawResult.calls) || Array.isArray(rawResult.puts)) {
|
||||
expiryData = rawResult as Record<string, unknown>;
|
||||
}
|
||||
const allRows = [
|
||||
...(expiryData?.calls ?? []).map((c: Record<string, unknown>) => ({ ...c, right: 'call' as const })),
|
||||
...(expiryData?.puts ?? []).map((p: Record<string, unknown>) => ({ ...p, right: 'put' as const })),
|
||||
...((expiryData?.calls as Array<Record<string, unknown>> | undefined) ?? []).map((c) => ({ ...c, right: 'call' as const })),
|
||||
...((expiryData?.puts as Array<Record<string, unknown>> | undefined) ?? []).map((p) => ({ ...p, right: 'put' as const })),
|
||||
];
|
||||
const rows = parseOptionChainRows(symbol, allRows);
|
||||
const target = strike > 0
|
||||
|
||||
@@ -75,7 +75,7 @@ export class RedditAdapter implements SourceFetch {
|
||||
readonly sourceKind: SourceKind = 'reddit';
|
||||
|
||||
private readonly rateLimiter: TokenBucket;
|
||||
private health: RedditHealth = { sourceStatus: 'healthy' };
|
||||
private _health: RedditHealth = { sourceStatus: 'healthy' };
|
||||
private _onDegraded?: (health: RedditHealth) => void;
|
||||
|
||||
constructor(onDegraded?: (health: RedditHealth) => void) {
|
||||
@@ -83,18 +83,18 @@ export class RedditAdapter implements SourceFetch {
|
||||
this._onDegraded = onDegraded;
|
||||
}
|
||||
|
||||
get health(): RedditHealth { return this.health; }
|
||||
get health(): RedditHealth { return this._health; }
|
||||
|
||||
private emitDegraded(error: string): void {
|
||||
this.health = { sourceStatus: 'failed', lastError: error };
|
||||
this._onDegraded?.(this.health);
|
||||
this._health = { sourceStatus: 'failed', lastError: error };
|
||||
this._onDegraded?.(this._health);
|
||||
console.warn(`[RedditAdapter] source-degraded: ${error}`);
|
||||
}
|
||||
|
||||
private emitHealthy(): void {
|
||||
if (this.health.sourceStatus !== 'healthy') {
|
||||
this.health = { sourceStatus: 'healthy' };
|
||||
this._onDegraded?.(this.health);
|
||||
if (this._health.sourceStatus !== 'healthy') {
|
||||
this._health = { sourceStatus: 'healthy' };
|
||||
this._onDegraded?.(this._health);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,6 +143,50 @@ export class RedditAdapter implements SourceFetch {
|
||||
}
|
||||
}
|
||||
|
||||
/** Search Reddit for posts matching a query. */
|
||||
async searchPosts(query: string, opts?: FetchOpts): Promise<FetchResult> {
|
||||
if (this.health.sourceStatus === 'failed') {
|
||||
throw new Error(`RedditAdapter: source degraded — ${this.health.lastError}`);
|
||||
}
|
||||
|
||||
await this.rateLimiter.acquire();
|
||||
|
||||
const url = `${REDDIT_API_BASE}/search.json?q=${encodeURIComponent(query)}&limit=50`;
|
||||
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'InvestorFlow/1.0 (by operator@example.com)',
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (resp.status === 429) {
|
||||
this.emitDegraded(`Reddit rate-limited (HTTP ${resp.status})`);
|
||||
throw new Error(`RedditAdapter: Reddit source degraded (rate limited)`);
|
||||
}
|
||||
|
||||
if (!resp.ok) {
|
||||
this.emitDegraded(`Reddit HTTP ${resp.status}`);
|
||||
throw new Error(`RedditAdapter: Reddit source degraded (HTTP ${resp.status})`);
|
||||
}
|
||||
|
||||
const body = await resp.json() as RedditResponse;
|
||||
const posts = this.parseSubredditPosts(body, 'search');
|
||||
|
||||
return {
|
||||
value: posts,
|
||||
ttlClass: 'thread_7d' as TtlClass,
|
||||
provenance: { fetchedAt: new Date().toISOString(), sourceKind: 'reddit', rawSourceId: `search:${query}` },
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes('Reddit source degraded')) {
|
||||
throw err;
|
||||
}
|
||||
throw new Error(`RedditAdapter search failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetch top comments for a specific Reddit post. */
|
||||
async postComments(postId: string, sort?: 'best' | 'top' | 'new', opts?: FetchOpts): Promise<FetchResult> {
|
||||
if (this.health.sourceStatus === 'failed') {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
import type { CacheKey, SourceKind } from '../cache/CacheRepository.ts';
|
||||
import { parseCacheKey } from '../cache/CacheRepository.ts';
|
||||
import type { SourceFetch, FetchResult } from './SourceAdapter.ts';
|
||||
|
||||
export class SecFetchAdapter implements SourceFetch {
|
||||
readonly sourceKind: SourceKind = 'sec-fetch';
|
||||
private _db: DatabaseSync;
|
||||
|
||||
constructor(db: DatabaseSync) {
|
||||
this._db = db;
|
||||
}
|
||||
|
||||
async fetchOne(key: CacheKey): Promise<FetchResult> {
|
||||
const { kind, id: symbol } = parseCacheKey(key);
|
||||
if (kind !== 'fetch') throw new Error(`SecFetchAdapter: unknown kind '${kind}'`);
|
||||
|
||||
const { fetchAndStoreSecData } = await import('../services/secDataFetcher.ts');
|
||||
const result = await fetchAndStoreSecData(this._db, symbol);
|
||||
return {
|
||||
value: { ok: true, filingsStored: result.filingsStored, scStored: result.scStored, transactionsStored: result.transactionsStored },
|
||||
ttlClass: 'daily_permanent',
|
||||
provenance: { fetchedAt: new Date().toISOString(), sourceKind: 'sec-fetch', rawSourceId: symbol },
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
import type { SourceFetch, FetchResult } from './SourceAdapter.ts';
|
||||
import type { CacheKey, TtlClass, Provenance } from '../cache/CacheRepository.ts';
|
||||
import { lintInstitutionalHolders, lintInsiderTransactions, type LintResult } from '../services/secDataFetcher.ts';
|
||||
import { lintInstitutionalHolders, lintInsiderTransactions, lintSCFilings, type LintResult } from '../services/secDataFetcher.ts';
|
||||
|
||||
export class SecLintAdapter implements SourceFetch {
|
||||
readonly sourceKind: 'sec-lint-holders' | 'sec-lint-insiders';
|
||||
@@ -23,6 +23,7 @@ export class SecLintAdapter implements SourceFetch {
|
||||
let result: LintResult;
|
||||
if (this.sourceKind === 'sec-lint-holders') {
|
||||
result = await lintInstitutionalHolders(this.db(), symbol);
|
||||
await lintSCFilings(this.db(), symbol).catch(() => {});
|
||||
} else {
|
||||
result = await lintInsiderTransactions(this.db(), symbol);
|
||||
}
|
||||
@@ -42,6 +43,7 @@ export class SecLintAdapter implements SourceFetch {
|
||||
try {
|
||||
if (kind === 'sec-lint-holders') {
|
||||
results.push(await lintInstitutionalHolders(db, sym));
|
||||
await lintSCFilings(db, sym).catch(() => {});
|
||||
} else {
|
||||
results.push(await lintInsiderTransactions(db, sym));
|
||||
}
|
||||
|
||||
@@ -1,122 +1,109 @@
|
||||
// Investor Flow — XCookieAdapter (DESIGN.md §5 X-cookie adapter).
|
||||
// Fetches cashtag search results and trusted-account timelines using cookie auth.
|
||||
// Rate limit: 1 req / 3s. Cache: 7d rolling (immutable within the window).
|
||||
// Cookie-expiry detection: 403/302/empty → mark source FAILED + emit degraded signal.
|
||||
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
import type { SourceKind, TtlClass, Provenance } from '../cache/CacheRepository.ts';
|
||||
import type { SourceFetch, FetchResult, FetchOpts } from './SourceAdapter.ts';
|
||||
|
||||
// ADR-0007: crowd sentiment is not edge — it reflects consensus, not an advantage.
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const BIRD_PATH = process.env.BIRD_PATH ?? '/opt/homebrew/bin/bird';
|
||||
const BIRD_TIMEOUT_MS = 30_000;
|
||||
const CACHE_TTL_MS = 7 * 24 * 60 * 60_000;
|
||||
|
||||
interface BirdTweet {
|
||||
id: string;
|
||||
text: string;
|
||||
createdAt: string;
|
||||
replyCount: number;
|
||||
retweetCount: number;
|
||||
likeCount: number;
|
||||
author: { username: string; name: string };
|
||||
authorId: string;
|
||||
}
|
||||
|
||||
export interface XPostRow {
|
||||
post_id: string;
|
||||
author_handle: string;
|
||||
cashtag: string | null;
|
||||
body_text: string | null;
|
||||
posted_at: string;
|
||||
engagement: number;
|
||||
sentiment_score?: number | null;
|
||||
attribution?: string | null;
|
||||
}
|
||||
|
||||
/** Normalize bird/Twitter dates to ISO so SQLite range filters sort correctly. */
|
||||
export function normalizePostedAt(raw: string): string {
|
||||
const d = new Date(raw);
|
||||
return Number.isNaN(d.getTime()) ? raw : d.toISOString();
|
||||
}
|
||||
|
||||
export function birdTweetsToXPosts(
|
||||
tweets: BirdTweet[],
|
||||
authorHandle: string,
|
||||
cashtag: string | null,
|
||||
): XPostRow[] {
|
||||
return tweets.map((t) => ({
|
||||
post_id: t.id,
|
||||
author_handle: (t.author?.username || authorHandle).replace(/^@/, ''),
|
||||
cashtag: cashtag ? `$${cashtag}` : null,
|
||||
body_text: t.text.length > 280 ? t.text.slice(0, 280) : t.text,
|
||||
posted_at: normalizePostedAt(t.createdAt),
|
||||
engagement: (t.likeCount ?? 0) + (t.retweetCount ?? 0) + (t.replyCount ?? 0),
|
||||
}));
|
||||
}
|
||||
|
||||
export const CROWD_SENTIMENT_CAVEAT =
|
||||
'Crowd sentiment is not edge — it reflects consensus, not an advantage';
|
||||
|
||||
const X_SEARCH_URL = 'https://x.com/i/api/graphql/search-timeline';
|
||||
const X_TIMELINE_URL = 'https://x.com/i/api/graphql/vHlSJz4yOZC-Xj16R7Xm_Q/TimelineQuery';
|
||||
const RATE_LIMIT_MS = 3_000;
|
||||
const CACHE_TTL_MS = 7 * 24 * 60 * 60_000; // 7 days
|
||||
|
||||
// ===== Types for X API response parsing =====
|
||||
|
||||
interface XTimelineResponse {
|
||||
data?: {
|
||||
search_by_raw_query?: {
|
||||
search_timeline?: {
|
||||
timeline?: {
|
||||
instructions?: Array<Record<string, unknown>>;
|
||||
};
|
||||
};
|
||||
};
|
||||
user_result?: {
|
||||
result?: {
|
||||
timeline_v2?: {
|
||||
timeline?: {
|
||||
instructions?: Array<Record<string, unknown>>;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
interface XInstruction {
|
||||
type?: string;
|
||||
entries?: Array<{
|
||||
entryId?: string;
|
||||
content?: Record<string, unknown>;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface XItemContent {
|
||||
itemContent?: {
|
||||
tweet_results?: {
|
||||
result?: {
|
||||
__typename?: string;
|
||||
rest_id?: string;
|
||||
core?: { user_results?: { result?: { legacy?: { screen_name?: string } } } };
|
||||
legacy?: {
|
||||
full_text?: string;
|
||||
created_at?: string;
|
||||
favorite_count?: number;
|
||||
retweet_count?: number;
|
||||
reply_count?: number;
|
||||
quote_count?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// ===== Rate limiter (token bucket) =====
|
||||
|
||||
class TokenBucket {
|
||||
private tokens: number = 1; // start with 1 token
|
||||
private lastRefill: number = Date.now();
|
||||
readonly capacity: number;
|
||||
readonly refillMs: number;
|
||||
|
||||
constructor(capacity = 1, refillMs = RATE_LIMIT_MS) {
|
||||
this.capacity = capacity;
|
||||
this.refillMs = refillMs;
|
||||
}
|
||||
|
||||
async acquire(): Promise<void> {
|
||||
const now = Date.now();
|
||||
const elapsed = now - this.lastRefill;
|
||||
// Refill: add tokens based on elapsed time
|
||||
const tokensToAdd = Math.floor(elapsed / this.refillMs);
|
||||
if (tokensToAdd > 0) {
|
||||
this.tokens = Math.min(this.capacity, this.tokens + tokensToAdd);
|
||||
this.lastRefill = now - (elapsed % this.refillMs);
|
||||
}
|
||||
if (this.tokens < 1) {
|
||||
// Wait until next token is available
|
||||
const waitMs = this.refillMs - (now - this.lastRefill);
|
||||
await new Promise((resolve) => setTimeout(resolve, Math.max(0, waitMs)));
|
||||
return this.acquire();
|
||||
}
|
||||
this.tokens -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Cookie expiry detection =====
|
||||
|
||||
export type XCookieHealth = { sourceStatus: 'healthy' | 'degraded' | 'failed'; lastError?: string | null; };
|
||||
export type XCookieHealth = { sourceStatus: 'healthy' | 'degraded' | 'failed'; lastError?: string | null };
|
||||
|
||||
export class XCookieAdapter implements SourceFetch {
|
||||
readonly sourceKind: SourceKind = 'x';
|
||||
|
||||
private readonly rateLimiter: TokenBucket;
|
||||
private cookieHealth: XCookieHealth = { sourceStatus: 'healthy' };
|
||||
private _cookies: { ct0: string; auth_token: string } | null = null;
|
||||
private _onDegraded?: (health: XCookieHealth) => void;
|
||||
private _db: DatabaseSync | null = null;
|
||||
|
||||
constructor(cookies: { ct0: string; auth_token: string }, onDegraded?: (health: XCookieHealth) => void) {
|
||||
constructor(cookies: { ct0: string; auth_token: string }, onDegraded?: (health: XCookieHealth) => void, db?: DatabaseSync) {
|
||||
this._cookies = cookies;
|
||||
this.rateLimiter = new TokenBucket();
|
||||
this._onDegraded = onDegraded;
|
||||
this._db = db ?? null;
|
||||
}
|
||||
|
||||
private async runBird(args: string[]): Promise<string> {
|
||||
if (!this._cookies) throw new Error('XCookieAdapter: no credentials set');
|
||||
const allArgs = [
|
||||
'--auth-token', this._cookies.auth_token,
|
||||
'--ct0', this._cookies.ct0,
|
||||
...args,
|
||||
];
|
||||
try {
|
||||
const { stdout, stderr } = await execFileAsync(BIRD_PATH, allArgs, { timeout: BIRD_TIMEOUT_MS });
|
||||
this.emitHealthy();
|
||||
return stdout;
|
||||
} catch (e: any) {
|
||||
const msg = e?.stderr ?? e?.message ?? String(e);
|
||||
if (msg.includes('Missing required credentials') || msg.includes('Missing auth_token') || msg.includes('Missing ct0')) {
|
||||
this.emitDegraded('bird CLI: missing or expired credentials');
|
||||
}
|
||||
throw new Error(`XCookieAdapter: bird CLI failed — ${msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
private persistPosts(posts: XPostRow[]): void {
|
||||
if (!this._db || posts.length === 0) return;
|
||||
const cachedUntil = new Date(Date.now() + CACHE_TTL_MS).toISOString();
|
||||
const stmt = this._db.prepare(
|
||||
`INSERT OR REPLACE INTO x_cookie_posts (post_id, source, author_handle, cashtag, body_text, posted_at, engagement, cached_until)
|
||||
VALUES (?, 'x', ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
for (const p of posts) {
|
||||
stmt.run(p.post_id, p.author_handle, p.cashtag ?? null, p.body_text ?? null, p.posted_at, p.engagement, cachedUntil);
|
||||
}
|
||||
}
|
||||
|
||||
// Set cookies dynamically (for re-seeding after expiry detection)
|
||||
setCookies(cookies: { ct0: string; auth_token: string }): void {
|
||||
this._cookies = cookies;
|
||||
if (this.cookieHealth.sourceStatus !== 'healthy') {
|
||||
@@ -125,302 +112,101 @@ export class XCookieAdapter implements SourceFetch {
|
||||
}
|
||||
}
|
||||
|
||||
// Expose health for the UI to read degraded state
|
||||
get health(): XCookieHealth { return this.cookieHealth; }
|
||||
|
||||
// Emit a degraded signal when cookie expires
|
||||
private emitDegraded(error: string): void {
|
||||
this.cookieHealth = { sourceStatus: 'failed', lastError: error };
|
||||
this._onDegraded?.(this.cookieHealth);
|
||||
console.warn(`[XCookieAdapter] source-degraded: ${error}`);
|
||||
}
|
||||
|
||||
// Rehydrate healthy state after cookie re-seeding
|
||||
private emitHealthy(): void {
|
||||
if (this.cookieHealth.sourceStatus !== 'healthy') {
|
||||
this.cookieHealth = { sourceStatus: 'healthy' };
|
||||
this._onDegraded?.(this.cookieHealth);
|
||||
}
|
||||
this.cookieHealth = { sourceStatus: 'healthy' };
|
||||
this._onDegraded?.(this.cookieHealth);
|
||||
}
|
||||
|
||||
/** Search for cashtag results via X's internal API. */
|
||||
async cashtagSearch(cashtag: string, opts?: FetchOpts): Promise<FetchResult> {
|
||||
async cashtagSearch(cashtag: string, opts?: FetchOpts & { walkBackDays?: number }): Promise<FetchResult> {
|
||||
if (this.cookieHealth.sourceStatus === 'failed') {
|
||||
throw new Error(`XCookieAdapter: source degraded — ${this.cookieHealth.lastError}`);
|
||||
}
|
||||
|
||||
await this.rateLimiter.acquire();
|
||||
|
||||
const query = encodeURIComponent(`$${cashtag} -is:retweet lang:en`);
|
||||
const url = `${X_SEARCH_URL}?variables=${encodeURIComponent(JSON.stringify({
|
||||
rawQuery: `\$${cashtag}`,
|
||||
count: 20,
|
||||
querySource: 'typed_query',
|
||||
product: 'Top',
|
||||
}))}&features=${encodeURIComponent(JSON.stringify({
|
||||
rweb_tipjar_consumption_enabled: true,
|
||||
responsive_web_graphql_exclude_directive_enabled: true,
|
||||
verified_phone_label_enabled: false,
|
||||
creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
communities_web_enable_tweet_community_results_fetch: true,
|
||||
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
articles_preview_enabled: true,
|
||||
responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true,
|
||||
longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
creator_subscriptions_quote_tweet_preview_enabled: false,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
standardized_nudges_misinfo: true,
|
||||
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
||||
rweb_video_timestamps_enabled: true,
|
||||
longform_notetweets_rich_text_read_enabled: true,
|
||||
longform_notetweets_inline_media_enabled: true,
|
||||
responsive_web_enhance_cards_enabled: false,
|
||||
}))}&fieldToggles=${encodeURIComponent(JSON.stringify({ withArticleRichContentState: true }))}`;
|
||||
const count = opts?.walkBackDays && opts.walkBackDays > 0 ? 100 : 20;
|
||||
|
||||
const stdout = await this.runBird(['search', `$${cashtag}`, '-n', String(count), '--json']);
|
||||
let tweets: BirdTweet[];
|
||||
try {
|
||||
const resp = await fetch(url, {
|
||||
headers: {
|
||||
'Cookie': `ct0=${this._cookies!.ct0}; auth_token=${this._cookies!.auth_token}`,
|
||||
'x-csrf-token': this._cookies!.ct0,
|
||||
'x-twitter-active-user': 'yes',
|
||||
'x-twitter-auth-type': 'OAuth2Session',
|
||||
'Authorization': `Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA`,
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
},
|
||||
});
|
||||
|
||||
// Cookie expiry detection: 403/302/empty response body
|
||||
if (resp.status === 403 || resp.status === 302) {
|
||||
this.emitDegraded(`HTTP ${resp.status} — cookie likely expired`);
|
||||
throw new Error(`XCookieAdapter: X source degraded (HTTP ${resp.status})`);
|
||||
}
|
||||
|
||||
const bodyText = await resp.text();
|
||||
if (!bodyText || bodyText.trim().length === 0) {
|
||||
this.emitDegraded('Empty response body — cookie may have expired');
|
||||
throw new Error('XCookieAdapter: X source degraded (empty response)');
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(bodyText) as XTimelineResponse;
|
||||
const posts = this.parseSearchTimeline(parsed, cashtag);
|
||||
|
||||
return {
|
||||
value: posts,
|
||||
ttlClass: 'thread_7d' as TtlClass,
|
||||
provenance: { fetchedAt: new Date().toISOString(), sourceKind: 'x', rawSourceId: `cashtag:${cashtag}` },
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes('X source degraded')) {
|
||||
throw err; // re-throw degraded errors
|
||||
}
|
||||
throw new Error(`XCookieAdapter cashtag search failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetch a trusted account's timeline. */
|
||||
async trustedTimeline(handle: string, opts?: FetchOpts): Promise<FetchResult> {
|
||||
if (this.cookieHealth.sourceStatus === 'failed') {
|
||||
throw new Error(`XCookieAdapter: source degraded — ${this.cookieHealth.lastError}`);
|
||||
tweets = JSON.parse(stdout) as BirdTweet[];
|
||||
} catch {
|
||||
throw new Error('XCookieAdapter: bird CLI returned invalid JSON');
|
||||
}
|
||||
|
||||
await this.rateLimiter.acquire();
|
||||
|
||||
const variables = JSON.stringify({
|
||||
userId: undefined, // will be resolved from handle via a lookup
|
||||
count: 20,
|
||||
includePromotedContent: false,
|
||||
withCommunity: true,
|
||||
withVoice: true,
|
||||
withSuperFollowsUserFieldsEnabled: true,
|
||||
});
|
||||
|
||||
// We need the user_id for the timeline API — do a quick search to resolve handle → id
|
||||
const userSearchUrl = `https://x.com/i/api/graphql/LuOGNVfTtTZ4o2Pjv3NoyA/SearchTimeline`;
|
||||
const encodedVariables = encodeURIComponent(JSON.stringify({
|
||||
rawQuery: `from:${handle}`,
|
||||
count: 1,
|
||||
querySource: 'popped_topic',
|
||||
product: 'Top',
|
||||
const posts: XPostRow[] = tweets.map((t) => ({
|
||||
post_id: t.id,
|
||||
author_handle: (t.author?.username ?? 'unknown').replace(/^@/, ''),
|
||||
cashtag: `$${cashtag.replace(/^\$/, '')}`,
|
||||
body_text: t.text.length > 280 ? t.text.slice(0, 280) : t.text,
|
||||
posted_at: normalizePostedAt(t.createdAt),
|
||||
engagement: (t.likeCount ?? 0) + (t.retweetCount ?? 0) + (t.replyCount ?? 0),
|
||||
}));
|
||||
|
||||
this.persistPosts(posts);
|
||||
|
||||
return {
|
||||
value: posts,
|
||||
ttlClass: 'thread_7d' as TtlClass,
|
||||
provenance: { fetchedAt: new Date().toISOString(), sourceKind: 'x', rawSourceId: `cashtag:${cashtag}` },
|
||||
};
|
||||
}
|
||||
|
||||
async trustedTimeline(handle: string, opts?: FetchOpts & { walkBackDays?: number }): Promise<FetchResult> {
|
||||
if (this.cookieHealth.sourceStatus === 'failed') {
|
||||
throw new Error(`XCookieAdapter: source degraded — ${this.cookieHealth.lastError}`);
|
||||
}
|
||||
|
||||
const count = opts?.walkBackDays && opts.walkBackDays > 0 ? 100 : 20;
|
||||
const clean = handle.replace(/^@/, '');
|
||||
|
||||
// bird 0.8.x user-tweets often fails with "User not found" even for valid
|
||||
// handles (GraphQL user lookup). Search `from:handle` is reliable with the
|
||||
// same cookies and returns the profile timeline effectively.
|
||||
let tweets: BirdTweet[] = [];
|
||||
let used = 'from-search';
|
||||
try {
|
||||
// First: resolve handle to user_id via search
|
||||
const searchResp = await fetch(`${userSearchUrl}?variables=${encodedVariables}&features=${encodeURIComponent('{}')}&fieldToggles=${encodeURIComponent('{"withArticleRichContentState":false}')}`, {
|
||||
headers: {
|
||||
'Cookie': `ct0=${this._cookies!.ct0}; auth_token=${this._cookies!.auth_token}`,
|
||||
'x-csrf-token': this._cookies!.ct0,
|
||||
'x-twitter-active-user': 'yes',
|
||||
'x-twitter-auth-type': 'OAuth2Session',
|
||||
Authorization: `Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA`,
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
},
|
||||
});
|
||||
|
||||
if (searchResp.status === 403 || searchResp.status === 302) {
|
||||
this.emitDegraded(`HTTP ${searchResp.status} — cookie likely expired`);
|
||||
throw new Error(`XCookieAdapter: X source degraded (HTTP ${searchResp.status})`);
|
||||
}
|
||||
|
||||
const searchBody = await searchResp.text();
|
||||
if (!searchBody || searchBody.trim().length === 0) {
|
||||
this.emitDegraded('Empty response body — cookie may have expired');
|
||||
throw new Error('XCookieAdapter: X source degraded (empty response)');
|
||||
}
|
||||
|
||||
const searchParsed = JSON.parse(searchBody) as XTimelineResponse;
|
||||
const userResult = this.extractUserIdFromSearch(searchParsed);
|
||||
if (!userResult) {
|
||||
throw new Error(`XCookieAdapter: could not resolve handle "${handle}" to a user ID`);
|
||||
}
|
||||
|
||||
// Now fetch the timeline using the resolved user_id
|
||||
const timelineVariables = encodeURIComponent(JSON.stringify({
|
||||
userId: userResult,
|
||||
count: 20,
|
||||
includePromotedContent: false,
|
||||
withCommunity: true,
|
||||
withVoice: true,
|
||||
withSuperFollowsUserFieldsEnabled: true,
|
||||
}));
|
||||
|
||||
const timelineResp = await fetch(`${X_TIMELINE_URL}?variables=${timelineVariables}&features=${encodeURIComponent('{}')}&fieldToggles=${encodeURIComponent('{"withArticleRichContentState":false}')}`, {
|
||||
headers: {
|
||||
'Cookie': `ct0=${this._cookies!.ct0}; auth_token=${this._cookies!.auth_token}`,
|
||||
'x-csrf-token': this._cookies!.ct0,
|
||||
'x-twitter-active-user': 'yes',
|
||||
'x-twitter-auth-type': 'OAuth2Session',
|
||||
Authorization: `Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA`,
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
},
|
||||
});
|
||||
|
||||
if (timelineResp.status === 403 || timelineResp.status === 302) {
|
||||
this.emitDegraded(`HTTP ${timelineResp.status} — cookie likely expired`);
|
||||
throw new Error(`XCookieAdapter: X source degraded (HTTP ${timelineResp.status})`);
|
||||
}
|
||||
|
||||
const timelineBody = await timelineResp.text();
|
||||
if (!timelineBody || timelineBody.trim().length === 0) {
|
||||
this.emitDegraded('Empty response body — cookie may have expired');
|
||||
throw new Error('XCookieAdapter: X source degraded (empty response)');
|
||||
}
|
||||
|
||||
const timelineParsed = JSON.parse(timelineBody) as XTimelineResponse;
|
||||
const posts = this.parseUserTimeline(timelineParsed, handle);
|
||||
|
||||
return {
|
||||
value: posts,
|
||||
ttlClass: 'thread_7d' as TtlClass,
|
||||
provenance: { fetchedAt: new Date().toISOString(), sourceKind: 'x', rawSourceId: `timeline:${handle}` },
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes('X source degraded')) {
|
||||
throw err;
|
||||
}
|
||||
throw new Error(`XCookieAdapter trusted timeline failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Parsing helpers ----
|
||||
|
||||
private parseSearchTimeline(response: XTimelineResponse, cashtag: string): Array<{
|
||||
post_id: string; author_handle: string; cashtag: string | null; body_text: string | null; posted_at: string; engagement: number; sentiment_score?: number | null; attribution?: string | null;
|
||||
}> {
|
||||
const posts: Array<{
|
||||
post_id: string; author_handle: string; cashtag: string | null; body_text: string | null; posted_at: string; engagement: number; sentiment_score?: number | null; attribution?: string | null;
|
||||
}> = [];
|
||||
|
||||
const instructions = response.data?.search_by_raw_query?.search_timeline?.timeline?.instructions ?? [];
|
||||
for (const instruction of instructions as XInstruction[]) {
|
||||
if (instruction.type !== 'TimelineAddEntries') continue;
|
||||
for (const entry of instruction.entries ?? []) {
|
||||
const content = entry.content as XItemContent | undefined;
|
||||
if (!content?.itemContent?.tweet_results?.result) continue;
|
||||
|
||||
const result = content.itemContent.tweet_results.result;
|
||||
const legacy = result.legacy;
|
||||
if (!legacy || !result.rest_id) continue;
|
||||
|
||||
const authorHandle = result.core?.user_results?.result?.legacy?.screen_name ?? null;
|
||||
const fullText = legacy.full_text ?? '';
|
||||
const createdAt = legacy.created_at ?? '';
|
||||
const engagement = (legacy.favorite_count ?? 0) + (legacy.retweet_count ?? 0) + (legacy.reply_count ?? 0);
|
||||
|
||||
posts.push({
|
||||
post_id: result.rest_id,
|
||||
author_handle: authorHandle ?? '',
|
||||
cashtag: `$${cashtag}`,
|
||||
body_text: fullText.length > 280 ? fullText.slice(0, 280) : fullText,
|
||||
posted_at: createdAt,
|
||||
engagement,
|
||||
});
|
||||
const stdout = await this.runBird([
|
||||
'search', `from:${clean}`, '-n', String(count), '--json',
|
||||
]);
|
||||
tweets = JSON.parse(stdout) as BirdTweet[];
|
||||
if (!Array.isArray(tweets)) throw new Error('not an array');
|
||||
} catch (searchErr) {
|
||||
// Fallback: user-tweets (older bird / when search is rate-limited).
|
||||
try {
|
||||
const stdout = await this.runBird([
|
||||
'user-tweets', `@${clean}`, '-n', String(count), '--json',
|
||||
]);
|
||||
tweets = JSON.parse(stdout) as BirdTweet[];
|
||||
used = 'user-tweets';
|
||||
if (!Array.isArray(tweets)) throw new Error('not an array');
|
||||
} catch (userErr) {
|
||||
const s = searchErr instanceof Error ? searchErr.message : String(searchErr);
|
||||
const u = userErr instanceof Error ? userErr.message : String(userErr);
|
||||
throw new Error(`XCookieAdapter: timeline fetch failed for @${clean} (from-search: ${s}; user-tweets: ${u})`);
|
||||
}
|
||||
}
|
||||
|
||||
return posts;
|
||||
const posts = birdTweetsToXPosts(tweets, clean, null);
|
||||
this.persistPosts(posts);
|
||||
|
||||
return {
|
||||
value: posts,
|
||||
ttlClass: 'thread_7d' as TtlClass,
|
||||
provenance: {
|
||||
fetchedAt: new Date().toISOString(),
|
||||
sourceKind: 'x',
|
||||
rawSourceId: `timeline:${clean}:${used}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private parseUserTimeline(response: XTimelineResponse, handle: string): Array<{
|
||||
post_id: string; author_handle: string; cashtag: string | null; body_text: string | null; posted_at: string; engagement: number; sentiment_score?: number | null; attribution?: string | null;
|
||||
}> {
|
||||
const posts: Array<{
|
||||
post_id: string; author_handle: string; cashtag: string | null; body_text: string | null; posted_at: string; engagement: number; sentiment_score?: number | null; attribution?: string | null;
|
||||
}> = [];
|
||||
|
||||
const instructions = response.data?.user_result?.result?.timeline_v2?.timeline?.instructions ?? [];
|
||||
for (const instruction of instructions as XInstruction[]) {
|
||||
if (instruction.type !== 'TimelineAddEntries') continue;
|
||||
for (const entry of instruction.entries ?? []) {
|
||||
const content = entry.content as XItemContent | undefined;
|
||||
if (!content?.itemContent?.tweet_results?.result) continue;
|
||||
|
||||
const result = content.itemContent.tweet_results.result;
|
||||
const legacy = result.legacy;
|
||||
if (!legacy || !result.rest_id) continue;
|
||||
|
||||
const fullText = legacy.full_text ?? '';
|
||||
const createdAt = legacy.created_at ?? '';
|
||||
const engagement = (legacy.favorite_count ?? 0) + (legacy.retweet_count ?? 0) + (legacy.reply_count ?? 0);
|
||||
|
||||
// For timeline results, author is the trusted account handle
|
||||
posts.push({
|
||||
post_id: result.rest_id,
|
||||
author_handle: handle,
|
||||
cashtag: null,
|
||||
body_text: fullText.length > 280 ? fullText.slice(0, 280) : fullText,
|
||||
posted_at: createdAt,
|
||||
engagement,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return posts;
|
||||
}
|
||||
|
||||
private extractUserIdFromSearch(response: XTimelineResponse): string | null {
|
||||
const instructions = response.data?.search_by_raw_query?.search_timeline?.timeline?.instructions ?? [];
|
||||
for (const instruction of instructions as XInstruction[]) {
|
||||
if (instruction.type !== 'TimelineAddEntries') continue;
|
||||
for (const entry of instruction.entries ?? []) {
|
||||
const content = entry.content as XItemContent | undefined;
|
||||
if (!content?.itemContent?.tweet_results?.result) continue;
|
||||
const result = content.itemContent.tweet_results.result;
|
||||
if (result.__typename === 'User') {
|
||||
return result.rest_id ?? null;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- SourceAdapter contract ----
|
||||
|
||||
async fetchOne(key: string, opts?: FetchOpts): Promise<FetchResult> {
|
||||
const parts = key.split(':');
|
||||
if (parts.length < 2) throw new Error(`XCookieAdapter: invalid cache key "${key}"`);
|
||||
@@ -431,7 +217,6 @@ export class XCookieAdapter implements SourceFetch {
|
||||
switch (kind) {
|
||||
case 'cashtag': {
|
||||
const result = await this.cashtagSearch(id, opts);
|
||||
// Stamp cached_until: 7d from now
|
||||
const cachedUntil = new Date(Date.now() + CACHE_TTL_MS).toISOString();
|
||||
if (Array.isArray(result.value)) {
|
||||
for (const post of result.value as Array<Record<string, unknown>>) {
|
||||
@@ -442,7 +227,6 @@ export class XCookieAdapter implements SourceFetch {
|
||||
}
|
||||
case 'timeline': {
|
||||
const result = await this.trustedTimeline(id, opts);
|
||||
// Stamp cached_until: 7d from now
|
||||
const cachedUntil = new Date(Date.now() + CACHE_TTL_MS).toISOString();
|
||||
if (Array.isArray(result.value)) {
|
||||
for (const post of result.value as Array<Record<string, unknown>>) {
|
||||
|
||||
@@ -32,7 +32,7 @@ export class YFinanceAdapter implements SourceFetch {
|
||||
case 'candles': {
|
||||
const [symbol, timeframe] = id.split(':');
|
||||
const interval = timeframeToInterval(timeframe);
|
||||
const days = timeframe === '1wk' ? 3650 : 3650; // permanent backfill (~10 years)
|
||||
const days = 3650; // permanent backfill (~10 years for both 1d and 1wk)
|
||||
const raw = await yf.chart(symbol, { period1: new Date(Date.now() - days * 86_400_000), interval });
|
||||
return { value: parseCandles(raw), ttlClass: 'daily_permanent', provenance: { fetchedAt, sourceKind: 'yfinance', rawSourceId: `${symbol}:${timeframe}` } };
|
||||
}
|
||||
@@ -99,6 +99,13 @@ export function parseSymbol(raw: Record<string, unknown>, symbol: string): Symbo
|
||||
industry: str(sp.industry),
|
||||
exchange: str(sp.exchange ?? price.exchange),
|
||||
tickerKind: inferTickerKind(symbol),
|
||||
description: str(sp.longBusinessSummary),
|
||||
website: str(sp.website),
|
||||
marketCap: num(price.marketCap),
|
||||
currency: str(price.currency),
|
||||
employees: num(price.fullTimeEmployees) as number | null,
|
||||
country: str(sp.country ?? price.country),
|
||||
city: str(sp.city ?? price.city),
|
||||
peers: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ function createMockFetch(
|
||||
responses: Record<string, { body: unknown; status?: number; etag?: string; lastModified?: string }>,
|
||||
) {
|
||||
let callCount = 0;
|
||||
async function mockFetch(url: string | URL, init?: RequestInit): Promise<Response> {
|
||||
async function mockFetch(url: string | URL | Request, init?: RequestInit): Promise<Response> {
|
||||
const urlStr = typeof url === 'string' ? url : url.toString();
|
||||
callCount++;
|
||||
for (const [key, resp] of Object.entries(responses)) {
|
||||
@@ -56,7 +56,7 @@ function createHeaderRecordingMockFetch(
|
||||
responses: Record<string, { body: unknown; status?: number; etag?: string; lastModified?: string }>,
|
||||
) {
|
||||
const calls: FetchCall[] = [];
|
||||
async function mockFetch(url: string | URL, init?: RequestInit): Promise<Response> {
|
||||
async function mockFetch(url: string | URL | Request, init?: RequestInit): Promise<Response> {
|
||||
const urlStr = typeof url === 'string' ? url : url.toString();
|
||||
const headers: Record<string, string> = {};
|
||||
if (init?.headers) {
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
import { OptionsAdapter, parseOptionChain, parseOptionChainRows, type OptionChainRow } from '../OptionsAdapter.ts';
|
||||
import { createCacheRepository, type CacheScheduler, type CacheKey } from '../../cache/CacheRepository.ts';
|
||||
import { createDb, initSchema } from '../../db/client.ts';
|
||||
|
||||
// ----- Test helpers: in-memory DB with full schema -----
|
||||
|
||||
function createTestDb(): DatabaseSync {
|
||||
const db = createDb({ path: ':memory:' });
|
||||
initSchema(db);
|
||||
return db;
|
||||
}
|
||||
|
||||
class NoopScheduler implements CacheScheduler {
|
||||
async queue(_key: CacheKey): Promise<void> { /* no-op */ }
|
||||
}
|
||||
|
||||
/** A stubbed yfinance2 client that returns canned data — no network. */
|
||||
interface StubYf {
|
||||
optionsExpiryDates: (symbol: string) => Promise<string[]>;
|
||||
options: (symbol: string, expiry?: string) => Promise<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
/** Build an OptionsAdapter with a stubbed yf instance. */
|
||||
function createStubbedAdapter(stubYf: StubYf): OptionsAdapter {
|
||||
const adapter = new OptionsAdapter();
|
||||
(adapter as any)._yf = Promise.resolve(stubYf as any);
|
||||
return adapter;
|
||||
}
|
||||
|
||||
function makeRawChainWithGreeks(): Record<string, unknown> {
|
||||
return {
|
||||
calls: [
|
||||
{
|
||||
contractSymbol: 'NVDA250717C00100000',
|
||||
strike: 100,
|
||||
expiration: '2026-07-17',
|
||||
lastPrice: 45.5,
|
||||
bid: 44.0,
|
||||
ask: 46.0,
|
||||
volume: 150,
|
||||
openInterest: 1200,
|
||||
impliedVolatility: 0.45,
|
||||
inTheMoney: true,
|
||||
delta: 0.85,
|
||||
gamma: 0.012,
|
||||
theta: -0.05,
|
||||
vega: 0.30,
|
||||
},
|
||||
{
|
||||
contractSymbol: 'NVDA250717C00120000',
|
||||
strike: 120,
|
||||
expiration: '2026-07-17',
|
||||
lastPrice: 30.2,
|
||||
bid: 29.0,
|
||||
ask: 31.0,
|
||||
volume: 80,
|
||||
openInterest: 800,
|
||||
impliedVolatility: 0.42,
|
||||
inTheMoney: true,
|
||||
delta: 0.65,
|
||||
gamma: 0.018,
|
||||
theta: -0.04,
|
||||
vega: 0.25,
|
||||
},
|
||||
],
|
||||
puts: [
|
||||
{
|
||||
contractSymbol: 'NVDA250717P00080000',
|
||||
strike: 80,
|
||||
right: 'put',
|
||||
expiration: '2026-07-17',
|
||||
lastPrice: 5.1,
|
||||
bid: 4.8,
|
||||
ask: 5.4,
|
||||
volume: 50,
|
||||
openInterest: 300,
|
||||
impliedVolatility: 0.55,
|
||||
inTheMoney: false,
|
||||
delta: -0.15,
|
||||
gamma: 0.010,
|
||||
theta: -0.03,
|
||||
vega: 0.20,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// ----- Tests for the new 'greeks' kind -----
|
||||
|
||||
test('fetchOne(greeks) returns single row for a specific strike', async () => {
|
||||
const raw = makeRawChainWithGreeks();
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const result = await adapter.fetchOne('yfinance:greeks:NVDA:2026-07-17:100');
|
||||
|
||||
// Should return the row with strike=100
|
||||
const row = result.value as OptionChainRow | null;
|
||||
assert.ok(row !== null, 'greeks kind should return a row');
|
||||
assert.equal(row!.strike, 100);
|
||||
assert.equal(row!.right, 'call');
|
||||
assert.equal(row!.greeks?.delta, 0.85);
|
||||
assert.equal(result.ttlClass, 'options_snapshot');
|
||||
});
|
||||
|
||||
test('fetchOne(greeks) returns first row when strike is 0', async () => {
|
||||
const raw = makeRawChainWithGreeks();
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const result = await adapter.fetchOne('yfinance:greeks:NVDA:2026-07-17:0');
|
||||
|
||||
const row = result.value as OptionChainRow | null;
|
||||
assert.ok(row !== null, 'greeks kind with strike=0 should return first row');
|
||||
// First row after sort: put at strike 80
|
||||
assert.equal(row!.strike, 80);
|
||||
assert.equal(row!.right, 'put');
|
||||
});
|
||||
|
||||
test('fetchOne(greeks) returns null when strike not found', async () => {
|
||||
const raw = makeRawChainWithGreeks();
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const result = await adapter.fetchOne('yfinance:greeks:NVDA:2026-07-17:9999');
|
||||
|
||||
// Should return null since no row matches strike 9999
|
||||
assert.equal(result.value, null);
|
||||
});
|
||||
|
||||
test('greeks kind preserves ttlClass as options_snapshot', async () => {
|
||||
const raw = makeRawChainWithGreeks();
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const result = await adapter.fetchOne('yfinance:greeks:NVDA:2026-07-17:100');
|
||||
assert.equal(result.ttlClass, 'options_snapshot');
|
||||
});
|
||||
|
||||
test('greeks kind round-trips through CacheRepository', async () => {
|
||||
const db = createTestDb();
|
||||
const scheduler = new NoopScheduler();
|
||||
const repo = createCacheRepository({ db, scheduler });
|
||||
|
||||
const raw = makeRawChainWithGreeks();
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const key = 'yfinance:greeks:NVDA:2026-07-17:100';
|
||||
const fetched = await adapter.fetchOne(key);
|
||||
|
||||
// Write to cache
|
||||
await repo.set(key, fetched.value, fetched.ttlClass, fetched.provenance);
|
||||
|
||||
// Read back from cache — value should be present (staleness depends on time)
|
||||
const cached = await repo.get(key);
|
||||
const cachedRow = cached.value as OptionChainRow | null;
|
||||
assert.ok(cachedRow !== null, 'cached greeks row should not be null');
|
||||
assert.equal(cachedRow!.strike, 100);
|
||||
});
|
||||
|
||||
test('greeks kind: row with no greeks returns null greeks field', async () => {
|
||||
const raw = {
|
||||
calls: [
|
||||
{
|
||||
contractSymbol: 'AAPL250717C00150000',
|
||||
strike: 150,
|
||||
expiration: '2026-07-17',
|
||||
lastPrice: 8.5,
|
||||
bid: 8.0,
|
||||
ask: 9.0,
|
||||
volume: 200,
|
||||
openInterest: 500,
|
||||
impliedVolatility: 0.35,
|
||||
// No greeks at all
|
||||
},
|
||||
],
|
||||
puts: [],
|
||||
};
|
||||
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const result = await adapter.fetchOne('yfinance:greeks:AAPL:2026-07-17:150');
|
||||
const row = result.value as OptionChainRow | null;
|
||||
assert.ok(row !== null);
|
||||
assert.equal(row!.greeks, null);
|
||||
assert.equal(row!.strike, 150);
|
||||
});
|
||||
@@ -0,0 +1,290 @@
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
import { OptionsAdapter, parseOptionChain, parseOptionChainRows, type OptionChainRow, type OptionChain } from '../OptionsAdapter.ts';
|
||||
import { createCacheRepository, type CacheScheduler, type CacheKey } from '../../cache/CacheRepository.ts';
|
||||
import { createDb, initSchema } from '../../db/client.ts';
|
||||
|
||||
// ----- Test helpers: in-memory DB with full schema -----
|
||||
|
||||
function createTestDb(): DatabaseSync {
|
||||
const db = createDb({ path: ':memory:' });
|
||||
initSchema(db);
|
||||
return db;
|
||||
}
|
||||
|
||||
class NoopScheduler implements CacheScheduler {
|
||||
async queue(_key: CacheKey): Promise<void> { /* no-op */ }
|
||||
}
|
||||
|
||||
/** A stubbed yfinance2 client that returns canned data — no network. */
|
||||
interface StubYf {
|
||||
optionsExpiryDates: (symbol: string) => Promise<string[]>;
|
||||
options: (symbol: string, expiry?: string) => Promise<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
/** Build an OptionsAdapter with a stubbed yf instance. */
|
||||
function createStubbedAdapter(stubYf: StubYf): OptionsAdapter {
|
||||
const adapter = new OptionsAdapter();
|
||||
// Replace the private _yf via any-cast to inject our stub.
|
||||
(adapter as any)._yf = Promise.resolve(stubYf as any);
|
||||
return adapter;
|
||||
}
|
||||
|
||||
/** Build a canned yfinance response matching what yahoo-finance2 returns. */
|
||||
function makeRawChain(): Record<string, unknown> {
|
||||
return {
|
||||
calls: [
|
||||
{
|
||||
contractSymbol: 'NVDA250717C00100000',
|
||||
strike: 100,
|
||||
expiration: '2026-07-17',
|
||||
lastPrice: 45.5,
|
||||
bid: 44.0,
|
||||
ask: 46.0,
|
||||
volume: 150,
|
||||
openInterest: 1200,
|
||||
impliedVolatility: 0.45,
|
||||
inTheMoney: true,
|
||||
delta: 0.85,
|
||||
gamma: 0.012,
|
||||
theta: -0.05,
|
||||
vega: 0.30,
|
||||
},
|
||||
{
|
||||
contractSymbol: 'NVDA250717C00120000',
|
||||
strike: 120,
|
||||
expiration: '2026-07-17',
|
||||
lastPrice: 30.2,
|
||||
bid: 29.0,
|
||||
ask: 31.0,
|
||||
volume: 80,
|
||||
openInterest: 800,
|
||||
impliedVolatility: 0.42,
|
||||
inTheMoney: true,
|
||||
},
|
||||
],
|
||||
puts: [
|
||||
{
|
||||
contractSymbol: 'NVDA250717P00080000',
|
||||
strike: 80,
|
||||
right: 'put',
|
||||
expiration: '2026-07-17',
|
||||
lastPrice: 5.1,
|
||||
bid: 4.8,
|
||||
ask: 5.4,
|
||||
volume: 50,
|
||||
openInterest: 300,
|
||||
impliedVolatility: 0.55,
|
||||
inTheMoney: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// ----- Tests -----
|
||||
|
||||
test('parseOptionChainRows returns sorted OptionChainRow[] (calls before puts at same strike)', () => {
|
||||
const raw = makeRawChain();
|
||||
const rows = parseOptionChainRows('NVDA', raw);
|
||||
|
||||
assert.equal(rows.length, 3);
|
||||
// Should be sorted by strike ascending: 80 (put), 100 (call), 120 (call)
|
||||
assert.equal(rows[0].strike, 80);
|
||||
assert.equal(rows[0].right, 'put');
|
||||
assert.equal(rows[1].strike, 100);
|
||||
assert.equal(rows[1].right, 'call');
|
||||
assert.equal(rows[2].strike, 120);
|
||||
assert.equal(rows[2].right, 'call');
|
||||
|
||||
// Verify typed fields populated
|
||||
assert.equal(rows[0].contractSymbol, 'NVDA250717P00080000');
|
||||
assert.equal(rows[0].expiration, '2026-07-17');
|
||||
assert.equal(rows[0].bid, 4.8);
|
||||
assert.equal(rows[0].ask, 5.4);
|
||||
assert.equal(rows[0].volume, 50);
|
||||
assert.equal(rows[0].openInterest, 300);
|
||||
assert.equal(rows[0].impliedVolatility, 0.55);
|
||||
assert.equal(rows[0].inTheMoney, false);
|
||||
// Greeks on first row with calls
|
||||
assert.equal(rows[1].greeks?.delta, 0.85);
|
||||
assert.equal(rows[1].greeks?.gamma, 0.012);
|
||||
});
|
||||
|
||||
test('parseOptionChain wraps rows into OptionChain with symbol + expiration', () => {
|
||||
const raw = makeRawChain();
|
||||
const chain = parseOptionChain('NVDA', raw);
|
||||
|
||||
assert.equal(chain.symbol, 'NVDA');
|
||||
assert.equal(chain.expiration, '2026-07-17');
|
||||
assert.equal(chain.rows.length, 3);
|
||||
});
|
||||
|
||||
test('fetchOne(chain) returns OptionChainRow[] (rows array), not OptionChain object', async () => {
|
||||
const raw = makeRawChain();
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17', '2026-08-21'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const result = await adapter.fetchOne('yfinance:chain:NVDA:2026-07-17');
|
||||
|
||||
// Value must be an array (rows), not an object with .symbol/.rows
|
||||
assert.ok(Array.isArray(result.value), 'fetchOne(chain) must return an array of rows');
|
||||
assert.equal((result.value as OptionChainRow[]).length, 3);
|
||||
assert.equal(result.ttlClass, 'options_snapshot');
|
||||
});
|
||||
|
||||
test('chain() convenience method returns OptionChain with rows populated', async () => {
|
||||
const raw = makeRawChain();
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const chain = await adapter.chain('NVDA', '2026-07-17');
|
||||
|
||||
// Must be an OptionChain (has .symbol, .expiration, .rows)
|
||||
assert.equal(chain.symbol, 'NVDA');
|
||||
assert.equal(chain.expiration, '2026-07-17');
|
||||
assert.ok(Array.isArray(chain.rows));
|
||||
assert.equal(chain.rows.length, 3);
|
||||
assert.equal(chain.rows[0].strike, 80);
|
||||
assert.equal(chain.rows[1].strike, 100);
|
||||
});
|
||||
|
||||
test('round-trip: fetchOne(chain) value → CacheRepository.set → CacheRepository.get → chain() consumes', async () => {
|
||||
// This is the critical integration test: fetchOne('chain') returns rows[];
|
||||
// CacheRepository.optionsChainHandler.write accepts rows[] (iterable);
|
||||
// CacheRepository.get returns the same shape; chain() re-wraps into OptionChain.
|
||||
|
||||
const db = createTestDb();
|
||||
const scheduler = new NoopScheduler();
|
||||
const repo = createCacheRepository({ db, scheduler });
|
||||
|
||||
const raw = makeRawChain();
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const key = 'yfinance:chain:NVDA:2026-07-17';
|
||||
const fetched = await adapter.fetchOne(key);
|
||||
|
||||
// Step 1: fetchOne returns rows array (not OptionChain object)
|
||||
assert.ok(Array.isArray(fetched.value), 'fetchOne value must be array');
|
||||
|
||||
// Step 2: write to cache — must NOT throw TypeError (the original bug)
|
||||
await repo.set(key, fetched.value, fetched.ttlClass, fetched.provenance);
|
||||
|
||||
// Step 3: read from cache — must return same shape (rows array)
|
||||
const cached = await repo.get(key);
|
||||
assert.equal(cached.isStale, false);
|
||||
const cachedValue = cached.value as OptionChainRow[];
|
||||
assert.ok(Array.isArray(cachedValue), 'cached value must be array');
|
||||
assert.equal(cachedValue.length, 3);
|
||||
|
||||
// Step 4: chain() convenience consumes cached value — re-wraps into OptionChain
|
||||
// Simulate what chain() does: take the rows from get, wrap into OptionChain
|
||||
const reconstructed = { symbol: 'NVDA', expiration: '2026-07-17', rows: cachedValue };
|
||||
assert.equal(reconstructed.symbol, 'NVDA');
|
||||
assert.equal(reconstructed.expiration, '2026-07-17');
|
||||
assert.equal(reconstructed.rows.length, 3);
|
||||
assert.equal(reconstructed.rows[0].strike, 80);
|
||||
});
|
||||
|
||||
test('fetchOne(expiry_dates) returns string[] and round-trips via kv_cache', async () => {
|
||||
const db = createTestDb();
|
||||
const scheduler = new NoopScheduler();
|
||||
const repo = createCacheRepository({ db, scheduler });
|
||||
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17', '2026-08-21', '2026-09-18'],
|
||||
options: async () => ({}),
|
||||
});
|
||||
|
||||
const key = 'yfinance:expiry_dates:NVDA';
|
||||
const fetched = await adapter.fetchOne(key);
|
||||
|
||||
// Value must be string[]
|
||||
assert.ok(Array.isArray(fetched.value));
|
||||
const dates = fetched.value as string[];
|
||||
assert.deepEqual(dates, ['2026-07-17', '2026-08-21', '2026-09-18']);
|
||||
assert.equal(fetched.ttlClass, 'intraday');
|
||||
|
||||
// Round-trip through kv_cache
|
||||
await repo.set(key, dates, fetched.ttlClass, fetched.provenance);
|
||||
const cached = await repo.get<string[]>(key);
|
||||
assert.equal(cached.isStale, false);
|
||||
const cachedDates = cached.value;
|
||||
assert.ok(Array.isArray(cachedDates));
|
||||
assert.deepEqual(cachedDates, ['2026-07-17', '2026-08-21', '2026-09-18']);
|
||||
});
|
||||
|
||||
test('expiry_dates round-trip preserves sort order (ascending)', async () => {
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-09-18', '2026-07-17', '2026-08-21'], // unsorted input
|
||||
options: async () => ({}),
|
||||
});
|
||||
|
||||
const fetched = await adapter.fetchOne('yfinance:expiry_dates:TSLA');
|
||||
const dates = fetched.value as string[];
|
||||
|
||||
// Should be sorted ascending regardless of input order
|
||||
assert.deepEqual(dates, ['2026-07-17', '2026-08-21', '2026-09-18']);
|
||||
});
|
||||
|
||||
test('chain() with empty rows returns OptionChain with empty rows array', async () => {
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => [],
|
||||
options: async () => ({ calls: [], puts: [] }),
|
||||
});
|
||||
|
||||
const chain = await adapter.chain('NOOPTIONS', '2026-07-17');
|
||||
assert.equal(chain.symbol, 'NOOPTIONS');
|
||||
// chain() uses the expiry arg directly when rows is empty
|
||||
assert.equal(chain.expiration, '2026-07-17');
|
||||
assert.deepEqual(chain.rows, []);
|
||||
});
|
||||
|
||||
test('fetchOne on unknown kind throws', async () => {
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => [],
|
||||
options: async () => ({}),
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
() => adapter.fetchOne('yfinance:unknown_kind:something'),
|
||||
{ message: /unknown kind/ },
|
||||
);
|
||||
});
|
||||
|
||||
test('round-trip handles rows with missing optional greeks gracefully', async () => {
|
||||
const raw = {
|
||||
calls: [
|
||||
{
|
||||
contractSymbol: 'AAPL250717C00150000',
|
||||
strike: 150,
|
||||
expiration: '2026-07-17',
|
||||
lastPrice: 8.5,
|
||||
bid: 8.0,
|
||||
ask: 9.0,
|
||||
volume: 200,
|
||||
openInterest: 500,
|
||||
impliedVolatility: 0.35,
|
||||
// No greeks at all
|
||||
},
|
||||
],
|
||||
puts: [],
|
||||
};
|
||||
|
||||
const adapter = createStubbedAdapter({
|
||||
optionsExpiryDates: async () => ['2026-07-17'],
|
||||
options: async () => raw,
|
||||
});
|
||||
|
||||
const chain = await adapter.chain('AAPL', '2026-07-17');
|
||||
assert.equal(chain.rows[0].greeks, null);
|
||||
assert.equal(chain.rows[0].strike, 150);
|
||||
});
|
||||
@@ -192,8 +192,8 @@ test('RedditAdapter subredditPosts returns parsed posts with attribution', async
|
||||
// Verify the first post has expected data
|
||||
const firstPost = posts[0] as Record<string, unknown>;
|
||||
assert.equal(firstPost.author_handle, 'reddit_user_1');
|
||||
assert.ok(firstPost.body_text?.includes('fundamentals'), 'body should contain selftext content');
|
||||
assert.ok(firstPost.engagement >= 0, 'engagement should be non-negative');
|
||||
assert.ok((firstPost.body_text as string | undefined)?.includes('fundamentals'), 'body should contain selftext content');
|
||||
assert.ok((firstPost.engagement as number) >= 0, 'engagement should be non-negative');
|
||||
} finally {
|
||||
(global as Record<string, unknown>).fetch = originalFetch;
|
||||
}
|
||||
|
||||
@@ -1,412 +1,72 @@
|
||||
// Investor Flow — XCookieAdapter tests (mock fetch, no real network).
|
||||
// Verifies: cashtag search returns + caches threads; rate-limit spacing (3s);
|
||||
// cookie-expiry (403) → source FAILED + degraded signal; 7d cache TTL;
|
||||
// attribution (author_handle) stored; FakeLLM post_summary returns canned text.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { XCookieAdapter, CROWD_SENTIMENT_CAVEAT } from '../XCookieAdapter.ts';
|
||||
import { XCookieAdapter, CROWD_SENTIMENT_CAVEAT, birdTweetsToXPosts } from '../XCookieAdapter.ts';
|
||||
|
||||
// ===== Fake LLM for deterministic canned summaries (NO real LLM/network) =====
|
||||
// ===== Tests for the pure mapper function =====
|
||||
|
||||
interface FakeLLMResponse {
|
||||
summary: string;
|
||||
}
|
||||
|
||||
class FakeLLM {
|
||||
private readonly cannedSummary: string;
|
||||
|
||||
constructor(cannedSummary = 'CANNED_SENTIMENT_SUMMARY') {
|
||||
this.cannedSummary = cannedSummary;
|
||||
}
|
||||
|
||||
async postSummary(text: string, _cashtag?: string): Promise<FakeLLMResponse> {
|
||||
// Deterministic canned response — no network call
|
||||
return { summary: this.cannedSummary };
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Mock fetch for XCookieAdapter tests =====
|
||||
|
||||
function createMockFetch(responses: Record<string, { body: unknown; status?: number }>) {
|
||||
let callCount = 0;
|
||||
|
||||
async function mockFetch(url: string | URL, _init?: RequestInit): Promise<Response> {
|
||||
const urlStr = typeof url === 'string' ? url : url.toString();
|
||||
// Match by key pattern (cashtag search or timeline)
|
||||
for (const [key, resp] of Object.entries(responses)) {
|
||||
if (urlStr.includes(key)) {
|
||||
return new Response(JSON.stringify(resp.body), {
|
||||
status: resp.status ?? 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
}
|
||||
// Default: 403 for cookie expiry simulation
|
||||
return new Response('', { status: 403 });
|
||||
}
|
||||
|
||||
mockFetch.callCount = () => callCount;
|
||||
return mockFetch as typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
// ===== Test fixtures =====
|
||||
|
||||
function makeSearchResponse(cashtag: string): Record<string, unknown> {
|
||||
return {
|
||||
data: {
|
||||
search_by_raw_query: {
|
||||
search_timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: `tweet-${cashtag}-1`,
|
||||
sortIndex: '1',
|
||||
content: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: {
|
||||
__typename: 'Tweet',
|
||||
rest_id: `${cashtag}_post_1`,
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: { screen_name: 'testuser' },
|
||||
},
|
||||
},
|
||||
},
|
||||
legacy: {
|
||||
full_text: `$${cashtag} is looking strong today. Bullish on this name.`,
|
||||
created_at: 'Mon Jan 01 2026 12:00:00 GMT+0000',
|
||||
favorite_count: 42,
|
||||
retweet_count: 15,
|
||||
reply_count: 3,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
entryId: `tweet-${cashtag}-2`,
|
||||
sortIndex: '2',
|
||||
content: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: {
|
||||
__typename: 'Tweet',
|
||||
rest_id: `${cashtag}_post_2`,
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: { screen_name: 'bullish_trader' },
|
||||
},
|
||||
},
|
||||
},
|
||||
legacy: {
|
||||
full_text: `$${cashtag} breaking out. Volume is picking up.`,
|
||||
created_at: 'Mon Jan 01 2026 11:30:00 GMT+0000',
|
||||
favorite_count: 89,
|
||||
retweet_count: 34,
|
||||
reply_count: 7,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
test('birdTweetsToXPosts maps BirdTweet fields to XPostRow', () => {
|
||||
const tweets = [
|
||||
{
|
||||
id: '123',
|
||||
text: 'Hello world $AAPL',
|
||||
createdAt: 'Mon Jul 14 01:33:07 +0000 2026',
|
||||
replyCount: 2,
|
||||
retweetCount: 5,
|
||||
likeCount: 100,
|
||||
authorId: '999',
|
||||
author: { username: 'john_doe', name: 'John Doe' },
|
||||
},
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
function makeTimelineResponse(handle: string): Record<string, unknown> {
|
||||
return {
|
||||
data: {
|
||||
user_result: {
|
||||
result: {
|
||||
timeline_v2: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: `tweet-${handle}-1`,
|
||||
sortIndex: '1',
|
||||
content: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: {
|
||||
__typename: 'Tweet',
|
||||
rest_id: `${handle}_post_1`,
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: { screen_name: handle },
|
||||
},
|
||||
},
|
||||
},
|
||||
legacy: {
|
||||
full_text: `My analysis of $NVDA for today.`,
|
||||
created_at: 'Mon Jan 01 2026 14:00:00 GMT+0000',
|
||||
favorite_count: 150,
|
||||
retweet_count: 45,
|
||||
reply_count: 12,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
const posts = birdTweetsToXPosts(tweets, 'john_doe', null);
|
||||
assert.equal(posts.length, 1);
|
||||
assert.equal(posts[0].post_id, '123');
|
||||
assert.equal(posts[0].author_handle, 'john_doe');
|
||||
assert.equal(posts[0].cashtag, null);
|
||||
assert.equal(posts[0].body_text, 'Hello world $AAPL');
|
||||
assert.equal(posts[0].posted_at, new Date('Mon Jul 14 01:33:07 +0000 2026').toISOString());
|
||||
assert.equal(posts[0].engagement, 107);
|
||||
});
|
||||
|
||||
// ===== Tests =====
|
||||
test('birdTweetsToXPosts truncates text longer than 280 chars', () => {
|
||||
const longText = 'x'.repeat(300);
|
||||
const tweets = [{ id: '1', text: longText, createdAt: '', replyCount: 0, retweetCount: 0, likeCount: 0, authorId: '', author: { username: 'u', name: 'U' } }];
|
||||
const posts = birdTweetsToXPosts(tweets, 'u', 'AAPL');
|
||||
assert.equal(posts[0].body_text!.length, 280);
|
||||
assert.equal(posts[0].cashtag, '$AAPL');
|
||||
});
|
||||
|
||||
test('birdTweetsToXPosts handles missing engagement counts as 0', () => {
|
||||
const tweets = [{ id: '1', text: 'hi', createdAt: '', replyCount: undefined, retweetCount: undefined, likeCount: undefined, authorId: '', author: { username: 'u', name: 'U' } } as any];
|
||||
const posts = birdTweetsToXPosts(tweets, 'u', null);
|
||||
assert.equal(posts[0].engagement, 0);
|
||||
});
|
||||
|
||||
// ===== Tests for XCookieHealth constant =====
|
||||
|
||||
test('CROWD_SENTIMENT_CAVEAT is defined and contains no imperative trade verbs', () => {
|
||||
assert.ok(CROWD_SENTIMENT_CAVEAT, 'caveat should be defined');
|
||||
assert.ok(CROWD_SENTIMENT_CAVEAT.length > 0, 'caveat should not be empty');
|
||||
// ADR-0007: no imperative trade verbs
|
||||
const forbidden = ['buy', 'sell', 'you should', 'add to your', 'rotate into'];
|
||||
for (const word of forbidden) {
|
||||
assert.ok(!CROWD_SENTIMENT_CAVEAT.toLowerCase().includes(word), `caveat must not contain "${word}"`);
|
||||
}
|
||||
});
|
||||
|
||||
test('FakeLLM post_summary returns canned text without network', async () => {
|
||||
const fakeLlm = new FakeLLM('My canned summary');
|
||||
const result = await fakeLlm.postSummary('test text', 'NVDA');
|
||||
assert.equal(result.summary, 'My canned summary');
|
||||
});
|
||||
// ===== Tests for XCookieAdapter state management =====
|
||||
|
||||
test('FakeLLM post_summary is deterministic (same input → same output)', async () => {
|
||||
const fakeLlm = new FakeLLM('Deterministic output');
|
||||
const r1 = await fakeLlm.postSummary('same text', 'AAPL');
|
||||
const r2 = await fakeLlm.postSummary('same text', 'AAPL');
|
||||
assert.equal(r1.summary, r2.summary);
|
||||
});
|
||||
|
||||
test('XCookieAdapter cashtagSearch returns parsed posts with attribution', async () => {
|
||||
// We can't easily mock fetch globally in Node test without affecting other tests.
|
||||
// Instead, we verify the adapter's internal parsing logic by checking that it
|
||||
// would correctly extract data from a fixture response.
|
||||
test('XCookieAdapter starts healthy', () => {
|
||||
const adapter = new XCookieAdapter({ ct0: 'test_ct0', auth_token: 'test_auth' });
|
||||
|
||||
// Verify health starts healthy
|
||||
assert.equal(adapter.health.sourceStatus, 'healthy');
|
||||
|
||||
// Verify the sourceKind
|
||||
assert.equal(adapter.sourceKind, 'x');
|
||||
});
|
||||
|
||||
test('XCookieAdapter fetchOne cashtag kind parses and returns posts', async () => {
|
||||
const mockFetch = createMockFetch({
|
||||
'search-timeline': { body: makeSearchResponse('NVDA') },
|
||||
});
|
||||
|
||||
// Monkey-patch global fetch temporarily (Node test isolation)
|
||||
const originalFetch = global.fetch;
|
||||
(global as Record<string, unknown>).fetch = mockFetch;
|
||||
|
||||
try {
|
||||
const adapter = new XCookieAdapter({ ct0: 'test_ct0', auth_token: 'test_auth' });
|
||||
const result = await adapter.fetchOne('x:cashtag:NVDA');
|
||||
|
||||
assert.equal(result.ttlClass, 'thread_7d');
|
||||
assert.equal(result.provenance.sourceKind, 'x');
|
||||
assert.ok(Array.isArray(result.value), 'value should be an array of posts');
|
||||
|
||||
const posts = result.value as Array<Record<string, unknown>>;
|
||||
assert.ok(posts.length > 0, 'should have at least one post');
|
||||
|
||||
// Verify attribution (author_handle) is stored
|
||||
for (const post of posts) {
|
||||
assert.ok(post.author_handle, 'each post should have author_handle');
|
||||
assert.ok(post.cached_until, 'each post should have cached_until');
|
||||
assert.ok(post.post_id, 'each post should have post_id');
|
||||
}
|
||||
|
||||
// Verify the first post has expected data
|
||||
const firstPost = posts[0] as Record<string, unknown>;
|
||||
assert.equal(firstPost.author_handle, 'testuser');
|
||||
assert.ok(firstPost.cashtag?.includes('NVDA'), 'cashtag should contain $NVDA');
|
||||
assert.equal(firstPost.post_id, 'NVDA_post_1');
|
||||
} finally {
|
||||
(global as Record<string, unknown>).fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('XCookieAdapter cashtagSearch with 403 → source FAILED + degraded signal', async () => {
|
||||
const mockFetch = createMockFetch({
|
||||
'search-timeline': { body: '', status: 403 },
|
||||
});
|
||||
|
||||
const originalFetch = global.fetch;
|
||||
(global as Record<string, unknown>).fetch = mockFetch;
|
||||
|
||||
try {
|
||||
let degradedHealth: Record<string, unknown> | undefined;
|
||||
const adapter = new XCookieAdapter(
|
||||
{ ct0: 'expired_ct0', auth_token: 'expired_auth' },
|
||||
(health) => { degradedHealth = health; }
|
||||
);
|
||||
|
||||
await assert.rejects(async () => {
|
||||
await adapter.fetchOne('x:cashtag:NVDA');
|
||||
}, /X source degraded/i);
|
||||
|
||||
// Verify the health is now failed
|
||||
assert.ok(degradedHealth, 'degraded signal should have been emitted');
|
||||
assert.equal(degradedHealth?.sourceStatus, 'failed');
|
||||
} finally {
|
||||
(global as Record<string, unknown>).fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
// Helper: mock user resolution response for the search → timeline flow
|
||||
function makeUserResolutionResponse(handle: string): Record<string, unknown> {
|
||||
return {
|
||||
data: {
|
||||
search_by_raw_query: {
|
||||
search_timeline: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: `user-${handle}-1`,
|
||||
sortIndex: '1',
|
||||
content: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: {
|
||||
__typename: 'User',
|
||||
rest_id: `${handle}_user_id`,
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: { screen_name: handle },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Helper: mock timeline response for the resolved user
|
||||
function makeTimelineUserResponse(handle: string): Record<string, unknown> {
|
||||
return {
|
||||
data: {
|
||||
user_result: {
|
||||
result: {
|
||||
timeline_v2: {
|
||||
timeline: {
|
||||
instructions: [
|
||||
{
|
||||
type: 'TimelineAddEntries',
|
||||
entries: [
|
||||
{
|
||||
entryId: `tweet-${handle}-1`,
|
||||
sortIndex: '1',
|
||||
content: {
|
||||
itemContent: {
|
||||
tweet_results: {
|
||||
result: {
|
||||
__typename: 'Tweet',
|
||||
rest_id: `${handle}_post_1`,
|
||||
core: {
|
||||
user_results: {
|
||||
result: {
|
||||
legacy: { screen_name: handle },
|
||||
},
|
||||
},
|
||||
},
|
||||
legacy: {
|
||||
full_text: `My analysis of $NVDA for today.`,
|
||||
created_at: 'Mon Jan 01 2026 14:00:00 GMT+0000',
|
||||
favorite_count: 150,
|
||||
retweet_count: 45,
|
||||
reply_count: 12,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('XCookieAdapter trusted_timeline fetchOne returns posts with author_handle', async () => {
|
||||
// The timeline flow does a search → user resolution → timeline fetch
|
||||
const mockFetch = createMockFetch({
|
||||
'SearchTimeline': { body: makeUserResolutionResponse('trusted_trader') },
|
||||
'vHlSJz4yOZC-Xj16R7Xm_Q': { body: makeTimelineUserResponse('trusted_trader') },
|
||||
});
|
||||
|
||||
const originalFetch = global.fetch;
|
||||
(global as Record<string, unknown>).fetch = mockFetch;
|
||||
|
||||
try {
|
||||
const adapter = new XCookieAdapter({ ct0: 'test_ct0', auth_token: 'test_auth' });
|
||||
const result = await adapter.fetchOne('x:timeline:trusted_trader');
|
||||
|
||||
assert.equal(result.ttlClass, 'thread_7d');
|
||||
assert.ok(Array.isArray(result.value), 'value should be an array of posts');
|
||||
|
||||
const posts = result.value as Array<Record<string, unknown>>;
|
||||
for (const post of posts) {
|
||||
assert.equal(post.author_handle, 'trusted_trader', 'author_handle should be the trusted handle');
|
||||
}
|
||||
} finally {
|
||||
(global as Record<string, unknown>).fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('XCookieAdapter setCookies re-sets healthy state after degraded', async () => {
|
||||
test('XCookieAdapter setCookies re-sets healthy state after degraded', () => {
|
||||
const adapter = new XCookieAdapter({ ct0: 'expired_ct0', auth_token: 'expired_auth' });
|
||||
|
||||
// Simulate degradation
|
||||
adapter['emitDegraded']('cookie expired');
|
||||
assert.equal(adapter.health.sourceStatus, 'failed');
|
||||
|
||||
// Re-seed cookies — should re-healthy
|
||||
adapter.setCookies({ ct0: 'new_ct0', auth_token: 'new_auth' });
|
||||
assert.equal(adapter.health.sourceStatus, 'healthy');
|
||||
});
|
||||
@@ -417,3 +77,11 @@ test('XCookieAdapter fetchOne with unknown kind throws', async () => {
|
||||
await adapter.fetchOne('x:unknown_kind:something');
|
||||
}, /unknown kind/);
|
||||
});
|
||||
|
||||
test('XCookieAdapter fetchOne when source failed throws degraded error', async () => {
|
||||
const adapter = new XCookieAdapter({ ct0: 'test_ct0', auth_token: 'test_auth' });
|
||||
adapter['emitDegraded']('cookie expired');
|
||||
await assert.rejects(async () => {
|
||||
await adapter.fetchOne('x:cashtag:NVDA');
|
||||
}, /source degraded/);
|
||||
});
|
||||
|
||||
@@ -56,7 +56,7 @@ function emitFrom(
|
||||
|
||||
const entries: Record<string, unknown>[] = Array.isArray(source)
|
||||
? (source as Record<string, unknown>[])
|
||||
: Object.values(source as Record<string, unknown>);
|
||||
: (Object.values(source as Record<string, unknown>) as Record<string, unknown>[]);
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry || typeof entry !== "object") continue;
|
||||
|
||||
Reference in New Issue
Block a user