feat: Unraid deploy, dealer-flow heatmap, confluence zones, 13F capture
Ship Node production images, Unraid compose, and Gitea CI/CD (test then push registry images; cron script if no runner). Rebuild dealer flow as a heatmap-first map with integrity gates and chart helpers. Add confluence zone rules, session clock, capture evidence, and tighter 13F/queue/options paths, plus the matching UI and tests.
This commit is contained in:
Investor Flow Build
@@ -214,6 +214,22 @@ export class XCookieAdapter implements SourceFetch {
|
||||
};
|
||||
}
|
||||
|
||||
/** Authenticated single-tweet read. Paywalled posts resolve; deleted do not. */
|
||||
async readTweetStatus(postId: string): Promise<'ok' | 'deleted' | 'unknown'> {
|
||||
try {
|
||||
const stdout = await this.runBird(['read', postId, '--json']);
|
||||
JSON.parse(stdout);
|
||||
return 'ok';
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
const { isTweetGoneMessage } = await import('../services/captureEvidence.ts');
|
||||
if (isTweetGoneMessage(msg)) {
|
||||
return 'deleted';
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
async fetchOne(key: string, opts?: FetchOpts): Promise<FetchResult> {
|
||||
const parts = key.split(':');
|
||||
if (parts.length < 2) throw new Error(`XCookieAdapter: invalid cache key "${key}"`);
|
||||
|
||||
@@ -12,11 +12,39 @@ import { withVendorGate } from '../services/vendorGate.ts';
|
||||
// Polite UA including operator contact (yfinance community norm; EDGAR requires similar).
|
||||
const OPERATOR_EMAIL = process.env.YF_OPERATOR_EMAIL ?? 'operator@example.com';
|
||||
|
||||
/** Full history window for cold candle fills (~10y). */
|
||||
/** Full history window for cold daily/weekly/monthly fills (~10y). */
|
||||
const FULL_HISTORY_DAYS = 3650;
|
||||
/** When we already have bars, only re-pull recent history + a small overlap. */
|
||||
/** When we already have daily bars, only re-pull recent history + a small overlap. */
|
||||
const INCREMENTAL_LOOKBACK_DAYS = 14;
|
||||
|
||||
/** Yahoo will not serve a longer 1m/5m window than this. */
|
||||
const INTRADAY_MAX_DAYS: Record<string, number> = { "1m": 7, "5m": 60 };
|
||||
const INTRADAY_WARM_DAYS: Record<string, number> = { "1m": 1, "5m": 3 };
|
||||
|
||||
/**
|
||||
* First timestamp Yahoo should be asked for. Minute TFs are capped;
|
||||
* daily stays on the existing 10y / 14d incremental rule.
|
||||
*/
|
||||
export function candleLookbackStart(
|
||||
timeframe: string,
|
||||
lastTs: string | null,
|
||||
now = Date.now(),
|
||||
): Date {
|
||||
const maxDays = INTRADAY_MAX_DAYS[timeframe];
|
||||
if (maxDays != null) {
|
||||
const floor = now - maxDays * 86_400_000;
|
||||
if (!lastTs) return new Date(floor);
|
||||
const last = Date.parse(lastTs);
|
||||
if (!Number.isFinite(last)) return new Date(floor);
|
||||
const warmDays = INTRADAY_WARM_DAYS[timeframe] ?? 1;
|
||||
return new Date(Math.max(floor, last - warmDays * 86_400_000));
|
||||
}
|
||||
if (!lastTs) return new Date(now - FULL_HISTORY_DAYS * 86_400_000);
|
||||
const last = Date.parse(lastTs);
|
||||
if (!Number.isFinite(last)) return new Date(now - FULL_HISTORY_DAYS * 86_400_000);
|
||||
return new Date(last - INCREMENTAL_LOOKBACK_DAYS * 86_400_000);
|
||||
}
|
||||
|
||||
export interface YFinanceAdapterOptions {
|
||||
/** Optional DB for incremental candle period1 (last bar ts). */
|
||||
db?: DatabaseSync;
|
||||
@@ -153,20 +181,10 @@ export class YFinanceAdapter implements SourceFetch {
|
||||
}
|
||||
}
|
||||
|
||||
/** Cold = full history; warm = last bar − lookback. */
|
||||
/** Cold = full history (or Yahoo cap); warm = last bar − overlap. */
|
||||
private candlePeriod1(symbol: string, timeframe: string): Date {
|
||||
if (!this._db) {
|
||||
return new Date(Date.now() - FULL_HISTORY_DAYS * 86_400_000);
|
||||
}
|
||||
const last = lastCandleTs(this._db, symbol, timeframe);
|
||||
if (!last) {
|
||||
return new Date(Date.now() - FULL_HISTORY_DAYS * 86_400_000);
|
||||
}
|
||||
const t = Date.parse(last);
|
||||
if (!Number.isFinite(t)) {
|
||||
return new Date(Date.now() - FULL_HISTORY_DAYS * 86_400_000);
|
||||
}
|
||||
return new Date(t - INCREMENTAL_LOOKBACK_DAYS * 86_400_000);
|
||||
const last = this._db ? lastCandleTs(this._db, symbol, timeframe) : null;
|
||||
return candleLookbackStart(timeframe, last);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import { strict as assert } from 'node:assert';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { parseQuote, parseCandles, parseSymbol, inferTickerKind } from '../YFinanceAdapter.ts';
|
||||
import { parseQuote, parseCandles, parseSymbol, inferTickerKind, candleLookbackStart } from '../YFinanceAdapter.ts';
|
||||
import { FakeSourceAdapter } from '../SourceAdapter.ts';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
@@ -99,6 +99,25 @@ test('parseCandles returns [] when quotes is absent/empty', () => {
|
||||
assert.deepEqual(parseCandles({ quotes: [null, { close: null }] }), []);
|
||||
});
|
||||
|
||||
test('candleLookbackStart caps 1m to 7d and 5m to 60d', () => {
|
||||
const now = Date.parse('2026-08-16T16:00:00Z');
|
||||
const m1 = candleLookbackStart('1m', null, now);
|
||||
const m5 = candleLookbackStart('5m', null, now);
|
||||
const d1 = candleLookbackStart('1d', null, now);
|
||||
assert.ok(now - m1.getTime() <= 7 * 86_400_000 + 1000);
|
||||
assert.ok(now - m1.getTime() >= 7 * 86_400_000 - 1000);
|
||||
assert.ok(now - m5.getTime() <= 60 * 86_400_000 + 1000);
|
||||
assert.ok(now - m5.getTime() >= 60 * 86_400_000 - 1000);
|
||||
assert.ok(now - d1.getTime() > 1000 * 86_400_000);
|
||||
});
|
||||
|
||||
test('candleLookbackStart warm 1m does not reach past the 7d floor', () => {
|
||||
const now = Date.parse('2026-08-16T16:00:00Z');
|
||||
const last = '2026-07-01T16:00:00Z';
|
||||
const start = candleLookbackStart('1m', last, now);
|
||||
assert.ok(start.getTime() >= now - 7 * 86_400_000 - 1000);
|
||||
});
|
||||
|
||||
test('parseSymbol extracts sector/industry from summaryProfile', () => {
|
||||
const raw = readFixture('nvda.summary.json');
|
||||
const s = parseSymbol(raw, 'NVDA');
|
||||
|
||||
@@ -126,7 +126,7 @@ export function alertTitle(type: AlertType, symbol?: string): string {
|
||||
case 'fund_13f':
|
||||
return `New 13F from tracked fund`;
|
||||
case 'mirror_diff':
|
||||
return `Mirror target changed${symbolTag(symbol)}`;
|
||||
return `Tracked fund book changed${symbolTag(symbol)}`;
|
||||
case 'vix_level':
|
||||
return `Volatility index level changed`;
|
||||
case 'confluence_change':
|
||||
@@ -164,7 +164,7 @@ export function alertDescription(type: AlertType, details?: string, symbol?: str
|
||||
case 'fund_13f':
|
||||
return `A tracked fund filed a new 13F. This is a disclosure, not advice.`;
|
||||
case 'mirror_diff':
|
||||
return `The mirror target changed${symbolTag(symbol)}. Showing the arithmetic delta; it is not advice.`;
|
||||
return `A tracked fund's book changed${symbolTag(symbol)}. Showing the arithmetic delta; it is not advice.`;
|
||||
case 'vix_level':
|
||||
return `The VIX, a market-wide measure of expected near-term volatility, has moved into a new level that historically mattered to market participants.`;
|
||||
case 'confluence_change':
|
||||
|
||||
@@ -37,6 +37,7 @@ test('alertTitle returns expected title for each alert type', () => {
|
||||
assert.equal(alertTitle('regime_shift'), 'Market regime changed');
|
||||
assert.equal(alertTitle('drawdown_halt'), 'Drawdown tolerance breached');
|
||||
assert.equal(alertTitle('informed_buy', 'AAPL'), 'Insider bought for AAPL');
|
||||
assert.equal(alertTitle('mirror_diff', 'SNOW'), 'Tracked fund book changed for SNOW');
|
||||
});
|
||||
|
||||
test('alertDescription never contains trade verbs', () => {
|
||||
@@ -56,6 +57,12 @@ test('alertDescription never contains trade verbs', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('alertDescription for mirror_diff names the tracked fund book', () => {
|
||||
const desc = alertDescription('mirror_diff', undefined, 'SHAZ');
|
||||
assert.match(desc, /tracked fund/i);
|
||||
assert.ok(!/mirror target/i.test(desc));
|
||||
});
|
||||
|
||||
test('alertDescription includes ADR-0007 compliant language', () => {
|
||||
const desc = alertDescription('drawdown_halt');
|
||||
assert.ok(desc.includes('circuit breaker'), 'Should mention circuit breaker');
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
// 2. fund_13f — a new 13F quarter landed from a tracked fund.
|
||||
// 3. mirror_diff — the live book behind the user's mirror changed materially.
|
||||
//
|
||||
// ADR-0010: every payload string is MECHANICAL ("to match your mirror target, the
|
||||
// ADR-0010: every payload string is MECHANICAL ("to match the tracked fund book, the
|
||||
// delta is ..."). No advice-shaped sentence. Lint-enforced in sub-task 9.
|
||||
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
@@ -136,8 +136,8 @@ export const mirrorDiffProducer: AlertProducer = {
|
||||
const dedup = `mirror_diff:${userId}:${fund.id}:${pos.symbol}:${sig}`;
|
||||
if (isDuplicate(db, buildDedupKey('mirror_diff', pos.symbol, dedup))) continue;
|
||||
const delta = pos.shares != null
|
||||
? `To match your mirror target for ${pos.symbol}, the delta is ${rnd(pos.shares).toLocaleString('en-US')} shares as disclosed ${pos.as_of}.`
|
||||
: `The ${pos.symbol} position in the mirror target updated as of ${pos.as_of}.`;
|
||||
? `To match the tracked fund book for ${pos.symbol}, the delta is ${rnd(pos.shares).toLocaleString('en-US')} shares as disclosed ${pos.as_of}.`
|
||||
: `The ${pos.symbol} position in the tracked fund book updated as of ${pos.as_of}.`;
|
||||
const alert = createAlert(
|
||||
crypto.randomUUID(), userId, 'mirror_diff', pos.symbol,
|
||||
`${fund.fund_name} updated its ${pos.symbol} book (${pos.as_of}). ${delta}`,
|
||||
|
||||
@@ -3,6 +3,7 @@ import { strict as assert } from 'node:assert';
|
||||
import {
|
||||
buildDealerMap,
|
||||
contractGex,
|
||||
contractVex,
|
||||
isDegradedDealerMap,
|
||||
isHealthyDealerMap,
|
||||
isSaneIv,
|
||||
@@ -50,6 +51,13 @@ test('contractGex: call positive, put negative', () => {
|
||||
assert.equal(call, -put);
|
||||
});
|
||||
|
||||
test('contractVex uses the same S² notional as GEX', () => {
|
||||
const gex = contractGex(0.02, 1000, 44, 'call');
|
||||
const vex = contractVex(0.02, 1000, 44, 'call');
|
||||
assert.equal(vex, gex);
|
||||
assert.equal(contractVex(0.02, 1000, 44, 'put'), -vex);
|
||||
});
|
||||
|
||||
test('buildDealerMap: expiries sorted nearest-left and capped', () => {
|
||||
const rows: NormalizedOptionRow[] = [];
|
||||
for (const exp of ['2026-08-08', '2026-08-15', '2026-08-22', '2026-09-19', '2026-10-17', '2026-11-21', '2026-12-19', '2027-01-16']) {
|
||||
|
||||
@@ -71,6 +71,41 @@ test('auditDealerMap: complete fat surface + map', () => {
|
||||
assert.match(integritySummary(r), /complete/i);
|
||||
});
|
||||
|
||||
test('auditDealerSurface: Friday chain on Sunday is not stale', () => {
|
||||
const s = fatSurface();
|
||||
s.asOf = '2026-08-16T17:00:00.000Z';
|
||||
s.quoteAsOf = '2026-08-16T17:00:00.000Z';
|
||||
s.chainAsOf = '2026-08-14T21:30:00.000Z';
|
||||
const r = auditDealerSurface(s);
|
||||
const age = r.checks.find((c) => c.id === 'chain_age');
|
||||
assert.ok(age);
|
||||
assert.equal(age!.ok, true);
|
||||
assert.ok(!r.hardFailed.includes('chain_age'));
|
||||
});
|
||||
|
||||
test('auditDealerSurface: 3-session-old book is incomplete', () => {
|
||||
const s = fatSurface();
|
||||
s.asOf = '2026-08-16T17:00:00.000Z';
|
||||
s.quoteAsOf = '2026-08-16T17:00:00.000Z';
|
||||
s.chainAsOf = '2026-08-12T03:46:00.000Z';
|
||||
const r = auditDealerSurface(s);
|
||||
assert.equal(r.status, 'incomplete');
|
||||
assert.ok(r.hardFailed.includes('chain_age'), JSON.stringify(r.hardFailed));
|
||||
});
|
||||
|
||||
test('auditDealerSurface: spot newer than book is soft degraded', () => {
|
||||
const s = fatSurface();
|
||||
s.asOf = '2026-08-14T20:00:00.000Z';
|
||||
s.quoteAsOf = '2026-08-14T20:00:00.000Z';
|
||||
s.chainAsOf = '2026-08-14T10:00:00.000Z';
|
||||
const r = auditDealerSurface(s);
|
||||
assert.notEqual(r.status, 'incomplete');
|
||||
const mix = r.checks.find((c) => c.id === 'spot_vs_chain');
|
||||
assert.ok(mix);
|
||||
assert.equal(mix!.ok, false);
|
||||
assert.equal(r.status, 'degraded');
|
||||
});
|
||||
|
||||
test('auditDealerSurface: delay alone does not hard-fail', () => {
|
||||
const s = fatSurface();
|
||||
s.delaySeconds = 900;
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { rthClosesBetween } from '../sessionClock.ts';
|
||||
|
||||
test('rthClosesBetween: Friday after-close book on Sunday is 0 sessions', () => {
|
||||
assert.equal(
|
||||
rthClosesBetween('2026-08-14T21:30:00.000Z', '2026-08-16T17:00:00.000Z'),
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('rthClosesBetween: Wednesday book on Sunday is 3 closes (Wed Thu Fri)', () => {
|
||||
assert.equal(
|
||||
rthClosesBetween('2026-08-12T03:46:00.000Z', '2026-08-16T17:00:00.000Z'),
|
||||
3,
|
||||
);
|
||||
});
|
||||
|
||||
test('rthClosesBetween: same timestamp is 0', () => {
|
||||
assert.equal(
|
||||
rthClosesBetween('2026-08-14T16:00:00.000Z', '2026-08-14T16:00:00.000Z'),
|
||||
0,
|
||||
);
|
||||
});
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
} from '../options/types.ts';
|
||||
import type { Provenance } from '../cache/CacheRepository.ts';
|
||||
import { DEALER_FLOW_EXPLAIN_NOTES } from './dealerFlowExplainNotes.ts';
|
||||
import { rthClosesBetween } from './sessionClock.ts';
|
||||
|
||||
// ----- Public types -----
|
||||
|
||||
@@ -85,6 +86,8 @@ export interface DealerMap {
|
||||
symbol: string;
|
||||
spot: number;
|
||||
asOf: string;
|
||||
quoteAsOf?: string | null;
|
||||
chainAsOf?: string | null;
|
||||
delayNote: string;
|
||||
delaySeconds: number | null;
|
||||
oiFreshness: OiFreshness;
|
||||
@@ -405,7 +408,7 @@ export function contractGex(
|
||||
return gamma * oi * MULTIPLIER * spot * spot * PCT_MOVE * signForRight(right, convention);
|
||||
}
|
||||
|
||||
/** Dollar VEX approximation for one contract lot. */
|
||||
/** Dollar VEX for one contract lot. Same notional skeleton as GEX (S² · 1% · 100 · OI). */
|
||||
export function contractVex(
|
||||
vanna: number,
|
||||
oi: number,
|
||||
@@ -413,8 +416,7 @@ export function contractVex(
|
||||
right: 'call' | 'put',
|
||||
convention: ExposureConvention = 'classic_call_pos_put_neg',
|
||||
): number {
|
||||
// Scale vanna similarly: notional delta change for 1% vol move × spot weight.
|
||||
return vanna * 0.01 * oi * MULTIPLIER * spot * signForRight(right, convention);
|
||||
return vanna * oi * MULTIPLIER * spot * spot * PCT_MOVE * signForRight(right, convention);
|
||||
}
|
||||
|
||||
function polarityOf(gex: number, nearZero: number): Polarity {
|
||||
@@ -642,15 +644,21 @@ function buildVelocity(
|
||||
return { asOfPrev: prior.asOf, nodes: nodes.slice(0, 20), growingNearAirPocket };
|
||||
}
|
||||
|
||||
function delayNote(delaySeconds: number | null, oi: OiFreshness): string {
|
||||
const parts: string[] = [];
|
||||
if (delaySeconds == null) parts.push('Delay unknown');
|
||||
else if (delaySeconds <= 0) parts.push('Quotes real-time');
|
||||
else parts.push(`Quotes delayed ~${Math.round(delaySeconds / 60)}m`);
|
||||
if (oi === 'prior_session') parts.push('Open interest typically from the prior session');
|
||||
else if (oi === 'intraday') parts.push('Open interest may update intraday');
|
||||
else parts.push('Open interest freshness unknown');
|
||||
return parts.join('. ') + '.';
|
||||
function delayNote(
|
||||
_delaySeconds: number | null,
|
||||
oi: OiFreshness,
|
||||
chainSessions: number | null,
|
||||
): string {
|
||||
const parts: string[] = ['Point-in-time snapshot of one option book'];
|
||||
if (chainSessions != null && chainSessions >= 2) {
|
||||
parts.push(`that book is ${chainSessions} regular sessions old`);
|
||||
} else if (oi === 'prior_session') {
|
||||
parts.push('open interest is the prior session official print, not a live book');
|
||||
} else if (oi === 'intraday') {
|
||||
parts.push('open interest may update intraday');
|
||||
}
|
||||
parts.push('gamma is local BSM when the vendor omits it');
|
||||
return `${parts[0]}. ${parts.slice(1).map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join('. ')}.`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -891,6 +899,11 @@ export function buildDealerMap(
|
||||
}
|
||||
: { isNearOpex: false, note: '' };
|
||||
|
||||
const chainSessions =
|
||||
surface.chainAsOf && surface.asOf
|
||||
? rthClosesBetween(surface.chainAsOf, surface.asOf)
|
||||
: null;
|
||||
|
||||
const coveragePct =
|
||||
totalContracts === 0 ? 0 : Math.round((includedContracts / totalContracts) * 1000) / 10;
|
||||
const quality = scoreGreekQuality({
|
||||
@@ -906,7 +919,9 @@ export function buildDealerMap(
|
||||
symbol: surface.symbol,
|
||||
spot: surface.spot,
|
||||
asOf: surface.asOf,
|
||||
delayNote: delayNote(surface.delaySeconds, surface.oiFreshness),
|
||||
quoteAsOf: surface.quoteAsOf ?? null,
|
||||
chainAsOf: surface.chainAsOf ?? null,
|
||||
delayNote: delayNote(surface.delaySeconds, surface.oiFreshness, chainSessions),
|
||||
delaySeconds: surface.delaySeconds,
|
||||
oiFreshness: surface.oiFreshness,
|
||||
providerId: surface.providerId,
|
||||
|
||||
@@ -78,7 +78,7 @@ export const DEALER_FLOW_EXPLAIN_NOTES: DealerFlowExplainNotes = {
|
||||
'Rug-like stack: support-looking structure that can fail and speed moves lower in the model if the soft floor breaks.',
|
||||
},
|
||||
caveats: [
|
||||
'Quotes may be delayed; open interest is often from the prior session.',
|
||||
'This is a point-in-time option-book snapshot. Official OI is the prior session print, not a live book.',
|
||||
'Numbers are modeled dollar hedging pressure, not reported dealer inventory.',
|
||||
'This explains a snapshot, not a price prediction.',
|
||||
],
|
||||
|
||||
@@ -6,6 +6,9 @@
|
||||
import type { NormalizedOptionSurface } from '../options/types.ts';
|
||||
import type { DealerMap, DealerMapQuality } from './dealerExposureEngine.ts';
|
||||
import { isSaneIv } from './dealerExposureEngine.ts';
|
||||
import { rthClosesBetween } from './sessionClock.ts';
|
||||
|
||||
export { rthClosesBetween } from './sessionClock.ts';
|
||||
|
||||
export type IntegrityStatus = 'complete' | 'degraded' | 'incomplete';
|
||||
|
||||
@@ -49,6 +52,10 @@ export interface DealerIntegrityThresholds {
|
||||
softBsmFillPct: number;
|
||||
/** Soft: bad IV skip above this → degraded (hard uses map quality / include) */
|
||||
softBadIvSkipPct: number;
|
||||
/** Soft: option book this many RTH closes behind the evaluation clock. */
|
||||
softChainSessions: number;
|
||||
/** Hard: option book this many RTH closes behind - not a delayed book, a dead one. */
|
||||
hardChainSessions: number;
|
||||
}
|
||||
|
||||
export const DEALER_INTEGRITY_DEFAULTS: DealerIntegrityThresholds = {
|
||||
@@ -61,8 +68,12 @@ export const DEALER_INTEGRITY_DEFAULTS: DealerIntegrityThresholds = {
|
||||
minByStrike: 5,
|
||||
softBsmFillPct: 90,
|
||||
softBadIvSkipPct: 15,
|
||||
softChainSessions: 2,
|
||||
hardChainSessions: 3,
|
||||
};
|
||||
|
||||
|
||||
|
||||
function check(
|
||||
id: string,
|
||||
level: 'hard' | 'soft',
|
||||
@@ -253,14 +264,14 @@ export function auditDealerSurface(
|
||||
}
|
||||
}
|
||||
|
||||
// Delay is soft informational only (always ok for hard)
|
||||
// Vendor quote delay is not the map clock. Informational only.
|
||||
if (surface.delaySeconds != null && surface.delaySeconds > 0) {
|
||||
checks.push(
|
||||
check(
|
||||
'delay',
|
||||
'soft',
|
||||
true,
|
||||
`Quotes delayed ~${Math.round(surface.delaySeconds / 60)}m (labeled, not a hard fail)`,
|
||||
'Underlying prints may be vendor-delayed; the map clock is the option-book timestamp',
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -270,11 +281,59 @@ export function auditDealerSurface(
|
||||
'oi_freshness',
|
||||
'soft',
|
||||
true,
|
||||
'OI typically prior session (labeled, not a hard fail)',
|
||||
'Official OI is the prior session print (Yahoo does not stream live OI)',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const chainAsOf = surface.chainAsOf ?? null;
|
||||
const evalNow = surface.asOf;
|
||||
if (chainAsOf && Date.parse(chainAsOf)) {
|
||||
const sessions = rthClosesBetween(chainAsOf, evalNow);
|
||||
if (sessions >= t.hardChainSessions) {
|
||||
checks.push(
|
||||
check(
|
||||
'chain_age',
|
||||
'hard',
|
||||
false,
|
||||
`Option book is ${sessions} regular sessions old (as of ${chainAsOf}). GEX/VEX are not a current book.`,
|
||||
),
|
||||
);
|
||||
} else if (sessions >= t.softChainSessions) {
|
||||
checks.push(
|
||||
check(
|
||||
'chain_age',
|
||||
'soft',
|
||||
false,
|
||||
`Option book is ${sessions} regular sessions old (as of ${chainAsOf})`,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
checks.push(
|
||||
check(
|
||||
'chain_age',
|
||||
'soft',
|
||||
true,
|
||||
sessions === 0
|
||||
? `Option book ${chainAsOf}`
|
||||
: `Option book ${sessions} session behind (${chainAsOf})`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const quoteAsOf = surface.quoteAsOf ?? null;
|
||||
if (quoteAsOf && Date.parse(quoteAsOf) && Date.parse(quoteAsOf) - Date.parse(chainAsOf) > 6 * 3600_000) {
|
||||
checks.push(
|
||||
check(
|
||||
'spot_vs_chain',
|
||||
'soft',
|
||||
false,
|
||||
`Spot print (${quoteAsOf}) is newer than the option book (${chainAsOf}). Cells mix a new price with older OI/IV.`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const metrics: DealerIntegrityMetrics = {
|
||||
expiries: liveExpiries.length,
|
||||
oiEligible,
|
||||
|
||||
@@ -310,9 +310,10 @@ export async function scheduleDealerMapInputs(
|
||||
await cache?.ensureInDemand(symbol, 'equity');
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Always want a quote for spot
|
||||
// Always want a quote for spot - jump the Yahoo pile so the open map is not 5h stale.
|
||||
try {
|
||||
await queue.queue(`yfinance:quote:${symbol}`);
|
||||
if (typeof queue.prioritize === "function") await queue.prioritize(`yfinance:quote:${symbol}`);
|
||||
else await queue.queue(`yfinance:quote:${symbol}`);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Regular-session clock for option-book freshness.
|
||||
* A session close is Mon–Fri 21:00 UTC (4pm ET DST / 5pm ET standard, close enough
|
||||
* to count "how many official OI prints we missed").
|
||||
*/
|
||||
|
||||
/** Count RTH closes strictly after `fromIso` and at or before `toIso`. */
|
||||
export function rthClosesBetween(fromIso: string, toIso: string): number {
|
||||
const from = Date.parse(fromIso);
|
||||
const to = Date.parse(toIso);
|
||||
if (!Number.isFinite(from) || !Number.isFinite(to) || to <= from) return 0;
|
||||
let n = 0;
|
||||
const d = new Date(from);
|
||||
d.setUTCHours(0, 0, 0, 0);
|
||||
for (let i = 0; i < 21; i++) {
|
||||
const dow = d.getUTCDay();
|
||||
if (dow !== 0 && dow !== 6) {
|
||||
const close = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate(), 21, 0, 0);
|
||||
if (close > from && close <= to) n += 1;
|
||||
}
|
||||
d.setUTCDate(d.getUTCDate() + 1);
|
||||
if (d.getTime() > to + 36 * 3600_000) break;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
+33
-14
@@ -42,7 +42,11 @@ export interface SymbolMeta { symbol: string; name?: string | null; sector?: str
|
||||
export interface PriceAdjustment { symbol: string; exDate: string; type: "split" | "dividend"; ratio: number }
|
||||
|
||||
/** Port CacheRepository depends on to schedule background refreshes. SourceAdapter/AdapterQueue satisfy this. */
|
||||
export interface CacheScheduler { queue(key: CacheKey): Promise<void>; }
|
||||
export interface CacheScheduler {
|
||||
queue(key: CacheKey): Promise<void>;
|
||||
/** Jump this key to the front of the next drain (header symbol / page view). */
|
||||
prioritize?(key: CacheKey): Promise<void>;
|
||||
}
|
||||
|
||||
export interface CacheEntry<T> { value: T | null; provenance: Provenance | null; isStale: boolean; }
|
||||
|
||||
@@ -89,7 +93,7 @@ interface KindHandler {
|
||||
/** Write value to cache, stamping provenance. */
|
||||
write(d: DatabaseSync, id: string, value: unknown, provenance: Provenance): void;
|
||||
/** Staleness verdict given the staleness timestamp (null = never cached) and now. */
|
||||
isStale(stalenessTs: string | null, now: number): boolean;
|
||||
isStale(stalenessTs: string | null, now: number, id?: string): boolean;
|
||||
}
|
||||
|
||||
function tsAgeMs(ts: string | null, now: number): number {
|
||||
@@ -155,16 +159,27 @@ const candlesHandler: KindHandler = {
|
||||
const rows = d.prepare('SELECT ts,o,h,l,c,v,adj_close,observed_at FROM price_candles WHERE symbol=? AND timeframe=? ORDER BY ts ASC').all(symbol, timeframe) as Array<Record<string, unknown>>;
|
||||
if (!rows.length) return null;
|
||||
const value: PriceCandle[] = rows.map((r) => ({ ts: r.ts as string, o: r.o as number, h: r.h as number, l: r.l as number, c: r.c as number, v: r.v as number, adjClose: r.adj_close as number | null }));
|
||||
// Staleness for SWR uses last *bar* ts (not observed_at) so schedule + get agree on lag.
|
||||
return { value, stalenessTs: rows[rows.length - 1].ts as string };
|
||||
const last = rows[rows.length - 1]!;
|
||||
// Minute TFs: freshness is last fetch (observed_at). Daily: last bar ts so schedule + get agree.
|
||||
const tf = timeframe ?? '';
|
||||
const stalenessTs = (tf === '1m' || tf === '5m')
|
||||
? (last.observed_at as string)
|
||||
: (last.ts as string);
|
||||
return { value, stalenessTs };
|
||||
},
|
||||
write(d, id, value, provenance) {
|
||||
const [symbol, timeframe] = id.split(':');
|
||||
const ins = d.prepare('INSERT OR REPLACE INTO price_candles (symbol,timeframe,ts,o,h,l,c,v,adj_close,observed_at) VALUES (?,?,?,?,?,?,?,?,?,?)');
|
||||
for (const c of value as PriceCandle[]) ins.run(symbol, timeframe, c.ts, c.o, c.h, c.l, c.c, c.v, c.adjClose ?? null, provenance.fetchedAt);
|
||||
},
|
||||
// Do not thrash on every read: schedule owns EOD refresh. Stale only when absent.
|
||||
isStale(ts) { return ts === null; },
|
||||
// Daily: schedule owns refresh (stale only when absent). 1m/5m: short observed_at TTL.
|
||||
isStale(ts, now, id) {
|
||||
if (ts === null) return true;
|
||||
const tf = id?.split(':')[1];
|
||||
if (tf === '1m') return tsAgeMs(ts, now) > 60_000;
|
||||
if (tf === '5m') return tsAgeMs(ts, now) > TTL_MS.intraday;
|
||||
return false;
|
||||
},
|
||||
};
|
||||
|
||||
/** True when daily candles are missing or last bar is older than CANDLE_FRESH_MS. */
|
||||
@@ -630,7 +645,7 @@ export interface CacheRepository {
|
||||
/** Permanent system pin (rotation universe, SPY, VIX) — survives unsubscribe. */
|
||||
pinSystemSymbol(symbol: string, tickerKind: TickerKind): Promise<void>;
|
||||
demandSet(): Promise<string[]>;
|
||||
getMany<T>(keys: CacheKey[]): Promise<Array<{ key: CacheKey; value: T | null; isStale: boolean }>>;
|
||||
getMany<T>(keys: CacheKey[]): Promise<Array<{ key: CacheKey; value: T | null; isStale: boolean; fetchedAt: string | null }>>;
|
||||
/** Delete a cache entry by key (or, for wildcard keys ending in `:*`, all matching entries). */
|
||||
del(key: CacheKey): Promise<void>;
|
||||
/** Underlying DB for schedule TTL checks (queue only). */
|
||||
@@ -655,7 +670,7 @@ export class CacheRepositoryImpl implements CacheRepository {
|
||||
const h = this.handler(kind);
|
||||
const row = h.read(this._db, id);
|
||||
const now = Date.now();
|
||||
let stale = h.isStale(row ? row.stalenessTs : null, now);
|
||||
let stale = h.isStale(row ? row.stalenessTs : null, now, id);
|
||||
// Incomplete symbol meta (null name) is always treated as stale for SWR re-fetch.
|
||||
if (kind === 'symbol' && row) {
|
||||
const meta = row.value as SymbolMeta;
|
||||
@@ -692,7 +707,7 @@ export class CacheRepositoryImpl implements CacheRepository {
|
||||
const meta = row.value as SymbolMeta;
|
||||
if (!meta?.name) return true;
|
||||
}
|
||||
return h.isStale(row ? row.stalenessTs : null, Date.now());
|
||||
return h.isStale(row ? row.stalenessTs : null, Date.now(), id);
|
||||
}
|
||||
|
||||
private ensureDemandRow(symbol: string, tickerKind: TickerKind): void {
|
||||
@@ -702,10 +717,14 @@ export class CacheRepositoryImpl implements CacheRepository {
|
||||
}
|
||||
|
||||
/** Queue only what is missing or past TTL — no refcount change. */
|
||||
private async queueIfNeeded(symbol: string): Promise<void> {
|
||||
private async queueIfNeeded(symbol: string, opts?: { prioritizeQuote?: boolean }): Promise<void> {
|
||||
const d = this._db;
|
||||
if (needsQuoteRefresh(d, symbol)) {
|
||||
try { await this._scheduler.queue(`yfinance:quote:${symbol}`); } catch { /* ignore */ }
|
||||
const key = `yfinance:quote:${symbol}`;
|
||||
try {
|
||||
if (opts?.prioritizeQuote && this._scheduler.prioritize) await this._scheduler.prioritize(key);
|
||||
else await this._scheduler.queue(key);
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
if (needsCandleRefresh(d, symbol)) {
|
||||
try { await this._scheduler.queue(`yfinance:candles:${symbol}:1d`); } catch { /* ignore */ }
|
||||
@@ -754,7 +773,7 @@ export class CacheRepositoryImpl implements CacheRepository {
|
||||
this._db.prepare(
|
||||
"UPDATE symbol_demand SET tier = MIN(tier, 2), last_viewed_at = ?, in_demand = 1 WHERE symbol=?",
|
||||
).run(now, symbol);
|
||||
await this.queueIfNeeded(symbol);
|
||||
await this.queueIfNeeded(symbol, { prioritizeQuote: true });
|
||||
}
|
||||
|
||||
async pinSystemSymbol(symbol: string, tickerKind: TickerKind): Promise<void> {
|
||||
@@ -778,10 +797,10 @@ export class CacheRepositoryImpl implements CacheRepository {
|
||||
'SELECT symbol FROM symbol_demand WHERE in_demand = 1 OR COALESCE(system_pin, 0) = 1 ORDER BY symbol',
|
||||
).all() as Array<{ symbol: string }>).map((r) => r.symbol);
|
||||
}
|
||||
async getMany<T>(keys: CacheKey[]): Promise<Array<{ key: CacheKey; value: T | null; isStale: boolean }>> {
|
||||
async getMany<T>(keys: CacheKey[]): Promise<Array<{ key: CacheKey; value: T | null; isStale: boolean; fetchedAt: string | null }>> {
|
||||
return Promise.all(keys.map(async (key) => {
|
||||
const e = await this.get<T>(key);
|
||||
return { key, value: e.value, isStale: e.isStale };
|
||||
return { key, value: e.value, isStale: e.isStale, fetchedAt: e.provenance?.fetchedAt ?? null };
|
||||
}));
|
||||
}
|
||||
private clearKvCachePrefix(prefix: string): void {
|
||||
|
||||
@@ -141,6 +141,42 @@ test('candles are permanent: not stale once present, stale when absent', async (
|
||||
assert.equal(scheduler.queued.length, 0);
|
||||
});
|
||||
|
||||
test('5m candles do not overwrite 1d rows', async () => {
|
||||
const { repo, db } = setup();
|
||||
const daily: PriceCandle[] = [{ ts: '2026-08-15T00:00:00.000Z', o: 100, h: 101, l: 99, c: 100.5, v: 1e6, adjClose: 100.5 }];
|
||||
const five: PriceCandle[] = [{ ts: '2026-08-15T14:30:00.000Z', o: 100.2, h: 100.4, l: 100.1, c: 100.3, v: 2e4, adjClose: 100.3 }];
|
||||
await repo.set('yfinance:candles:NVDA:1d', daily, 'daily_permanent', { fetchedAt: iso(), sourceKind: 'yfinance' });
|
||||
await repo.set('yfinance:candles:NVDA:5m', five, 'daily_permanent', { fetchedAt: iso(), sourceKind: 'yfinance' });
|
||||
const d = await repo.get<PriceCandle[]>('yfinance:candles:NVDA:1d');
|
||||
const m = await repo.get<PriceCandle[]>('yfinance:candles:NVDA:5m');
|
||||
assert.equal(d.value?.[0]?.c, 100.5);
|
||||
assert.equal(m.value?.[0]?.c, 100.3);
|
||||
const n = db.prepare('SELECT timeframe, COUNT(*) AS n FROM price_candles WHERE symbol=? GROUP BY timeframe').all('NVDA') as Array<{ timeframe: string; n: number }>;
|
||||
assert.equal(n.find((r) => r.timeframe === '1d')?.n, 1);
|
||||
assert.equal(n.find((r) => r.timeframe === '5m')?.n, 1);
|
||||
});
|
||||
|
||||
test('fresh 5m candles are not stale; old observed_at is', async () => {
|
||||
const { repo, scheduler } = setup();
|
||||
const five: PriceCandle[] = [{ ts: '2026-08-15T14:30:00.000Z', o: 1, h: 2, l: 1, c: 1.5, v: 10, adjClose: 1.5 }];
|
||||
await repo.set('yfinance:candles:NVDA:5m', five, 'daily_permanent', { fetchedAt: iso(-10 * 60_000), sourceKind: 'yfinance' });
|
||||
scheduler.reset();
|
||||
let e = await repo.get<PriceCandle[]>('yfinance:candles:NVDA:5m');
|
||||
assert.equal(e.isStale, true);
|
||||
assert.ok(scheduler.queued.includes('yfinance:candles:NVDA:5m'));
|
||||
await repo.set('yfinance:candles:NVDA:5m', five, 'daily_permanent', { fetchedAt: iso(), sourceKind: 'yfinance' });
|
||||
scheduler.reset();
|
||||
e = await repo.get<PriceCandle[]>('yfinance:candles:NVDA:5m');
|
||||
assert.equal(e.isStale, false);
|
||||
assert.equal(scheduler.queued.length, 0);
|
||||
});
|
||||
|
||||
test('subscribe does not enqueue 1m or 5m', async () => {
|
||||
const { repo, scheduler } = setup();
|
||||
await repo.subscribe('NVDA', 'equity');
|
||||
assert.ok(!scheduler.queued.some((k) => k.includes(':1m') || k.endsWith(':5m')));
|
||||
});
|
||||
|
||||
test('symbol_meta stale after 7d, fresh within', async () => {
|
||||
const { repo, scheduler } = setup();
|
||||
const s: SymbolMeta = { symbol: 'NVDA', name: 'NVIDIA', sector: 'Technology', tickerKind: 'equity' };
|
||||
|
||||
@@ -34,7 +34,7 @@ class FakeCache implements CacheRepository {
|
||||
async ensureInDemand(): Promise<void> {}
|
||||
async pinSystemSymbol(): Promise<void> {}
|
||||
async demandSet(): Promise<string[]> { return []; }
|
||||
async getMany<T>(): Promise<Array<{ key: string; value: T | null; isStale: boolean }>> { return []; }
|
||||
async getMany<T>(): Promise<Array<{ key: string; value: T | null; isStale: boolean; fetchedAt: string | null }>> { return []; }
|
||||
async del(): Promise<void> {}
|
||||
readonly db: never = undefined as never;
|
||||
}
|
||||
@@ -138,4 +138,20 @@ describe('CacheCandleProvider', () => {
|
||||
assert.deepEqual(r.candles, []);
|
||||
assert.match(r.asOf, /^\d{4}-\d{2}-\d{2}$/);
|
||||
});
|
||||
|
||||
it('resolveAsOf slices to the cutoff and never folds a live quote', async () => {
|
||||
const cache = new FakeCache()
|
||||
.setValue('yfinance:candles:SPY:1d', [
|
||||
bar('2026-08-07', 100),
|
||||
bar('2026-08-10', 105),
|
||||
bar('2026-08-11', 110),
|
||||
], false)
|
||||
.setValue('yfinance:quote:SPY', { symbol: 'SPY', price: 999 }, false);
|
||||
const r = await new CacheCandleProvider(cache).resolveAsOf('SPY', '1d', '2026-08-10');
|
||||
assert.equal(r.candles.length, 2);
|
||||
assert.equal(r.asOf, '2026-08-10');
|
||||
assert.equal(r.realtimeFolded, false);
|
||||
assert.equal(r.lastBar, 'yfinance');
|
||||
assert.equal(r.candles[r.candles.length - 1].c, 105);
|
||||
});
|
||||
});
|
||||
@@ -13,7 +13,7 @@ import { createCacheRepository, type CacheRepository, type CacheEntry, type Pric
|
||||
import { FakeSourceAdapter } from '../../adapters/SourceAdapter.ts';
|
||||
import { AdapterQueue } from '../../queue/AdapterQueue.ts';
|
||||
import { seedConfluence, CONFLUENCE_UNIVERSE, BENCHMARK_SYMBOL } from '../confluenceSeed.ts';
|
||||
import { runConfluenceEvaluationCycle } from '../confluenceEngine.ts';
|
||||
import { runConfluenceEvaluationCycle, runConfluenceReplay } from '../confluenceEngine.ts';
|
||||
|
||||
// ---- fake cache that returns whatever we seeded ---------------------------------
|
||||
|
||||
@@ -31,7 +31,7 @@ class FakeCache implements CacheRepository {
|
||||
async ensureInDemand(): Promise<void> {}
|
||||
async pinSystemSymbol(): Promise<void> {}
|
||||
async demandSet(): Promise<string[]> { return []; }
|
||||
async getMany<T>(): Promise<Array<{ key: string; value: T | null; isStale: boolean }>> { return []; }
|
||||
async getMany<T>(): Promise<Array<{ key: string; value: T | null; isStale: boolean; fetchedAt: string | null }>> { return []; }
|
||||
async del(): Promise<void> {}
|
||||
readonly db: never = undefined as never;
|
||||
}
|
||||
@@ -126,4 +126,31 @@ describe('runConfluenceEvaluationCycle', () => {
|
||||
assert.equal(summary.symbolsSkipped.length, 1);
|
||||
assert.equal(summary.symbolsSkipped[0].symbol, CONFLUENCE_UNIVERSE[0].symbol);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('runConfluenceReplay', () => {
|
||||
it('fills missing as-of evaluations from cached candles and is idempotent', async () => {
|
||||
const first = await runConfluenceReplay(db, cache as unknown as CacheRepository, {
|
||||
symbols: ['PLTR'],
|
||||
lookbackDays: 30,
|
||||
budgetDaysPerSymbol: 30,
|
||||
symbolsPerTick: 1,
|
||||
});
|
||||
assert.ok(first.evaluationsStored > 0, 'replay stored historical evaluations');
|
||||
assert.ok(first.symbolsTouched.includes('PLTR'));
|
||||
|
||||
const second = await runConfluenceReplay(db, cache as unknown as CacheRepository, {
|
||||
symbols: ['PLTR'],
|
||||
lookbackDays: 30,
|
||||
budgetDaysPerSymbol: 30,
|
||||
symbolsPerTick: 1,
|
||||
});
|
||||
assert.equal(second.evaluationsStored, 0);
|
||||
assert.equal(confluenceRepo.countEvaluations('PLTR', repoFullRack()), first.evaluationsStored / 3);
|
||||
});
|
||||
});
|
||||
|
||||
function repoFullRack(): string {
|
||||
const rack = confluenceRepo.listSystemRacks().find((r) => r.id === 'confluence-full');
|
||||
return rack?.id ?? confluenceRepo.listSystemRacks()[0]!.id;
|
||||
}
|
||||
@@ -123,6 +123,17 @@ describe('evaluateRack', () => {
|
||||
]);
|
||||
assert.equal(ev.quality, 'mixed');
|
||||
});
|
||||
|
||||
it('scales a fired slot by its reliability weight; missing weight stays 1.0', () => {
|
||||
const unweighted = evaluateRack('PLTR', '2026-01-05', [fired('goldenCross')]);
|
||||
assert.equal(unweighted.bullEvidence, 1);
|
||||
|
||||
const half = evaluateRack('PLTR', '2026-01-05', [fired('goldenCross')], new Map([['goldenCross', 0.5]]));
|
||||
assert.equal(half.bullEvidence, 0.5);
|
||||
|
||||
const boosted = evaluateRack('PLTR', '2026-01-05', [fired('goldenCross')], new Map([['goldenCross', 1.25]]));
|
||||
assert.equal(boosted.bullEvidence, 1.25);
|
||||
});
|
||||
});
|
||||
|
||||
describe('detectPictureChange', () => {
|
||||
|
||||
@@ -83,9 +83,10 @@ describe('defineSystemRackPresets', () => {
|
||||
});
|
||||
|
||||
describe('seedConfluence', () => {
|
||||
it('pins16 symbols (15 universe + benchmark) and creates 3 system racks', async () => {
|
||||
it('pins the research universe, learning sector ETFs, and benchmark, and creates 3 system racks', async () => {
|
||||
const { symbolsPinned, racksCreated } = await seedConfluence(db, cache);
|
||||
assert.equal(symbolsPinned, 16);
|
||||
// 11 GICS + SPY + 15 research names (SMH is extra; no other overlap)
|
||||
assert.equal(symbolsPinned, 27);
|
||||
assert.equal(racksCreated, 3);
|
||||
const systemRacks = repo.listSystemRacks();
|
||||
assert.equal(systemRacks.length, 3);
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import type { PriceCandle } from '../../cache/CacheRepository.ts';
|
||||
import type { PictureQuality, SlotAssessment } from '../confluenceRack.ts';
|
||||
import {
|
||||
attachOutcomes,
|
||||
collapseZones,
|
||||
deriveZoneRules,
|
||||
findOnsets,
|
||||
ONSET_EXTREME_15_SLIDE,
|
||||
ONSET_TURN_ON,
|
||||
lastNZones,
|
||||
matchRule,
|
||||
zoneOnsetWindow,
|
||||
ZONE_DISPLAY_BARS,
|
||||
recentZonesForSymbol,
|
||||
selectWinner,
|
||||
type EvaluationPoint,
|
||||
type ZonePredicate,
|
||||
} from '../confluenceZones.ts';
|
||||
|
||||
const fired = (id: string): SlotAssessment => ({ id, state: 'fired' });
|
||||
const notFired = (id: string): SlotAssessment => ({ id, state: 'not-fired' });
|
||||
|
||||
function ev(asOf: string, quality: PictureQuality, net = 2, extra: Partial<EvaluationPoint> = {}): EvaluationPoint {
|
||||
const bullish = quality.includes('bullish');
|
||||
return {
|
||||
symbol: extra.symbol ?? 'XLK',
|
||||
asOf,
|
||||
quality,
|
||||
netEvidence: extra.netEvidence ?? net,
|
||||
bullEvidence: extra.bullEvidence ?? (bullish ? Math.max(net, 2) : 0.2),
|
||||
bearEvidence: extra.bearEvidence ?? (bullish ? 0.2 : Math.max(-net, 2)),
|
||||
assessments: extra.assessments ?? [fired('goldenCross')],
|
||||
};
|
||||
}
|
||||
|
||||
function candles(start: string, days: number, startPx = 100, daily = 0.002): PriceCandle[] {
|
||||
const out: PriceCandle[] = [];
|
||||
const t0 = Date.parse(`${start}T00:00:00Z`);
|
||||
for (let i = 0; i < days; i++) {
|
||||
const ts = new Date(t0 + i * 86400000).toISOString().slice(0, 10);
|
||||
const c = startPx * Math.pow(1 + daily, i);
|
||||
out.push({ ts, o: c, h: c * 1.01, l: c * 0.99, c, v: 1_000_000, adjClose: c });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const entryPred: ZonePredicate = {
|
||||
id: 'entry-mod-strong',
|
||||
label: 'Moderate or strong bullish picture',
|
||||
qualities: ['moderate-bullish', 'strong-bullish'],
|
||||
};
|
||||
const exitPred: ZonePredicate = {
|
||||
id: 'exit-mod-strong',
|
||||
label: 'Moderate or strong bearish picture',
|
||||
qualities: ['moderate-bearish', 'strong-bearish'],
|
||||
};
|
||||
|
||||
describe('matchRule', () => {
|
||||
it('matches a quality band and rejects mixed', () => {
|
||||
assert.equal(matchRule(ev('2024-01-02', 'moderate-bullish'), entryPred), true);
|
||||
assert.equal(matchRule(ev('2024-01-02', 'mixed'), entryPred), false);
|
||||
});
|
||||
|
||||
it('requires a named slot to have fired when listed', () => {
|
||||
const pred: ZonePredicate = { id: 'p', label: 'pullback', qualities: ['moderate-bullish'], requireAnyFired: ['pullbackToEMA21'] };
|
||||
assert.equal(matchRule(ev('2024-01-02', 'moderate-bullish', 2, { assessments: [fired('goldenCross')] }), pred), false);
|
||||
assert.equal(matchRule(ev('2024-01-02', 'moderate-bullish', 2, { assessments: [fired('pullbackToEMA21')] }), pred), true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('collapseZones', () => {
|
||||
it('merges a one-session gap and splits a two-session gap', () => {
|
||||
const series = [
|
||||
ev('2024-01-02', 'moderate-bullish'),
|
||||
ev('2024-01-03', 'moderate-bullish'),
|
||||
ev('2024-01-04', 'mixed', 0),
|
||||
ev('2024-01-05', 'moderate-bullish'),
|
||||
ev('2024-01-08', 'mixed', 0),
|
||||
ev('2024-01-09', 'mixed', 0),
|
||||
ev('2024-01-10', 'moderate-bullish'),
|
||||
];
|
||||
const zones = collapseZones(series, 'entry', entryPred);
|
||||
assert.equal(zones.length, 2);
|
||||
assert.equal(zones[0].startAsOf, '2024-01-02');
|
||||
assert.equal(zones[0].endAsOf, '2024-01-05');
|
||||
assert.equal(zones[1].startAsOf, '2024-01-10');
|
||||
assert.equal(zones[1].endAsOf, '2024-01-10');
|
||||
});
|
||||
|
||||
it('closes an entry zone when the exit predicate fires', () => {
|
||||
const series = [
|
||||
ev('2024-01-02', 'moderate-bullish'),
|
||||
ev('2024-01-03', 'moderate-bullish'),
|
||||
ev('2024-01-04', 'moderate-bearish', -2, { assessments: [fired('deathCross')] }),
|
||||
ev('2024-01-05', 'moderate-bullish'),
|
||||
];
|
||||
const zones = collapseZones(series, 'entry', entryPred, exitPred);
|
||||
assert.equal(zones.length, 2);
|
||||
assert.equal(zones[0].endAsOf, '2024-01-03');
|
||||
assert.equal(zones[1].startAsOf, '2024-01-05');
|
||||
});
|
||||
|
||||
it('lastNZones keeps the newest windows', () => {
|
||||
const zones = collapseZones([
|
||||
ev('2024-01-02', 'moderate-bullish'),
|
||||
ev('2024-01-10', 'mixed', 0),
|
||||
ev('2024-01-11', 'mixed', 0),
|
||||
ev('2024-02-01', 'moderate-bullish'),
|
||||
ev('2024-02-10', 'mixed', 0),
|
||||
ev('2024-02-11', 'mixed', 0),
|
||||
ev('2024-03-01', 'strong-bullish'),
|
||||
], 'entry', entryPred);
|
||||
const last = lastNZones(zones, 2);
|
||||
assert.equal(last.length, 2);
|
||||
assert.equal(last[0].startAsOf, '2024-02-01');
|
||||
assert.equal(last[1].startAsOf, '2024-03-01');
|
||||
});
|
||||
});
|
||||
|
||||
describe('zoneOnsetWindow', () => {
|
||||
it('caps a long open stretch to the first 4 weeks from start', () => {
|
||||
const px = candles('2025-06-20', 400, 10, 0.01);
|
||||
const onset = zoneOnsetWindow('2025-06-20', '2026-08-12', px);
|
||||
assert.ok(onset);
|
||||
assert.equal(onset.startAsOf, '2025-06-20');
|
||||
const startIdx = px.findIndex((c) => c.ts.slice(0, 10) === '2025-06-20');
|
||||
const endIdx = px.findIndex((c) => c.ts.slice(0, 10) === onset.endAsOf);
|
||||
assert.equal(endIdx - startIdx + 1, ZONE_DISPLAY_BARS);
|
||||
assert.ok(onset.endAsOf < '2026-08-12');
|
||||
});
|
||||
|
||||
it('does not extend past a short natural end', () => {
|
||||
const px = candles('2025-06-04', 40, 10, 0.01);
|
||||
const onset = zoneOnsetWindow('2025-06-04', '2025-06-16', px);
|
||||
assert.ok(onset);
|
||||
assert.equal(onset.startAsOf, '2025-06-04');
|
||||
assert.equal(onset.endAsOf, '2025-06-16');
|
||||
});
|
||||
|
||||
it('returns null when the start bar is missing', () => {
|
||||
const px = candles('2025-07-01', 20, 10, 0);
|
||||
assert.equal(zoneOnsetWindow('2025-06-01', '2025-07-10', px), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findOnsets / pullback re-entry', () => {
|
||||
it('opens a second entry window on an N-session low inside a still-bullish picture', () => {
|
||||
const evals: EvaluationPoint[] = [];
|
||||
const px: PriceCandle[] = [];
|
||||
const start = Date.UTC(2025, 5, 2); // Jun 2
|
||||
let price = 20;
|
||||
for (let i = 0; i < 80; i++) {
|
||||
const asOf = new Date(start + i * 86400000).toISOString().slice(0, 10);
|
||||
// Ramp, then a 15-day dip around day 50, then ramp again.
|
||||
if (i < 40) price *= 1.01;
|
||||
else if (i < 55) price *= 0.985;
|
||||
else price *= 1.012;
|
||||
evals.push(ev(asOf, 'moderate-bullish', 2.5));
|
||||
px.push({ ts: asOf, o: price, h: price * 1.01, l: price * 0.99, c: price, v: 1_000_000, adjClose: price });
|
||||
}
|
||||
const onsets = findOnsets(evals, 'entry', entryPred, exitPred, px, ONSET_EXTREME_15_SLIDE);
|
||||
assert.ok(onsets.length >= 2, `expected a re-entry, got ${onsets.join(', ')}`);
|
||||
assert.equal(onsets[0], evals[0].asOf);
|
||||
// The second onset should sit in the dip, not on day 1.
|
||||
assert.ok(onsets[1] > evals[30].asOf);
|
||||
const turnOn = findOnsets(evals, 'entry', entryPred, exitPred, px, ONSET_TURN_ON);
|
||||
assert.equal(turnOn.length, 1);
|
||||
});
|
||||
|
||||
it('slides the onset to a deeper confirmed trough inside the quiet gap', () => {
|
||||
const evals: EvaluationPoint[] = [];
|
||||
const px: PriceCandle[] = [];
|
||||
const start = Date.UTC(2025, 5, 2);
|
||||
// 25 days: drift down to day 10, bounce, then a lower low at day 18, then up.
|
||||
const closes = [
|
||||
20, 19.5, 19.2, 18.8, 18.4, 18.0, 17.6, 17.2, 16.8, 16.4,
|
||||
16.0, 16.5, 16.8, 16.6, 16.2, 15.8, 15.4, 15.0, 14.4, 15.2,
|
||||
15.6, 16.0, 16.4, 16.8, 17.2,
|
||||
];
|
||||
for (let i = 0; i < closes.length; i++) {
|
||||
const asOf = new Date(start + i * 86400000).toISOString().slice(0, 10);
|
||||
const price = closes[i];
|
||||
evals.push(ev(asOf, 'moderate-bullish', 2));
|
||||
px.push({ ts: asOf, o: price, h: price, l: price, c: price, v: 1, adjClose: price });
|
||||
}
|
||||
const onsets = findOnsets(evals, 'entry', entryPred, exitPred, px, ONSET_EXTREME_15_SLIDE);
|
||||
assert.ok(onsets.length >= 1);
|
||||
// The first stretch slides to the deeper confirmed trough (14.4) before the bounce.
|
||||
assert.equal(onsets[onsets.length - 1], evals[18].asOf);
|
||||
});
|
||||
|
||||
it('does not re-enter without a quiet gap', () => {
|
||||
const evals: EvaluationPoint[] = [];
|
||||
const px: PriceCandle[] = [];
|
||||
const start = Date.UTC(2025, 5, 2);
|
||||
let price = 20;
|
||||
for (let i = 0; i < 12; i++) {
|
||||
const asOf = new Date(start + i * 86400000).toISOString().slice(0, 10);
|
||||
price *= i === 11 ? 0.9 : 1.01;
|
||||
evals.push(ev(asOf, 'moderate-bullish', 2));
|
||||
px.push({ ts: asOf, o: price, h: price, l: price, c: price, v: 1, adjClose: price });
|
||||
}
|
||||
const onsets = findOnsets(evals, 'entry', entryPred, exitPred, px, ONSET_EXTREME_15_SLIDE);
|
||||
assert.equal(onsets.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('attachOutcomes', () => {
|
||||
it('marks a rising stretch confirmed for an entry zone', () => {
|
||||
const series = [ev('2024-01-02', 'moderate-bullish')];
|
||||
const zones = attachOutcomes(collapseZones(series, 'entry', entryPred), candles('2024-01-02', 40, 100, 0.01));
|
||||
assert.equal(zones[0].verdict, 'confirmed');
|
||||
assert.ok((zones[0].forwardReturn ?? 0) > 0);
|
||||
});
|
||||
|
||||
it('leaves a late zone open when forward bars are missing', () => {
|
||||
const series = [ev('2024-01-20', 'moderate-bullish')];
|
||||
const zones = attachOutcomes(collapseZones(series, 'entry', entryPred), candles('2024-01-02', 25, 100, 0.01));
|
||||
assert.equal(zones[0].verdict, 'open');
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectWinner / deriveZoneRules', () => {
|
||||
it('refuses a noisy winner and falls back to the conservative quality band', () => {
|
||||
// Short mixed series: not enough resolved train zones.
|
||||
const evals: EvaluationPoint[] = [];
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const d = new Date(Date.UTC(2023, 0, 2 + i)).toISOString().slice(0, 10);
|
||||
evals.push(ev(d, i % 2 === 0 ? 'mixed' : 'sparse', 0));
|
||||
}
|
||||
const px = new Map([['XLK', candles('2023-01-02', 80, 100, 0)]]);
|
||||
const rule = selectWinner('entry', [{ id: 'entry-net-3', label: 'Net 3', minNetEvidence: 3 }], evals, px, entryPred);
|
||||
assert.equal(rule.source, 'baseline');
|
||||
assert.equal(rule.predicate.id, entryPred.id);
|
||||
});
|
||||
|
||||
it('picks the template that beats baseline on the held-out year', () => {
|
||||
const evals: EvaluationPoint[] = [];
|
||||
const px: PriceCandle[] = [];
|
||||
// Two years of daily points. Moderate-bullish days sit at local troughs
|
||||
// and are followed by a rise; other days drift flat then drop.
|
||||
let price = 100;
|
||||
const start = Date.UTC(2023, 0, 2);
|
||||
for (let i = 0; i < 520; i++) {
|
||||
const asOf = new Date(start + i * 86400000).toISOString().slice(0, 10);
|
||||
const isSignal = i % 25 === 0;
|
||||
if (isSignal) {
|
||||
evals.push(ev(asOf, 'moderate-bullish', 2.5, { assessments: [fired('goldenCross'), fired('pullbackToEMA21')] }));
|
||||
price = price * 0.98;
|
||||
} else {
|
||||
evals.push(ev(asOf, 'mixed', 0.2, { assessments: [notFired('goldenCross')] }));
|
||||
price = price * (i % 25 < 12 ? 1.008 : 0.997);
|
||||
}
|
||||
px.push({ ts: asOf, o: price, h: price * 1.01, l: price * 0.99, c: price, v: 1_000_000, adjClose: price });
|
||||
}
|
||||
const candlesBySymbol = new Map([['XLK', px]]);
|
||||
const { entry } = deriveZoneRules(evals, candlesBySymbol);
|
||||
// Either validated (if the synthetic series beat baseline by 10pp) or the
|
||||
// conservative fallback. Never a random high-threshold that never fires.
|
||||
assert.ok(entry.predicate.id === 'entry-mod-strong' || entry.source === 'validated' || entry.source === 'baseline');
|
||||
assert.ok(entry.predicate.qualities?.includes('moderate-bullish'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('recentZonesForSymbol', () => {
|
||||
it('returns the last few entry and exit windows with outcomes', () => {
|
||||
const evals = [
|
||||
ev('2024-01-02', 'moderate-bullish'),
|
||||
ev('2024-01-03', 'moderate-bullish'),
|
||||
ev('2024-03-01', 'moderate-bearish', -2, { assessments: [fired('deathCross')] }),
|
||||
ev('2024-03-02', 'moderate-bearish', -2, { assessments: [fired('deathCross')] }),
|
||||
ev('2024-06-03', 'strong-bullish', 4),
|
||||
];
|
||||
const { entries, exits } = recentZonesForSymbol(evals, candles('2024-01-02', 200, 100, 0.003), entryPred, exitPred, 3, 3);
|
||||
assert.ok(entries.length >= 1);
|
||||
assert.ok(exits.length >= 1);
|
||||
assert.equal(entries[entries.length - 1].startAsOf, '2024-06-03');
|
||||
assert.equal(exits[0].startAsOf, '2024-03-01');
|
||||
});
|
||||
});
|
||||
@@ -53,6 +53,11 @@ export interface CandleResolution {
|
||||
*/
|
||||
export interface CandleProvider {
|
||||
resolve(symbol: string, granularity: SlotGranularity): Promise<CandleResolution>;
|
||||
/**
|
||||
* Honest historical cut: candles with ts <= asOf only. Never folds a live quote.
|
||||
* Used by replay so a past evaluation cannot see tomorrow's close.
|
||||
*/
|
||||
resolveAsOf(symbol: string, granularity: SlotGranularity, asOf: string): Promise<CandleResolution>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -173,4 +178,22 @@ export class CacheCandleProvider implements CandleProvider {
|
||||
isStale,
|
||||
};
|
||||
}
|
||||
|
||||
async resolveAsOf(symbol: string, granularity: SlotGranularity, asOf: string): Promise<CandleResolution> {
|
||||
const sym = symbol.toUpperCase();
|
||||
const cutoff = asOf.slice(0, 10);
|
||||
const entry = await this._cache.get<PriceCandle[]>(`yfinance:candles:${sym}:${granularity}`);
|
||||
const stored = (entry?.value ?? []).slice();
|
||||
const candles = stored.filter((c) => (c.ts ?? '').slice(0, 10) <= cutoff);
|
||||
const lastTs = candles.length > 0 ? (candles[candles.length - 1].ts ?? '').slice(0, 10) : '';
|
||||
return {
|
||||
symbol: sym,
|
||||
granularity,
|
||||
candles,
|
||||
asOf: lastTs || cutoff,
|
||||
lastBar: 'yfinance',
|
||||
realtimeFolded: false,
|
||||
isStale: entry?.isStale ?? true,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -30,8 +30,25 @@ import { buildSeasonalitySnapshot } from '../analysis/seasonality.ts';
|
||||
import { resolveCorridorSnapshot } from './corridorData.ts';
|
||||
import { evaluateRack, type ConfluenceEvaluation, type SlotAssessment } from './confluenceRack.ts';
|
||||
import { ConfluenceRepository, type ConfluenceRack } from '../db/confluenceRepository.ts';
|
||||
import { resolveSignalHistory } from './confluenceBacktest.ts';
|
||||
import { CONFLUENCE_UNIVERSE, BENCHMARK_SYMBOL } from './confluenceSeed.ts';
|
||||
import { resolveSignalHistory, signalHistoryToStats } from './confluenceBacktest.ts';
|
||||
import {
|
||||
CONFLUENCE_UNIVERSE,
|
||||
BENCHMARK_SYMBOL,
|
||||
CONFLUENCE_LEARNING_UNIVERSE,
|
||||
REPLAY_LOOKBACK_DAYS,
|
||||
} from './confluenceSeed.ts';
|
||||
import {
|
||||
deriveZoneRules,
|
||||
FALLBACK_ENTRY,
|
||||
FALLBACK_EXIT,
|
||||
recentZonesForSymbol,
|
||||
attachOutcomes,
|
||||
collapseZones,
|
||||
scoreZones,
|
||||
type EvaluationPoint,
|
||||
type ZonePredicate,
|
||||
type ZoneRuleStats,
|
||||
} from './confluenceZones.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unwired slots → honest fallback assessments
|
||||
@@ -155,7 +172,8 @@ export async function runConfluenceEvaluationCycle(
|
||||
continue;
|
||||
}
|
||||
|
||||
const evaluation: ConfluenceEvaluation = evaluateRack(symbol, asOf, sliced);
|
||||
const weights = reliabilityWeights(db);
|
||||
const evaluation: ConfluenceEvaluation = evaluateRack(symbol, asOf, sliced, weights);
|
||||
const firesLogged = evaluation.assessments.filter((a) => a.state === 'fired').length;
|
||||
repo.saveEvaluation(evaluation, rack.id, randomUUID());
|
||||
repo.logSignalFires(evaluation, rack.id);
|
||||
@@ -185,4 +203,307 @@ function corridorUnavailableAssessments(): SlotAssessment[] {
|
||||
'corridorUpsideHigh', 'corridorUpsideLow',
|
||||
'spyCorridorCheap', 'spyCorridorStretched',
|
||||
].map((id) => ({ id, state: 'not-fired', note: 'No valuation-corridor snapshot cached for this symbol yet.' }));
|
||||
}
|
||||
|
||||
export const REPLAY_DAYS_PER_SYMBOL = 40;
|
||||
export const REPLAY_SYMBOLS_PER_TICK = 3;
|
||||
export const LEARNING_REPLAY_DAYS = 120;
|
||||
export const LEARNING_REPLAY_SYMBOLS = 6;
|
||||
export const LEARNING_MIN_DAYS = 400;
|
||||
|
||||
export interface ReplayOptions {
|
||||
symbols?: string[];
|
||||
lookbackDays?: number;
|
||||
budgetDaysPerSymbol?: number;
|
||||
symbolsPerTick?: number;
|
||||
/** When true, do not apply live reliability weights (historical ledger stays raw). */
|
||||
unweighted?: boolean;
|
||||
}
|
||||
|
||||
export interface ReplayRunSummary {
|
||||
symbolsTouched: string[];
|
||||
evaluationsStored: number;
|
||||
signalsLogged: number;
|
||||
remainingDays: number;
|
||||
complete: boolean;
|
||||
}
|
||||
|
||||
function reliabilityWeights(db: DatabaseSync): Map<string, number> {
|
||||
const stats = signalHistoryToStats(db);
|
||||
// Unresolved / thin samples stay at the rack default (1.0). Only apply a
|
||||
// learned weight once the slot has enough follow-through to score.
|
||||
return new Map(stats.filter((s) => s.resolvedFires >= 2).map((s) => [s.slotId, s.reliabilityWeight]));
|
||||
}
|
||||
|
||||
function toPoint(ev: { symbol: string; asOf: string; quality: import('./confluenceRack.ts').PictureQuality; netEvidence: number; bullEvidence: number; bearEvidence: number; assessments: SlotAssessment[] }): EvaluationPoint {
|
||||
return {
|
||||
symbol: ev.symbol,
|
||||
asOf: ev.asOf,
|
||||
quality: ev.quality,
|
||||
netEvidence: ev.netEvidence,
|
||||
bullEvidence: ev.bullEvidence,
|
||||
bearEvidence: ev.bearEvidence,
|
||||
assessments: ev.assessments,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay missing as-of evaluations from cached candles. Honest cut: no live
|
||||
* quote fold-in, no today's corridor snapshot applied to a past date.
|
||||
*/
|
||||
export async function runConfluenceReplay(
|
||||
db: DatabaseSync,
|
||||
cache: CacheRepository,
|
||||
opts: ReplayOptions = {},
|
||||
): Promise<ReplayRunSummary> {
|
||||
const repo = new ConfluenceRepository(db);
|
||||
const provider: CandleProvider = new CacheCandleProvider(cache);
|
||||
const racks = repo.listSystemRacks();
|
||||
if (racks.length === 0) throw new Error('confluence replay: no system racks seeded');
|
||||
|
||||
const lookback = opts.lookbackDays ?? REPLAY_LOOKBACK_DAYS;
|
||||
const budget = opts.budgetDaysPerSymbol ?? REPLAY_DAYS_PER_SYMBOL;
|
||||
const symbolCap = opts.symbolsPerTick ?? REPLAY_SYMBOLS_PER_TICK;
|
||||
const symbols = (opts.symbols ?? [...CONFLUENCE_LEARNING_UNIVERSE])
|
||||
.map((s) => s.toUpperCase());
|
||||
const unique = [...new Set(symbols)];
|
||||
|
||||
const summary: ReplayRunSummary = {
|
||||
symbolsTouched: [],
|
||||
evaluationsStored: 0,
|
||||
signalsLogged: 0,
|
||||
remainingDays: 0,
|
||||
complete: true,
|
||||
};
|
||||
|
||||
let symbolsUsed = 0;
|
||||
for (const symbol of unique) {
|
||||
if (symbolsUsed >= symbolCap) {
|
||||
summary.complete = false;
|
||||
break;
|
||||
}
|
||||
const daily = await provider.resolveAsOf(symbol, '1d', '9999-12-31');
|
||||
if (daily.candles.length === 0) continue;
|
||||
|
||||
const dates = daily.candles.map((c) => (c.ts ?? '').slice(0, 10)).filter(Boolean);
|
||||
const window = dates.slice(Math.max(0, dates.length - lookback));
|
||||
const primaryRack = racks[0];
|
||||
const have = new Set(repo.listEvaluationAsOfs(symbol, primaryRack.id));
|
||||
const missing = window.filter((d) => !have.has(d));
|
||||
summary.remainingDays += missing.length;
|
||||
if (missing.length === 0) continue;
|
||||
|
||||
const slice = missing.slice(0, budget);
|
||||
if (missing.length > slice.length) summary.complete = false;
|
||||
symbolsUsed += 1;
|
||||
summary.symbolsTouched.push(symbol);
|
||||
|
||||
const weeklyFull = await provider.resolveAsOf(symbol, '1wk', '9999-12-31');
|
||||
const benchFull = await provider.resolveAsOf(BENCHMARK_SYMBOL, '1d', '9999-12-31');
|
||||
|
||||
for (const asOf of slice) {
|
||||
const dailyCut = daily.candles.filter((c) => (c.ts ?? '').slice(0, 10) <= asOf);
|
||||
const weeklyCut = weeklyFull.candles.filter((c) => (c.ts ?? '').slice(0, 10) <= asOf);
|
||||
const benchCut = benchFull.candles.filter((c) => (c.ts ?? '').slice(0, 10) <= asOf);
|
||||
if (dailyCut.length === 0) continue;
|
||||
|
||||
const wired: SlotAssessment[] = [
|
||||
...evaluateTechnicalSlots(symbol, dailyCut, { weekly: weeklyCut, benchmarkDaily: benchCut }),
|
||||
...evaluateSeasonalSlots(buildSeasonalitySnapshot(symbol, dailyCut), asOf),
|
||||
...corridorUnavailableAssessments(),
|
||||
];
|
||||
const assessments = [...wired, ...unwiredFallbacks(wired)];
|
||||
|
||||
for (const rack of racks) {
|
||||
if (repo.getEvaluation(symbol, asOf, rack.id)) continue;
|
||||
const rackSlots = new Set(rack.slotIds);
|
||||
const sliced = assessments.filter((a) => rackSlots.has(a.id));
|
||||
if (sliced.length === 0) continue;
|
||||
const evaluation: ConfluenceEvaluation = evaluateRack(symbol, asOf, sliced);
|
||||
repo.saveEvaluation(evaluation, rack.id, randomUUID());
|
||||
repo.logSignalFires(evaluation, rack.id);
|
||||
summary.evaluationsStored += 1;
|
||||
summary.signalsLogged += evaluation.assessments.filter((a) => a.state === 'fired').length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await resolveSignalHistory(db, async (sym) => {
|
||||
const res = await provider.resolveAsOf(sym.toUpperCase(), '1d', '9999-12-31');
|
||||
return res.candles;
|
||||
});
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
export interface LearningLedgerStatus {
|
||||
ready: boolean;
|
||||
symbols: Array<{ symbol: string; evaluatedDays: number; targetDays: number; ready: boolean }>;
|
||||
}
|
||||
|
||||
export async function learningLedgerStatus(
|
||||
db: DatabaseSync,
|
||||
cache: CacheRepository,
|
||||
lookback = REPLAY_LOOKBACK_DAYS,
|
||||
): Promise<LearningLedgerStatus> {
|
||||
const repo = new ConfluenceRepository(db);
|
||||
const rack = repo.listSystemRacks()[0];
|
||||
const provider = new CacheCandleProvider(cache);
|
||||
const symbols: LearningLedgerStatus['symbols'] = [];
|
||||
if (!rack) return { ready: false, symbols };
|
||||
for (const symbol of CONFLUENCE_LEARNING_UNIVERSE) {
|
||||
const n = repo.countEvaluations(symbol, rack.id);
|
||||
const daily = await provider.resolveAsOf(symbol, '1d', '9999-12-31');
|
||||
const available = daily.candles.length;
|
||||
const target = Math.min(lookback, available > 0 ? available : lookback);
|
||||
const ready = n >= Math.min(target, LEARNING_MIN_DAYS) || (target < LEARNING_MIN_DAYS && n >= target);
|
||||
symbols.push({ symbol, evaluatedDays: n, targetDays: target, ready });
|
||||
}
|
||||
return { ready: symbols.every((s) => s.ready), symbols };
|
||||
}
|
||||
|
||||
export async function fillLearningLedger(
|
||||
db: DatabaseSync,
|
||||
cache: CacheRepository,
|
||||
maxMs = 90_000,
|
||||
): Promise<ReplayRunSummary & { ledger: LearningLedgerStatus }> {
|
||||
const started = Date.now();
|
||||
let last: ReplayRunSummary = {
|
||||
symbolsTouched: [],
|
||||
evaluationsStored: 0,
|
||||
signalsLogged: 0,
|
||||
remainingDays: 0,
|
||||
complete: false,
|
||||
};
|
||||
while (Date.now() - started < maxMs) {
|
||||
const status = await learningLedgerStatus(db, cache);
|
||||
if (status.ready) return { ...last, complete: true, ledger: status };
|
||||
const batch = await runConfluenceReplay(db, cache, {
|
||||
symbols: CONFLUENCE_LEARNING_UNIVERSE,
|
||||
symbolsPerTick: 12,
|
||||
budgetDaysPerSymbol: 80,
|
||||
});
|
||||
last = {
|
||||
symbolsTouched: [...new Set([...last.symbolsTouched, ...batch.symbolsTouched])],
|
||||
evaluationsStored: last.evaluationsStored + batch.evaluationsStored,
|
||||
signalsLogged: last.signalsLogged + batch.signalsLogged,
|
||||
remainingDays: batch.remainingDays,
|
||||
complete: batch.complete,
|
||||
};
|
||||
if (batch.evaluationsStored === 0) break;
|
||||
}
|
||||
return { ...last, ledger: await learningLedgerStatus(db, cache) };
|
||||
}
|
||||
|
||||
export async function deriveAndPersistZoneRules(db: DatabaseSync, cache: CacheRepository, rackId?: string): Promise<{ entryId: string; exitId: string; source: string; skipped?: string }> {
|
||||
const repo = new ConfluenceRepository(db);
|
||||
const rack = rackId
|
||||
? repo.getRack(rackId)
|
||||
: repo.listSystemRacks()[0] ?? null;
|
||||
if (!rack) throw new Error('confluence zone rules: no rack');
|
||||
const ledger = await learningLedgerStatus(db, cache);
|
||||
if (!ledger.ready) {
|
||||
return { entryId: '', exitId: '', source: 'deferred', skipped: 'learning ledger is not full enough to derive' };
|
||||
}
|
||||
|
||||
const evals: EvaluationPoint[] = [];
|
||||
const candlesBySymbol = new Map<string, import('../cache/CacheRepository.ts').PriceCandle[]>();
|
||||
const provider = new CacheCandleProvider(cache);
|
||||
|
||||
for (const symbol of CONFLUENCE_LEARNING_UNIVERSE) {
|
||||
const rows = repo.listEvaluationsForSymbolRack(symbol, rack.id);
|
||||
for (const r of rows) evals.push(toPoint(r));
|
||||
const daily = await provider.resolveAsOf(symbol, '1d', '9999-12-31');
|
||||
candlesBySymbol.set(symbol, daily.candles);
|
||||
}
|
||||
|
||||
const derived = deriveZoneRules(evals, candlesBySymbol);
|
||||
const caveat = (rule: typeof derived.entry) =>
|
||||
rule.source === 'baseline'
|
||||
? 'Held-out year did not beat the unconditional baseline by 10 percentage points. Showing the conservative quality-band rule.'
|
||||
: `Validated on the most recent year. Train resolved windows: ${rule.train.resolved}.`;
|
||||
|
||||
const entry = repo.saveActiveZoneRule({
|
||||
rackId: rack.id,
|
||||
kind: 'entry',
|
||||
derivedAt: new Date().toISOString(),
|
||||
predicateJson: JSON.stringify(derived.entry.predicate),
|
||||
trainStatsJson: JSON.stringify(derived.entry.train),
|
||||
validateStatsJson: JSON.stringify(derived.entry.validate),
|
||||
sampleCaveat: caveat(derived.entry),
|
||||
source: derived.entry.source,
|
||||
});
|
||||
const exit = repo.saveActiveZoneRule({
|
||||
rackId: rack.id,
|
||||
kind: 'exit',
|
||||
derivedAt: new Date().toISOString(),
|
||||
predicateJson: JSON.stringify(derived.exit.predicate),
|
||||
trainStatsJson: JSON.stringify(derived.exit.train),
|
||||
validateStatsJson: JSON.stringify(derived.exit.validate),
|
||||
sampleCaveat: caveat(derived.exit),
|
||||
source: derived.exit.source,
|
||||
});
|
||||
return { entryId: entry.id, exitId: exit.id, source: `${derived.entry.source}/${derived.exit.source}` };
|
||||
}
|
||||
|
||||
export function replayCoverage(
|
||||
db: DatabaseSync,
|
||||
symbol: string,
|
||||
rackId: string,
|
||||
lookback = REPLAY_LOOKBACK_DAYS,
|
||||
availableDays?: number,
|
||||
): { evaluatedDays: number; lookbackDays: number; replayComplete: boolean } {
|
||||
const repo = new ConfluenceRepository(db);
|
||||
const n = repo.countEvaluations(symbol.toUpperCase(), rackId);
|
||||
const target = Math.min(lookback, availableDays && availableDays > 0 ? availableDays : lookback);
|
||||
return {
|
||||
evaluatedDays: n,
|
||||
lookbackDays: target,
|
||||
replayComplete: n >= target,
|
||||
};
|
||||
}
|
||||
|
||||
export function loadActivePredicates(db: DatabaseSync, rackId: string): { entry: ZonePredicate; exit: ZonePredicate; entryRule: ReturnType<ConfluenceRepository['getActiveZoneRule']>; exitRule: ReturnType<ConfluenceRepository['getActiveZoneRule']> } {
|
||||
const repo = new ConfluenceRepository(db);
|
||||
const entryRule = repo.getActiveZoneRule(rackId, 'entry');
|
||||
const exitRule = repo.getActiveZoneRule(rackId, 'exit');
|
||||
const parse = (raw: string | undefined, fallback: ZonePredicate): ZonePredicate => {
|
||||
if (!raw) return fallback;
|
||||
try { return JSON.parse(raw) as ZonePredicate; } catch { return fallback; }
|
||||
};
|
||||
return {
|
||||
entry: parse(entryRule?.predicateJson, FALLBACK_ENTRY),
|
||||
exit: parse(exitRule?.predicateJson, FALLBACK_EXIT),
|
||||
entryRule,
|
||||
exitRule,
|
||||
};
|
||||
}
|
||||
|
||||
export function scoreSymbolUnderPrior(
|
||||
db: DatabaseSync,
|
||||
symbol: string,
|
||||
rackId: string,
|
||||
candles: import('../cache/CacheRepository.ts').PriceCandle[],
|
||||
kind: 'entry' | 'exit',
|
||||
predicate: ZonePredicate,
|
||||
opposite: ZonePredicate,
|
||||
): ZoneRuleStats {
|
||||
const repo = new ConfluenceRepository(db);
|
||||
const evals = repo.listEvaluationsForSymbolRack(symbol.toUpperCase(), rackId).map(toPoint);
|
||||
const zones = attachOutcomes(collapseZones(evals, kind, predicate, opposite, candles), candles);
|
||||
return scoreZones(zones, null);
|
||||
}
|
||||
|
||||
export function buildRecentZones(
|
||||
db: DatabaseSync,
|
||||
symbol: string,
|
||||
rackId: string,
|
||||
candles: import('../cache/CacheRepository.ts').PriceCandle[],
|
||||
entries = 3,
|
||||
exits = 3,
|
||||
) {
|
||||
const repo = new ConfluenceRepository(db);
|
||||
const evals = repo.listEvaluationsForSymbolRack(symbol.toUpperCase(), rackId).map(toPoint);
|
||||
const { entry, exit } = loadActivePredicates(db, rackId);
|
||||
return recentZonesForSymbol(evals, candles, entry, exit, entries, exits);
|
||||
}
|
||||
@@ -57,12 +57,13 @@ export function redundancyGroupFor(slotId: string): RedundancyGroup | undefined
|
||||
* `REDUNDANCY_DISCOUNT` of the previous marginal — so N same-group firings
|
||||
* weigh 1 + 0.5 + 0.25 + ... , never N.
|
||||
*/
|
||||
export function groupWeight(firingSlotIds: string[]): number {
|
||||
export function groupWeight(firingSlotIds: string[], weights?: ReadonlyMap<string, number>): number {
|
||||
if (firingSlotIds.length === 0) return 0;
|
||||
let w = 0;
|
||||
let margin = 1;
|
||||
for (let i = 0; i < firingSlotIds.length; i++) {
|
||||
w += margin;
|
||||
for (const id of firingSlotIds) {
|
||||
const slotW = weights?.get(id) ?? 1;
|
||||
w += margin * slotW;
|
||||
margin *= REDUNDANCY_DISCOUNT;
|
||||
}
|
||||
return w;
|
||||
|
||||
@@ -101,6 +101,7 @@ export function evaluateRack(
|
||||
symbol: string,
|
||||
asOf: string,
|
||||
assessments: SlotAssessment[],
|
||||
weights?: ReadonlyMap<string, number>,
|
||||
): ConfluenceEvaluation {
|
||||
const slotById = new Map(CONFLUENCE_SLOTS.map((s) => [s.id, s]));
|
||||
|
||||
@@ -139,9 +140,9 @@ export function evaluateRack(
|
||||
}
|
||||
|
||||
let bullEvidence = 0;
|
||||
for (const ids of bullGroups.values()) bullEvidence += groupWeight(ids);
|
||||
for (const ids of bullGroups.values()) bullEvidence += groupWeight(ids, weights);
|
||||
let bearEvidence = 0;
|
||||
for (const ids of bearGroups.values()) bearEvidence += groupWeight(ids);
|
||||
for (const ids of bearGroups.values()) bearEvidence += groupWeight(ids, weights);
|
||||
|
||||
const netEvidence = bullEvidence - bearEvidence;
|
||||
const totalEvidence = bullEvidence + bearEvidence;
|
||||
|
||||
@@ -47,6 +47,13 @@ export const CONFLUENCE_UNIVERSE: ConfluenceSymbol[] = [
|
||||
|
||||
export const BENCHMARK_SYMBOL = 'SPY';
|
||||
|
||||
/** 11 GICS sector ETFs + SPY. Universe-level zone rules are learned here only. */
|
||||
export const CONFLUENCE_LEARNING_UNIVERSE: string[] = [
|
||||
'XLK', 'XLF', 'XLE', 'XLI', 'XLV', 'XLY', 'XLP', 'XLU', 'XLRE', 'XLC', 'XLB', BENCHMARK_SYMBOL,
|
||||
];
|
||||
|
||||
export const REPLAY_LOOKBACK_DAYS = 756;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// System rack presets
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -110,10 +117,18 @@ export async function seedConfluence(
|
||||
cache: CacheRepository,
|
||||
): Promise<{ symbolsPinned: number; racksCreated: number }> {
|
||||
// 1. Pin universe symbols + benchmark into the permanent demand set.
|
||||
const allSymbols: ConfluenceSymbol[] = [
|
||||
...CONFLUENCE_UNIVERSE,
|
||||
{ symbol: BENCHMARK_SYMBOL, kind: 'etf', label: 'S&P 500 benchmark' },
|
||||
];
|
||||
const learning: ConfluenceSymbol[] = CONFLUENCE_LEARNING_UNIVERSE.map((symbol) => ({
|
||||
symbol,
|
||||
kind: 'etf' as TickerKind,
|
||||
label: symbol,
|
||||
}));
|
||||
const seen = new Set<string>();
|
||||
const allSymbols: ConfluenceSymbol[] = [];
|
||||
for (const row of [...learning, ...CONFLUENCE_UNIVERSE, { symbol: BENCHMARK_SYMBOL, kind: 'etf' as TickerKind, label: 'S&P 500 benchmark' }]) {
|
||||
if (seen.has(row.symbol)) continue;
|
||||
seen.add(row.symbol);
|
||||
allSymbols.push(row);
|
||||
}
|
||||
let symbolsPinned = 0;
|
||||
for (const { symbol, kind } of allSymbols) {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
// Investor Flow — Confluence entry/exit zones (derived from picture follow-through)
|
||||
//
|
||||
// Zones are contiguous stretches of rack evaluations that match a small, named
|
||||
// predicate. Predicates are scored walk-forward against 4-week price
|
||||
// follow-through. The winner becomes the active rule; last-N matching windows
|
||||
// are what the Confluence page shows.
|
||||
//
|
||||
// Pure: no I/O. ADR-0007: a zone describes a historical window, never a directive.
|
||||
|
||||
import type { PriceCandle } from '../cache/CacheRepository.ts';
|
||||
import { candleIndexForDate, checkFollowThrough } from './confluenceBacktest.ts';
|
||||
import { confluenceslotById } from './confluenceSlots.ts';
|
||||
import type { PictureQuality, SlotAssessment } from './confluenceRack.ts';
|
||||
|
||||
export type ZoneKind = 'entry' | 'exit';
|
||||
|
||||
export interface EvaluationPoint {
|
||||
symbol: string;
|
||||
asOf: string;
|
||||
quality: PictureQuality;
|
||||
netEvidence: number;
|
||||
bullEvidence: number;
|
||||
bearEvidence: number;
|
||||
assessments: SlotAssessment[];
|
||||
}
|
||||
|
||||
export interface OnsetStyle {
|
||||
lookback: number | null;
|
||||
slide: boolean;
|
||||
}
|
||||
|
||||
export const ONSET_TURN_ON: OnsetStyle = { lookback: null, slide: false };
|
||||
export const ONSET_EXTREME_15_SLIDE: OnsetStyle = { lookback: 15, slide: true };
|
||||
|
||||
export const ONSET_STYLE_CATALOG: Array<{ id: string; label: string; style: OnsetStyle }> = [
|
||||
{ id: 'turn-on', label: 'turn-on only', style: ONSET_TURN_ON },
|
||||
{ id: 'extreme-10', label: '10-session extreme', style: { lookback: 10, slide: false } },
|
||||
{ id: 'extreme-15', label: '15-session extreme', style: { lookback: 15, slide: false } },
|
||||
{ id: 'extreme-15-slide', label: '15-session extreme, slide to deeper trough', style: ONSET_EXTREME_15_SLIDE },
|
||||
{ id: 'extreme-20', label: '20-session extreme', style: { lookback: 20, slide: false } },
|
||||
{ id: 'extreme-20-slide', label: '20-session extreme, slide to deeper trough', style: { lookback: 20, slide: true } },
|
||||
];
|
||||
|
||||
export interface ZonePredicate {
|
||||
id: string;
|
||||
label: string;
|
||||
qualities?: PictureQuality[];
|
||||
minNetEvidence?: number;
|
||||
maxNetEvidence?: number;
|
||||
minBullEvidence?: number;
|
||||
minBearOrExitFired?: number;
|
||||
requireAnyFired?: string[];
|
||||
onsetId?: string;
|
||||
onsetLookback?: number | null;
|
||||
onsetSlide?: boolean;
|
||||
}
|
||||
|
||||
export interface ConfluenceZone {
|
||||
kind: ZoneKind;
|
||||
startAsOf: string;
|
||||
endAsOf: string;
|
||||
open: boolean;
|
||||
qualityAtStart: PictureQuality;
|
||||
netEvidenceAtStart: number;
|
||||
firedSlotIds: string[];
|
||||
forwardReturn: number | null;
|
||||
verdict: 'confirmed' | 'false_alarm' | 'open';
|
||||
}
|
||||
|
||||
export interface ZoneRuleStats {
|
||||
zones: number;
|
||||
resolved: number;
|
||||
confirmed: number;
|
||||
hitRate: number | null;
|
||||
avgReturn: number | null;
|
||||
baselineHitRate: number | null;
|
||||
}
|
||||
|
||||
export interface DerivedZoneRule {
|
||||
kind: ZoneKind;
|
||||
predicate: ZonePredicate;
|
||||
source: 'validated' | 'baseline';
|
||||
train: ZoneRuleStats;
|
||||
validate: ZoneRuleStats;
|
||||
}
|
||||
|
||||
export const MIN_TRAIN_ZONES = 8;
|
||||
export const MIN_VALIDATE_BEAT = 0.10;
|
||||
export const ZONE_HORIZON_WEEKS = 4;
|
||||
/** Bars painted from zone start. Matches the 4-week follow-through window. */
|
||||
export const ZONE_DISPLAY_BARS = ZONE_HORIZON_WEEKS * 5;
|
||||
/** A re-entry needs this many sessions of lower/higher closes behind it. */
|
||||
export const REENTRY_LOOKBACK = 15;
|
||||
/** Do not open another window inside this many evaluations of the last onset. */
|
||||
export const REENTRY_MIN_GAP = 20;
|
||||
|
||||
const BULLISH: PictureQuality[] = ['weak-bullish', 'moderate-bullish', 'strong-bullish'];
|
||||
const STRONG_BULL: PictureQuality[] = ['moderate-bullish', 'strong-bullish'];
|
||||
const BEARISH: PictureQuality[] = ['weak-bearish', 'moderate-bearish', 'strong-bearish'];
|
||||
const STRONG_BEAR: PictureQuality[] = ['moderate-bearish', 'strong-bearish'];
|
||||
|
||||
export const ENTRY_TEMPLATES: ZonePredicate[] = [
|
||||
{ id: 'entry-mod-strong', label: 'Moderate or strong bullish picture', qualities: STRONG_BULL },
|
||||
{ id: 'entry-any-bull', label: 'Any bullish picture', qualities: BULLISH },
|
||||
{ id: 'entry-net-1', label: 'Net evidence at least 1.0', minNetEvidence: 1 },
|
||||
{ id: 'entry-net-2', label: 'Net evidence at least 2.0', minNetEvidence: 2 },
|
||||
{ id: 'entry-net-3', label: 'Net evidence at least 3.0', minNetEvidence: 3 },
|
||||
{ id: 'entry-bull-2', label: 'Bull evidence at least 2.0', minBullEvidence: 2 },
|
||||
{ id: 'entry-bull-3', label: 'Bull evidence at least 3.0', minBullEvidence: 3 },
|
||||
{ id: 'entry-bull-4', label: 'Bull evidence at least 4.0', minBullEvidence: 4 },
|
||||
{ id: 'entry-mod-net2', label: 'Moderate or strong bullish picture and net evidence at least 2.0', qualities: STRONG_BULL, minNetEvidence: 2 },
|
||||
{ id: 'entry-mod-cheap', label: 'Moderate or strong bullish picture and corridor in the cheap band', qualities: STRONG_BULL, requireAnyFired: ['corridorEntryCheap'] },
|
||||
{ id: 'entry-mod-pullback', label: 'Moderate or strong bullish picture and a pullback toward EMA-21', qualities: STRONG_BULL, requireAnyFired: ['pullbackToEMA21'] },
|
||||
];
|
||||
|
||||
export const EXIT_TEMPLATES: ZonePredicate[] = [
|
||||
{ id: 'exit-mod-strong', label: 'Moderate or strong bearish picture', qualities: STRONG_BEAR },
|
||||
{ id: 'exit-any-bear', label: 'Any bearish picture', qualities: BEARISH },
|
||||
{ id: 'exit-net-m1', label: 'Net evidence at most -1.0', maxNetEvidence: -1 },
|
||||
{ id: 'exit-net-m2', label: 'Net evidence at most -2.0', maxNetEvidence: -2 },
|
||||
{ id: 'exit-fired-1', label: 'At least one exit or bear slot fired', minBearOrExitFired: 1 },
|
||||
{ id: 'exit-fired-2', label: 'At least two exit or bear slots fired', minBearOrExitFired: 2 },
|
||||
{ id: 'exit-mod-stretch', label: 'Moderate or strong bearish picture and a stretch marker', qualities: STRONG_BEAR, requireAnyFired: ['deathCross', 'rsiOverbought', 'corridorEntryStretched'] },
|
||||
];
|
||||
|
||||
export const FALLBACK_ENTRY = ENTRY_TEMPLATES[0];
|
||||
export const FALLBACK_EXIT = EXIT_TEMPLATES[0];
|
||||
|
||||
export function matchRule(ev: EvaluationPoint, predicate: ZonePredicate): boolean {
|
||||
if (predicate.qualities && !predicate.qualities.includes(ev.quality)) return false;
|
||||
if (predicate.minNetEvidence != null && ev.netEvidence < predicate.minNetEvidence) return false;
|
||||
if (predicate.maxNetEvidence != null && ev.netEvidence > predicate.maxNetEvidence) return false;
|
||||
if (predicate.minBullEvidence != null && ev.bullEvidence < predicate.minBullEvidence) return false;
|
||||
if (predicate.minBearOrExitFired != null) {
|
||||
const n = ev.assessments.filter((a) => {
|
||||
if (a.state !== 'fired') return false;
|
||||
const body = confluenceslotById.get(a.id)?.body;
|
||||
return body === 'bear' || body === 'exit';
|
||||
}).length;
|
||||
if (n < predicate.minBearOrExitFired) return false;
|
||||
}
|
||||
if (predicate.requireAnyFired && predicate.requireAnyFired.length > 0) {
|
||||
const fired = new Set(ev.assessments.filter((a) => a.state === 'fired').map((a) => a.id));
|
||||
if (!predicate.requireAnyFired.some((id) => fired.has(id))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function predicateComplexity(predicate: ZonePredicate): number {
|
||||
let n = 0;
|
||||
if (predicate.qualities) n += 1;
|
||||
if (predicate.minNetEvidence != null) n += 1;
|
||||
if (predicate.maxNetEvidence != null) n += 1;
|
||||
if (predicate.minBullEvidence != null) n += 1;
|
||||
if (predicate.minBearOrExitFired != null) n += 1;
|
||||
if (predicate.requireAnyFired) n += 1;
|
||||
if (predicate.onsetLookback != null) n += 1;
|
||||
if (predicate.onsetSlide) n += 1;
|
||||
return n;
|
||||
}
|
||||
|
||||
export function onsetStyleOf(predicate: ZonePredicate): OnsetStyle {
|
||||
if (predicate.onsetLookback === undefined && predicate.onsetSlide === undefined) {
|
||||
return ONSET_TURN_ON;
|
||||
}
|
||||
return {
|
||||
lookback: predicate.onsetLookback ?? null,
|
||||
slide: predicate.onsetSlide === true,
|
||||
};
|
||||
}
|
||||
|
||||
export function expandTemplates(pictures: ZonePredicate[]): ZonePredicate[] {
|
||||
const out: ZonePredicate[] = [];
|
||||
for (const p of pictures) {
|
||||
for (const o of ONSET_STYLE_CATALOG) {
|
||||
out.push({
|
||||
...p,
|
||||
id: `${p.id}+${o.id}`,
|
||||
label: o.id === 'turn-on' ? p.label : `${p.label}; ${o.label}`,
|
||||
onsetId: o.id,
|
||||
onsetLookback: o.style.lookback,
|
||||
onsetSlide: o.style.slide,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** True when `asOf` is the lowest (entry) or highest (exit) close in the lookback. */
|
||||
export function isLookbackExtreme(
|
||||
candles: PriceCandle[],
|
||||
asOf: string,
|
||||
kind: ZoneKind,
|
||||
lookback = REENTRY_LOOKBACK,
|
||||
): boolean {
|
||||
const idx = candleIndexForDate(candles, asOf.slice(0, 10));
|
||||
if (idx < 0 || lookback <= 0) return false;
|
||||
const start = Math.max(0, idx - lookback + 1);
|
||||
const px = candles[idx].c;
|
||||
if (!Number.isFinite(px)) return false;
|
||||
for (let i = start; i <= idx; i++) {
|
||||
const c = candles[i].c;
|
||||
if (!Number.isFinite(c)) continue;
|
||||
if (kind === 'entry' && c < px) return false;
|
||||
if (kind === 'exit' && c > px) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Lookback extreme that has started to reverse (next close turns the other way). Last bar is allowed without confirmation. */
|
||||
export function isConfirmedExtreme(
|
||||
candles: PriceCandle[],
|
||||
asOf: string,
|
||||
kind: ZoneKind,
|
||||
lookback = REENTRY_LOOKBACK,
|
||||
): boolean {
|
||||
if (!isLookbackExtreme(candles, asOf, kind, lookback)) return false;
|
||||
const idx = candleIndexForDate(candles, asOf.slice(0, 10));
|
||||
if (idx < 0 || idx + 1 >= candles.length) return true;
|
||||
const px = candles[idx].c;
|
||||
const next = candles[idx + 1].c;
|
||||
if (!Number.isFinite(next)) return true;
|
||||
return kind === 'entry' ? next > px : next < px;
|
||||
}
|
||||
|
||||
function closeOn(candles: PriceCandle[], asOf: string): number | null {
|
||||
const idx = candleIndexForDate(candles, asOf.slice(0, 10));
|
||||
if (idx < 0) return null;
|
||||
const px = candles[idx].c;
|
||||
return Number.isFinite(px) ? px : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Onsets: the picture turns on, or it stays on and price prints a fresh
|
||||
* N-session extreme after a quiet gap. That second case is a pullback
|
||||
* (entry) or a bounce into strength (exit) inside the same picture.
|
||||
*/
|
||||
export function findOnsets(
|
||||
evals: EvaluationPoint[],
|
||||
kind: ZoneKind,
|
||||
predicate: ZonePredicate,
|
||||
opposite?: ZonePredicate,
|
||||
candles?: PriceCandle[],
|
||||
style: OnsetStyle = onsetStyleOf(predicate),
|
||||
): string[] {
|
||||
const ordered = [...evals].sort((a, b) => a.asOf.localeCompare(b.asOf));
|
||||
const onsets: string[] = [];
|
||||
let prevMatch = false;
|
||||
let lastOnsetEvalIdx = -REENTRY_MIN_GAP;
|
||||
const lookback = style.lookback;
|
||||
|
||||
for (let i = 0; i < ordered.length; i++) {
|
||||
const ev = ordered[i];
|
||||
const matches = matchRule(ev, predicate);
|
||||
const flipped = opposite ? matchRule(ev, opposite) : false;
|
||||
if (flipped || !matches) {
|
||||
prevMatch = false;
|
||||
continue;
|
||||
}
|
||||
const fresh = !prevMatch;
|
||||
if (lookback == null || !candles) {
|
||||
if (fresh) {
|
||||
onsets.push(ev.asOf);
|
||||
lastOnsetEvalIdx = i;
|
||||
}
|
||||
prevMatch = true;
|
||||
continue;
|
||||
}
|
||||
const gapOk = i - lastOnsetEvalIdx >= REENTRY_MIN_GAP;
|
||||
const confirmed = isConfirmedExtreme(candles, ev.asOf, kind, lookback);
|
||||
const pullback = confirmed && gapOk;
|
||||
if (fresh || pullback) {
|
||||
onsets.push(ev.asOf);
|
||||
lastOnsetEvalIdx = i;
|
||||
} else if (style.slide && confirmed && onsets.length > 0) {
|
||||
const lastPx = closeOn(candles, onsets[onsets.length - 1]);
|
||||
const px = closeOn(candles, ev.asOf);
|
||||
const deeper = lastPx != null && px != null && (
|
||||
kind === 'entry' ? px < lastPx : px > lastPx
|
||||
);
|
||||
if (deeper && i - lastOnsetEvalIdx < REENTRY_MIN_GAP) {
|
||||
onsets[onsets.length - 1] = ev.asOf;
|
||||
lastOnsetEvalIdx = i;
|
||||
}
|
||||
}
|
||||
prevMatch = true;
|
||||
}
|
||||
return onsets;
|
||||
}
|
||||
|
||||
function zoneFromOnset(
|
||||
ordered: EvaluationPoint[],
|
||||
startAsOf: string,
|
||||
kind: ZoneKind,
|
||||
candles: PriceCandle[] | undefined,
|
||||
isLast: boolean,
|
||||
): ConfluenceZone | null {
|
||||
const start = ordered.find((e) => e.asOf === startAsOf);
|
||||
if (!start) return null;
|
||||
let endAsOf = startAsOf;
|
||||
if (candles) {
|
||||
const win = zoneOnsetWindow(startAsOf, '9999-12-31', candles);
|
||||
if (win) endAsOf = win.endAsOf;
|
||||
}
|
||||
const last = ordered[ordered.length - 1];
|
||||
return {
|
||||
kind,
|
||||
startAsOf,
|
||||
endAsOf,
|
||||
open: isLast && last != null && last.asOf >= startAsOf,
|
||||
qualityAtStart: start.quality,
|
||||
netEvidenceAtStart: start.netEvidence,
|
||||
firedSlotIds: start.assessments.filter((a) => a.state === 'fired').map((a) => a.id),
|
||||
forwardReturn: null,
|
||||
verdict: 'open',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse a chronological evaluation series into contiguous zones.
|
||||
* A single off-rule session does not split a zone. Two off-rule sessions, or
|
||||
* a day that matches the opposite predicate, closes it.
|
||||
*
|
||||
* When `candles` are passed, long stretches are split on N-session pullbacks
|
||||
* (entry) or bounce highs (exit) so a year-long bullish picture is not one zone.
|
||||
*/
|
||||
export function collapseZones(
|
||||
evals: EvaluationPoint[],
|
||||
kind: ZoneKind,
|
||||
predicate: ZonePredicate,
|
||||
opposite?: ZonePredicate,
|
||||
candles?: PriceCandle[],
|
||||
): ConfluenceZone[] {
|
||||
const ordered = [...evals].sort((a, b) => a.asOf.localeCompare(b.asOf));
|
||||
if (ordered.length === 0) return [];
|
||||
|
||||
if (candles && candles.length > 0) {
|
||||
const onsets = findOnsets(ordered, kind, predicate, opposite, candles, onsetStyleOf(predicate));
|
||||
return onsets
|
||||
.map((asOf, i) => zoneFromOnset(ordered, asOf, kind, candles, i === onsets.length - 1))
|
||||
.filter((z): z is ConfluenceZone => z != null);
|
||||
}
|
||||
|
||||
const zones: ConfluenceZone[] = [];
|
||||
let startIdx = -1;
|
||||
let lastMatchIdx = -1;
|
||||
|
||||
const close = (endIdx: number) => {
|
||||
if (startIdx < 0 || lastMatchIdx < 0) return;
|
||||
const start = ordered[startIdx];
|
||||
const end = ordered[lastMatchIdx];
|
||||
const open = lastMatchIdx >= ordered.length - 2;
|
||||
zones.push({
|
||||
kind,
|
||||
startAsOf: start.asOf,
|
||||
endAsOf: end.asOf,
|
||||
open,
|
||||
qualityAtStart: start.quality,
|
||||
netEvidenceAtStart: start.netEvidence,
|
||||
firedSlotIds: start.assessments.filter((a) => a.state === 'fired').map((a) => a.id),
|
||||
forwardReturn: null,
|
||||
verdict: 'open',
|
||||
});
|
||||
startIdx = -1;
|
||||
lastMatchIdx = -1;
|
||||
};
|
||||
|
||||
for (let i = 0; i < ordered.length; i++) {
|
||||
const ev = ordered[i];
|
||||
const matches = matchRule(ev, predicate);
|
||||
const flipped = opposite ? matchRule(ev, opposite) : false;
|
||||
|
||||
if (flipped && startIdx >= 0) {
|
||||
close(i - 1);
|
||||
}
|
||||
|
||||
if (matches) {
|
||||
if (startIdx < 0) startIdx = i;
|
||||
lastMatchIdx = i;
|
||||
} else if (startIdx >= 0 && i - lastMatchIdx > 1) {
|
||||
close(i);
|
||||
}
|
||||
}
|
||||
if (startIdx >= 0) close(ordered.length - 1);
|
||||
return zones;
|
||||
}
|
||||
|
||||
export function lastNZones(zones: ConfluenceZone[], n: number): ConfluenceZone[] {
|
||||
if (n <= 0) return [];
|
||||
return zones.slice(-n);
|
||||
}
|
||||
|
||||
/**
|
||||
* The chart marks the onset of a zone (the first `bars` sessions), not the
|
||||
* whole stretch the picture stayed on. A 14-month open bullish picture is
|
||||
* still one onset in June, not a year of green.
|
||||
*/
|
||||
export function zoneOnsetWindow(
|
||||
startAsOf: string,
|
||||
endAsOf: string,
|
||||
candles: PriceCandle[],
|
||||
bars = ZONE_DISPLAY_BARS,
|
||||
): { startAsOf: string; endAsOf: string } | null {
|
||||
const startIdx = candleIndexForDate(candles, startAsOf.slice(0, 10));
|
||||
if (startIdx < 0 || bars <= 0) return null;
|
||||
const naturalEndIdx = candleIndexForDate(candles, endAsOf.slice(0, 10));
|
||||
const capIdx = startIdx + bars - 1;
|
||||
const endIdx = naturalEndIdx >= 0
|
||||
? Math.min(naturalEndIdx, capIdx, candles.length - 1)
|
||||
: Math.min(capIdx, candles.length - 1);
|
||||
if (endIdx < startIdx) return null;
|
||||
return {
|
||||
startAsOf: (candles[startIdx].ts ?? startAsOf).slice(0, 10),
|
||||
endAsOf: (candles[endIdx].ts ?? endAsOf).slice(0, 10),
|
||||
};
|
||||
}
|
||||
|
||||
export function attachOutcomes(
|
||||
zones: ConfluenceZone[],
|
||||
candles: PriceCandle[],
|
||||
weeks = ZONE_HORIZON_WEEKS,
|
||||
): ConfluenceZone[] {
|
||||
return zones.map((z) => {
|
||||
const direction = z.kind === 'entry' ? 'up' : 'down';
|
||||
const idx = candleIndexForDate(candles, z.startAsOf);
|
||||
const ret = forwardReturnFrom(candles, z.startAsOf, weeks);
|
||||
if (idx < 0 || ret === null) {
|
||||
return { ...z, forwardReturn: null, verdict: 'open' as const };
|
||||
}
|
||||
const confirmed = checkFollowThrough(candles, idx, direction, weeks);
|
||||
if (confirmed === null) {
|
||||
return { ...z, forwardReturn: ret, verdict: 'open' as const };
|
||||
}
|
||||
return {
|
||||
...z,
|
||||
forwardReturn: ret,
|
||||
verdict: confirmed ? 'confirmed' : 'false_alarm',
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function forwardReturnFrom(candles: PriceCandle[], asOf: string, weeks = ZONE_HORIZON_WEEKS): number | null {
|
||||
const idx = candleIndexForDate(candles, asOf);
|
||||
if (idx < 0) return null;
|
||||
const target = idx + weeks * 5;
|
||||
if (target >= candles.length) return null;
|
||||
const entry = candles[idx].c;
|
||||
const exit = candles[target].c;
|
||||
if (!Number.isFinite(entry) || !Number.isFinite(exit) || entry <= 0) return null;
|
||||
return (exit - entry) / entry;
|
||||
}
|
||||
|
||||
export function emptyStats(): ZoneRuleStats {
|
||||
return { zones: 0, resolved: 0, confirmed: 0, hitRate: null, avgReturn: null, baselineHitRate: null };
|
||||
}
|
||||
|
||||
export function scoreZones(zones: ConfluenceZone[], baselineHitRate: number | null): ZoneRuleStats {
|
||||
const resolved = zones.filter((z) => z.verdict !== 'open');
|
||||
const confirmed = resolved.filter((z) => z.verdict === 'confirmed');
|
||||
const rets = resolved.map((z) => z.forwardReturn).filter((v): v is number => v != null);
|
||||
const avgReturn = rets.length > 0 ? rets.reduce((a, b) => a + b, 0) / rets.length : null;
|
||||
return {
|
||||
zones: zones.length,
|
||||
resolved: resolved.length,
|
||||
confirmed: confirmed.length,
|
||||
hitRate: resolved.length > 0 ? confirmed.length / resolved.length : null,
|
||||
avgReturn,
|
||||
baselineHitRate,
|
||||
};
|
||||
}
|
||||
|
||||
export function baselineHitRate(
|
||||
evals: EvaluationPoint[],
|
||||
candlesBySymbol: Map<string, PriceCandle[]>,
|
||||
kind: ZoneKind,
|
||||
weeks = ZONE_HORIZON_WEEKS,
|
||||
): number | null {
|
||||
let resolved = 0;
|
||||
let confirmed = 0;
|
||||
const direction = kind === 'entry' ? 'up' : 'down';
|
||||
for (const ev of evals) {
|
||||
const candles = candlesBySymbol.get(ev.symbol);
|
||||
if (!candles) continue;
|
||||
const idx = candleIndexForDate(candles, ev.asOf);
|
||||
if (idx < 0) continue;
|
||||
const hit = checkFollowThrough(candles, idx, direction, weeks);
|
||||
if (hit === null) continue;
|
||||
resolved += 1;
|
||||
if (hit) confirmed += 1;
|
||||
}
|
||||
return resolved > 0 ? confirmed / resolved : null;
|
||||
}
|
||||
|
||||
export function splitTrainValidate(evals: EvaluationPoint[], trainEnd: string): { train: EvaluationPoint[]; validate: EvaluationPoint[] } {
|
||||
const ordered = [...evals].sort((a, b) => a.asOf.localeCompare(b.asOf));
|
||||
return {
|
||||
train: ordered.filter((e) => e.asOf <= trainEnd),
|
||||
validate: ordered.filter((e) => e.asOf > trainEnd),
|
||||
};
|
||||
}
|
||||
|
||||
/** Default train cut: last year of the series is held out. */
|
||||
export function defaultTrainEnd(evals: EvaluationPoint[]): string | null {
|
||||
const dates = [...new Set(evals.map((e) => e.asOf))].sort();
|
||||
if (dates.length === 0) return null;
|
||||
const last = dates[dates.length - 1];
|
||||
const cut = new Date(`${last}T00:00:00Z`);
|
||||
cut.setUTCFullYear(cut.getUTCFullYear() - 1);
|
||||
return cut.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function scoreTemplate(
|
||||
kind: ZoneKind,
|
||||
predicate: ZonePredicate,
|
||||
opposite: ZonePredicate | undefined,
|
||||
train: EvaluationPoint[],
|
||||
validate: EvaluationPoint[],
|
||||
candlesBySymbol: Map<string, PriceCandle[]>,
|
||||
trainBaseline: number | null,
|
||||
validateBaseline: number | null,
|
||||
): { train: ZoneRuleStats; validate: ZoneRuleStats } {
|
||||
const scoreSplit = (rows: EvaluationPoint[], baseline: number | null): ZoneRuleStats => {
|
||||
const bySymbol = new Map<string, EvaluationPoint[]>();
|
||||
for (const ev of rows) {
|
||||
const arr = bySymbol.get(ev.symbol) ?? [];
|
||||
arr.push(ev);
|
||||
bySymbol.set(ev.symbol, arr);
|
||||
}
|
||||
const all: ConfluenceZone[] = [];
|
||||
for (const [symbol, series] of bySymbol) {
|
||||
const candles = candlesBySymbol.get(symbol) ?? [];
|
||||
all.push(...attachOutcomes(collapseZones(series, kind, predicate, opposite, candles), candles));
|
||||
}
|
||||
return scoreZones(all, baseline);
|
||||
};
|
||||
return {
|
||||
train: scoreSplit(train, trainBaseline),
|
||||
validate: scoreSplit(validate, validateBaseline),
|
||||
};
|
||||
}
|
||||
|
||||
export function selectWinner(
|
||||
kind: ZoneKind,
|
||||
templates: ZonePredicate[],
|
||||
evals: EvaluationPoint[],
|
||||
candlesBySymbol: Map<string, PriceCandle[]>,
|
||||
fallback: ZonePredicate,
|
||||
oppositeFallback?: ZonePredicate,
|
||||
): DerivedZoneRule {
|
||||
const cut = defaultTrainEnd(evals);
|
||||
if (!cut || evals.length === 0) {
|
||||
return {
|
||||
kind,
|
||||
predicate: fallback,
|
||||
source: 'baseline',
|
||||
train: emptyStats(),
|
||||
validate: emptyStats(),
|
||||
};
|
||||
}
|
||||
const { train, validate } = splitTrainValidate(evals, cut);
|
||||
const trainBaseline = baselineHitRate(train, candlesBySymbol, kind);
|
||||
const validateBaseline = baselineHitRate(validate, candlesBySymbol, kind);
|
||||
|
||||
type Cand = { predicate: ZonePredicate; train: ZoneRuleStats; validate: ZoneRuleStats };
|
||||
const survivors: Cand[] = [];
|
||||
|
||||
for (const predicate of templates) {
|
||||
const scored = scoreTemplate(kind, predicate, oppositeFallback, train, validate, candlesBySymbol, trainBaseline, validateBaseline);
|
||||
if (scored.train.resolved < MIN_TRAIN_ZONES) continue;
|
||||
if (scored.validate.hitRate == null || validateBaseline == null) continue;
|
||||
if (scored.validate.hitRate < validateBaseline + MIN_VALIDATE_BEAT) continue;
|
||||
survivors.push({ predicate, train: scored.train, validate: scored.validate });
|
||||
}
|
||||
|
||||
survivors.sort((a, b) => {
|
||||
const hit = (b.validate.hitRate ?? 0) - (a.validate.hitRate ?? 0);
|
||||
if (Math.abs(hit) > 1e-9) return hit;
|
||||
const simpler = predicateComplexity(a.predicate) - predicateComplexity(b.predicate);
|
||||
if (simpler !== 0) return simpler;
|
||||
return b.train.resolved - a.train.resolved;
|
||||
});
|
||||
|
||||
if (survivors.length === 0) {
|
||||
const scored = scoreTemplate(kind, fallback, oppositeFallback, train, validate, candlesBySymbol, trainBaseline, validateBaseline);
|
||||
return { kind, predicate: fallback, source: 'baseline', train: scored.train, validate: scored.validate };
|
||||
}
|
||||
|
||||
const win = survivors[0];
|
||||
return { kind, predicate: win.predicate, source: 'validated', train: win.train, validate: win.validate };
|
||||
}
|
||||
|
||||
export function deriveZoneRules(
|
||||
evals: EvaluationPoint[],
|
||||
candlesBySymbol: Map<string, PriceCandle[]>,
|
||||
): { entry: DerivedZoneRule; exit: DerivedZoneRule } {
|
||||
const entry = selectWinner('entry', expandTemplates(ENTRY_TEMPLATES), evals, candlesBySymbol, FALLBACK_ENTRY, FALLBACK_EXIT);
|
||||
const exit = selectWinner('exit', expandTemplates(EXIT_TEMPLATES), evals, candlesBySymbol, FALLBACK_EXIT, FALLBACK_ENTRY);
|
||||
return { entry, exit };
|
||||
}
|
||||
|
||||
export function recentZonesForSymbol(
|
||||
evals: EvaluationPoint[],
|
||||
candles: PriceCandle[],
|
||||
entryRule: ZonePredicate,
|
||||
exitRule: ZonePredicate,
|
||||
entries = 3,
|
||||
exits = 3,
|
||||
): { entries: ConfluenceZone[]; exits: ConfluenceZone[] } {
|
||||
const entryZones = attachOutcomes(collapseZones(evals, 'entry', entryRule, exitRule, candles), candles);
|
||||
const exitZones = attachOutcomes(collapseZones(evals, 'exit', exitRule, entryRule, candles), candles);
|
||||
return {
|
||||
entries: lastNZones(entryZones, entries),
|
||||
exits: lastNZones(exitZones, exits),
|
||||
};
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { upsertTrackedFund, sync13FIntoRecords, liveBook, quarterEnd } from '../fundRepository.ts';
|
||||
import { upsertTrackedFund, sync13FIntoRecords, liveBook, quarterEnd, insertPositionRecord } from '../fundRepository.ts';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const SCHEMA_SQL = readFileSync(join(__dirname, '..', 'schema.sql'), 'utf8');
|
||||
@@ -41,4 +41,86 @@ test('quarterEnd helper', () => {
|
||||
assert.equal(quarterEnd('2025-Q4'), '2025-12-31');
|
||||
assert.equal(quarterEnd('2026-Q2'), '2026-06-30');
|
||||
assert.equal(quarterEnd('2026-Q1'), '2026-03-31');
|
||||
});
|
||||
|
||||
test('sync13FIntoRecords: missing from latest quarter is an exit (shares=0)', () => {
|
||||
const db = freshDb();
|
||||
const fund = upsertTrackedFund(db, { ci_key: '0002096493', fund_name: 'Alpine Fox Capital', manager_name: 'M' });
|
||||
db.prepare(
|
||||
`INSERT INTO institution_filings (filer_cik, filer_name, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at, accession)
|
||||
VALUES ('0002096493','AFC','IREN','13F-HR',750000,30000000,'2026-Q1','2026-05-14','2026-05-14','A1'),
|
||||
('0002096493','AFC','CIFR','13F-HR',100000,1287000,'2026-Q1','2026-05-14','2026-05-14','A1'),
|
||||
('0002096493','AFC','IREN','13F-HR',750000,34000000,'2026-Q2','2026-08-14','2026-08-14','A2')`,
|
||||
).run();
|
||||
|
||||
const n = sync13FIntoRecords(db, fund.id);
|
||||
assert.ok(n >= 3, 'IREN Q1+Q2, CIFR Q1, plus CIFR exit');
|
||||
assert.equal(sync13FIntoRecords(db, fund.id), 0);
|
||||
|
||||
const book = liveBook(db, fund.id);
|
||||
const iren = book.find((p) => p.symbol === 'IREN')!;
|
||||
const cifr = book.find((p) => p.symbol === 'CIFR')!;
|
||||
assert.equal(iren.status, 'held');
|
||||
assert.equal(iren.shares, 750000);
|
||||
assert.equal(iren.as_of, '2026-06-30');
|
||||
assert.equal(cifr.status, 'exited');
|
||||
assert.equal(cifr.shares, 0);
|
||||
assert.equal(cifr.as_of, '2026-06-30');
|
||||
assert.equal(cifr.last_held_shares, 100000);
|
||||
assert.equal(cifr.source, '13f');
|
||||
});
|
||||
|
||||
test('sync13FIntoRecords: later capture after 13F exit is held again', () => {
|
||||
const db = freshDb();
|
||||
const fund = upsertTrackedFund(db, { ci_key: '0002096493', fund_name: 'Alpine Fox Capital', manager_name: 'M' });
|
||||
db.prepare(
|
||||
`INSERT INTO institution_filings (filer_cik, filer_name, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at, accession)
|
||||
VALUES ('0002096493','AFC','DEO','13F-HR',122000,10000000,'2026-Q1','2026-05-14','2026-05-14','A1'),
|
||||
('0002096493','AFC','IREN','13F-HR',750000,34000000,'2026-Q2','2026-08-14','2026-08-14','A2')`,
|
||||
).run();
|
||||
sync13FIntoRecords(db, fund.id);
|
||||
assert.equal(liveBook(db, fund.id).find((p) => p.symbol === 'DEO')!.status, 'exited');
|
||||
|
||||
insertPositionRecord(db, {
|
||||
fund_id: fund.id, symbol: 'DEO', shares: 50000, as_of: '2026-07-15', source: 'capture',
|
||||
});
|
||||
const deo = liveBook(db, fund.id).find((p) => p.symbol === 'DEO')!;
|
||||
assert.equal(deo.status, 'held');
|
||||
assert.equal(deo.shares, 50000);
|
||||
assert.equal(deo.source, 'capture');
|
||||
});
|
||||
|
||||
test('sync13FIntoRecords: put_call Call is a call line; share change vs prior quarter', () => {
|
||||
const db = freshDb();
|
||||
const fund = upsertTrackedFund(db, { ci_key: '0002096493', fund_name: 'Alpine Fox Capital', manager_name: 'M' });
|
||||
db.prepare(
|
||||
`INSERT INTO institution_filings (filer_cik, filer_name, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at, accession, put_call)
|
||||
VALUES ('0002096493','AFC','IBIT','13F-HR',1172200,45000000,'2026-Q1','2026-05-14','2026-05-14','A1','Call'),
|
||||
('0002096493','AFC','IBIT','13F-HR',1120000,37000000,'2026-Q2','2026-08-14','2026-08-14','A2','Call'),
|
||||
('0002096493','AFC','IREN','13F-HR',750461,34000000,'2026-Q2','2026-08-14','2026-08-14','A2','')`,
|
||||
).run();
|
||||
sync13FIntoRecords(db, fund.id);
|
||||
const ibit = liveBook(db, fund.id).find((p) => p.symbol === 'IBIT')!;
|
||||
assert.equal(ibit.notes, 'call');
|
||||
assert.equal(ibit.status, 'held');
|
||||
assert.equal(ibit.shares, 1120000);
|
||||
assert.equal(ibit.prior_shares, 1172200);
|
||||
assert.equal(ibit.change, 'cut');
|
||||
assert.equal(ibit.share_delta, -52200);
|
||||
const iren = liveBook(db, fund.id).find((p) => p.symbol === 'IREN')!;
|
||||
assert.equal(iren.notes, null);
|
||||
assert.equal(iren.change, 'new');
|
||||
});
|
||||
|
||||
test('sync13FIntoRecords: empty latest quarter does not mass-exit', () => {
|
||||
const db = freshDb();
|
||||
const fund = upsertTrackedFund(db, { ci_key: '0002096493', fund_name: 'Alpine Fox Capital', manager_name: 'M' });
|
||||
db.prepare(
|
||||
`INSERT INTO institution_filings (filer_cik, filer_name, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at, accession)
|
||||
VALUES ('0002096493','AFC','IREN','13F-HR',750000,30000000,'2026-Q1','2026-05-14','2026-05-14','A1')`,
|
||||
).run();
|
||||
sync13FIntoRecords(db, fund.id);
|
||||
const book = liveBook(db, fund.id);
|
||||
assert.equal(book.length, 1);
|
||||
assert.equal(book[0].status, 'held');
|
||||
});
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
listTrackedFunds, getTrackedFund, upsertTrackedFund,
|
||||
setFundEnabled, deleteTrackedFund,
|
||||
insertPositionRecord, liveBook, listRecords,
|
||||
fundFreshness,
|
||||
} from '../fundRepository.ts';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
@@ -52,6 +53,23 @@ test('registry: upsert creates and is idempotent by CIK, then lists enabled firs
|
||||
assert.equal(got!.enabled, false);
|
||||
});
|
||||
|
||||
test('freshness: queued X handle with no posts is pending / never posted', () => {
|
||||
const db = freshDb();
|
||||
const fund = upsertTrackedFund(db, {
|
||||
ci_key: '000111', fund_name: 'The Prof Investor', manager_name: 'Prof',
|
||||
x_handle: 'TheProfInvestor',
|
||||
});
|
||||
db.prepare(
|
||||
`INSERT INTO adapter_queue (key, status, last_attempt, retry_count) VALUES (?, 'pending', NULL, 0)`,
|
||||
).run('x:timeline:TheProfInvestor');
|
||||
const fresh = fundFreshness(db, getTrackedFund(db, fund.id)!);
|
||||
assert.equal(fresh.xHandle, 'TheProfInvestor');
|
||||
assert.equal(fresh.xPull.status, 'pending');
|
||||
assert.equal(fresh.lastPostAt, null);
|
||||
assert.equal(fresh.lastCaptureAt, null);
|
||||
assert.equal(fresh.last13fAt, null);
|
||||
});
|
||||
|
||||
test('registry: enable/disable toggle and delete', () => {
|
||||
const db = freshDb();
|
||||
const id = seedFund(db);
|
||||
@@ -85,6 +103,22 @@ test('liveBook: most recent record per (fund, symbol) by as_of, any source', ()
|
||||
assert.equal(book2[0].shares, 950000);
|
||||
});
|
||||
|
||||
test('liveBook: capture with no 13F for that instrument is x_only', () => {
|
||||
const db = freshDb();
|
||||
const fund = seedFund(db);
|
||||
insertPositionRecord(db, { fund_id: fund, symbol: 'ANY', shares: 200000, as_of: '2026-07-28', source: 'capture' });
|
||||
insertPositionRecord(db, { fund_id: fund, symbol: 'OPEN', shares: 1670199, as_of: '2026-03-31', source: '13f' });
|
||||
insertPositionRecord(db, { fund_id: fund, symbol: 'OPEN', shares: 7050000, as_of: '2026-08-11', source: 'capture' });
|
||||
insertPositionRecord(db, { fund_id: fund, symbol: 'WULF', shares: 100000, as_of: '2025-12-31', source: '13f' });
|
||||
insertPositionRecord(db, { fund_id: fund, symbol: 'WULF', as_of: '2026-07-30', source: 'capture', notes: 'call' });
|
||||
const book = liveBook(db, fund);
|
||||
const by = Object.fromEntries(book.map((p) => [`${p.symbol}:${p.notes ?? 'eq'}`, p]));
|
||||
assert.equal(by['ANY:eq'].x_only, true);
|
||||
assert.equal(by['OPEN:eq'].x_only, false);
|
||||
assert.equal(by['OPEN:eq'].source, 'capture');
|
||||
assert.equal(by['WULF:call'].x_only, true);
|
||||
});
|
||||
|
||||
test('live: multiple symbols independent recency', () => {
|
||||
const db = freshDb();
|
||||
const fund = seedFund(db);
|
||||
|
||||
@@ -19,7 +19,7 @@ const EXPECTED_TABLES = [
|
||||
'screener_filters', 'reports', 'alerts', 'trusted_accounts',
|
||||
'saved_posts', 'trade_key_levels',
|
||||
'llm_summaries', 'rotation_signals', 'rotation_rank_snapshots', 'llm_providers', 'llm_dispatch_audit',
|
||||
'confluence_racks', 'confluence_evaluations', 'confluence_signal_history',
|
||||
'confluence_racks', 'confluence_evaluations', 'confluence_signal_history', 'confluence_zone_rules',
|
||||
];
|
||||
|
||||
test('schema creates all locked tables', () => {
|
||||
|
||||
@@ -35,7 +35,7 @@ export const ALERT_TYPE_CATALOG: Array<{ type: string; label: string; descriptio
|
||||
{ type: 'asymmetry_warning', label: 'Asymmetry Warning', description: 'Portfolio reward-to-risk below threshold.' },
|
||||
{ type: 'fund_capture', label: 'Fund Capture', description: 'A tracked fund posted a position update.' },
|
||||
{ type: 'fund_13f', label: 'Tracked Fund 13F', description: 'A tracked fund filed a new 13F.' },
|
||||
{ type: 'mirror_diff', label: 'Mirror Diff', description: 'The mirror target book changed materially.' },
|
||||
{ type: 'mirror_diff', label: 'Tracked Fund Book', description: "A tracked fund's disclosed book changed materially." },
|
||||
{ type: 'vix_level', label: 'VIX Level', description: 'VIX moves into a new historical volatility band.' },
|
||||
{ type: 'confluence_change', label: 'Confluence Picture Change', description: 'A symbol\'s confluence picture improved or became more cautious.' },
|
||||
];
|
||||
|
||||
@@ -88,6 +88,7 @@ function runMigrations(db: DatabaseSync): void {
|
||||
// Extended-hours quote fields (pre/post last vs RTH close)
|
||||
`ALTER TABLE quotes ADD COLUMN session TEXT`,
|
||||
`ALTER TABLE quotes ADD COLUMN regular_price REAL`,
|
||||
`ALTER TABLE x_cookie_posts ADD COLUMN quoted_post_id TEXT`,
|
||||
`DROP INDEX IF EXISTS idx_watchlists_owner_name`,
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS idx_watchlists_owner_kind_name ON watchlists(owner_id, kind, name)`,
|
||||
`CREATE TABLE IF NOT EXISTS rotation_state (
|
||||
@@ -328,7 +329,13 @@ function runMigrations(db: DatabaseSync): void {
|
||||
|
||||
// 3. Idempotency index for institution_filings (now safe after de-dupe).
|
||||
try {
|
||||
db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_inst_filings ON institution_filings(filer_cik, symbol, reported_quarter, form)`);
|
||||
db.exec(`UPDATE institution_filings SET put_call = '' WHERE put_call IS NULL`);
|
||||
} catch { /* ignore */ }
|
||||
try {
|
||||
db.exec(`DROP INDEX IF EXISTS uq_inst_filings`);
|
||||
} catch { /* ignore */ }
|
||||
try {
|
||||
db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_inst_filings ON institution_filings(filer_cik, symbol, reported_quarter, form, put_call)`);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// 4. Rename legacy 'Starter' watchlist to 'default'.
|
||||
@@ -670,6 +677,36 @@ function runMigrations(db: DatabaseSync): void {
|
||||
db.exec(`CREATE INDEX IF NOT EXISTS idx_mentor_calls_owner ON mentor_calls(owner_id, logged_at DESC)`);
|
||||
db.exec(`CREATE INDEX IF NOT EXISTS idx_mentor_calls_handle ON mentor_calls(owner_id, handle)`);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Confluence zone rules + unique signal-fire key (replay must not double-log).
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS confluence_zone_rules (
|
||||
id TEXT PRIMARY KEY,
|
||||
rack_id TEXT NOT NULL REFERENCES confluence_racks(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL,
|
||||
version INTEGER NOT NULL,
|
||||
derived_at TEXT NOT NULL,
|
||||
predicate_json TEXT NOT NULL,
|
||||
train_stats_json TEXT NOT NULL,
|
||||
validate_stats_json TEXT NOT NULL,
|
||||
sample_caveat TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
is_active INTEGER NOT NULL DEFAULT 0
|
||||
)
|
||||
`);
|
||||
db.exec(`CREATE INDEX IF NOT EXISTS idx_confluence_zone_rules_rack ON confluence_zone_rules(rack_id, kind, is_active)`);
|
||||
} catch { /* ignore */ }
|
||||
try {
|
||||
db.exec(`
|
||||
DELETE FROM confluence_signal_history
|
||||
WHERE rowid NOT IN (
|
||||
SELECT MIN(rowid) FROM confluence_signal_history
|
||||
GROUP BY symbol, slot_id, rack_id, fired_at
|
||||
)
|
||||
`);
|
||||
db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_confluence_signal_fire ON confluence_signal_history(symbol, slot_id, rack_id, fired_at)`);
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
let _db: DatabaseSync | null = null;
|
||||
|
||||
@@ -47,6 +47,21 @@ export interface StoredConfluenceEvaluation {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** Persisted walk-forward zone rule. */
|
||||
export interface StoredZoneRule {
|
||||
id: string;
|
||||
rackId: string;
|
||||
kind: 'entry' | 'exit';
|
||||
version: number;
|
||||
derivedAt: string;
|
||||
predicateJson: string;
|
||||
trainStatsJson: string;
|
||||
validateStatsJson: string;
|
||||
sampleCaveat: string;
|
||||
source: 'validated' | 'baseline';
|
||||
isActive: boolean;
|
||||
}
|
||||
|
||||
/** A slot-fire logged from an evaluation, awaiting verdict resolution. */
|
||||
export interface ConfluenceSignalHistoryRow {
|
||||
id: string;
|
||||
@@ -133,10 +148,40 @@ function stmts(db: DatabaseSync) {
|
||||
|
||||
// --- confluence_signal_history ---
|
||||
insertSignal: db.prepare(
|
||||
`INSERT INTO confluence_signal_history
|
||||
`INSERT OR IGNORE INTO confluence_signal_history
|
||||
(id, symbol, slot_id, rack_id, fired_at, quality_at_fire, price_resolved, price_confirmed, resolved_at, verdict)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 0, 0, NULL, NULL)`,
|
||||
),
|
||||
countEvaluationsForSymbolRack: db.prepare(
|
||||
`SELECT COUNT(*) AS n FROM confluence_evaluations WHERE symbol = ? AND rack_id = ?`,
|
||||
),
|
||||
listEvaluationAsOfs: db.prepare(
|
||||
`SELECT as_of FROM confluence_evaluations WHERE symbol = ? AND rack_id = ? ORDER BY as_of ASC`,
|
||||
),
|
||||
selectEvaluationsForSymbolRack: db.prepare(
|
||||
`SELECT id, symbol, as_of, rack_id, assessments_json, bull_evidence, bear_evidence,
|
||||
bull_count, bear_count, assessed_count, net_evidence, total_evidence, quality, created_at
|
||||
FROM confluence_evaluations WHERE symbol = ? AND rack_id = ? ORDER BY as_of ASC`,
|
||||
),
|
||||
insertZoneRule: db.prepare(
|
||||
`INSERT INTO confluence_zone_rules
|
||||
(id, rack_id, kind, version, derived_at, predicate_json, train_stats_json,
|
||||
validate_stats_json, sample_caveat, source, is_active)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
),
|
||||
deactivateZoneRules: db.prepare(
|
||||
`UPDATE confluence_zone_rules SET is_active = 0 WHERE rack_id = ? AND kind = ?`,
|
||||
),
|
||||
selectActiveZoneRule: db.prepare(
|
||||
`SELECT id, rack_id, kind, version, derived_at, predicate_json, train_stats_json,
|
||||
validate_stats_json, sample_caveat, source, is_active
|
||||
FROM confluence_zone_rules
|
||||
WHERE rack_id = ? AND kind = ? AND is_active = 1
|
||||
ORDER BY version DESC LIMIT 1`,
|
||||
),
|
||||
maxZoneRuleVersion: db.prepare(
|
||||
`SELECT COALESCE(MAX(version), 0) AS v FROM confluence_zone_rules WHERE rack_id = ? AND kind = ?`,
|
||||
),
|
||||
selectPendingSignals: db.prepare(
|
||||
`SELECT id, symbol, slot_id, rack_id, fired_at, quality_at_fire, price_resolved, price_confirmed, resolved_at, verdict
|
||||
FROM confluence_signal_history WHERE price_resolved = 0 ORDER BY fired_at`,
|
||||
@@ -339,6 +384,61 @@ export class ConfluenceRepository {
|
||||
resolveSignal(id: string, confirmed: boolean, verdict: 'real' | 'false_alarm'): void {
|
||||
stmts(this.db).resolveSignal.run(confirmed ? 1 : 0, new Date().toISOString(), verdict, id);
|
||||
}
|
||||
|
||||
countEvaluations(symbol: string, rackId: string): number {
|
||||
const row = stmts(this.db).countEvaluationsForSymbolRack.get(symbol, rackId) as { n: number } | undefined;
|
||||
return Number(row?.n ?? 0);
|
||||
}
|
||||
|
||||
listEvaluationAsOfs(symbol: string, rackId: string): string[] {
|
||||
return (stmts(this.db).listEvaluationAsOfs.all(symbol, rackId) as Array<{ as_of: string }>).map((r) => r.as_of);
|
||||
}
|
||||
|
||||
listEvaluationsForSymbolRack(symbol: string, rackId: string): StoredConfluenceEvaluation[] {
|
||||
return (stmts(this.db).selectEvaluationsForSymbolRack.all(symbol, rackId) as Record<string, unknown>[]).map(mapEvaluation);
|
||||
}
|
||||
|
||||
getActiveZoneRule(rackId: string, kind: 'entry' | 'exit'): StoredZoneRule | null {
|
||||
const row = stmts(this.db).selectActiveZoneRule.get(rackId, kind) as Record<string, unknown> | undefined;
|
||||
return row ? mapZoneRule(row) : null;
|
||||
}
|
||||
|
||||
saveActiveZoneRule(rule: Omit<StoredZoneRule, 'id' | 'version' | 'isActive'>): StoredZoneRule {
|
||||
const s = stmts(this.db);
|
||||
const version = Number((s.maxZoneRuleVersion.get(rule.rackId, rule.kind) as { v: number }).v) + 1;
|
||||
const id = randomUUID();
|
||||
s.deactivateZoneRules.run(rule.rackId, rule.kind);
|
||||
s.insertZoneRule.run(
|
||||
id,
|
||||
rule.rackId,
|
||||
rule.kind,
|
||||
version,
|
||||
rule.derivedAt,
|
||||
rule.predicateJson,
|
||||
rule.trainStatsJson,
|
||||
rule.validateStatsJson,
|
||||
rule.sampleCaveat,
|
||||
rule.source,
|
||||
1,
|
||||
);
|
||||
return { ...rule, id, version, isActive: true };
|
||||
}
|
||||
}
|
||||
|
||||
function mapZoneRule(row: Record<string, unknown>): StoredZoneRule {
|
||||
return {
|
||||
id: row.id as string,
|
||||
rackId: row.rack_id as string,
|
||||
kind: row.kind as 'entry' | 'exit',
|
||||
version: Number(row.version),
|
||||
derivedAt: row.derived_at as string,
|
||||
predicateJson: row.predicate_json as string,
|
||||
trainStatsJson: row.train_stats_json as string,
|
||||
validateStatsJson: row.validate_stats_json as string,
|
||||
sampleCaveat: row.sample_caveat as string,
|
||||
source: row.source as 'validated' | 'baseline',
|
||||
isActive: Number(row.is_active) === 1,
|
||||
};
|
||||
}
|
||||
|
||||
/** Build a rack from the catalog subset; validates slot ids exist. */
|
||||
|
||||
@@ -48,6 +48,29 @@ export interface LiveBookPosition {
|
||||
evidence_url: string | null;
|
||||
notes: string | null;
|
||||
post_text: string | null;
|
||||
/** held = latest size > 0 or unknown; exited = latest shares are exactly 0. */
|
||||
status: 'held' | 'exited';
|
||||
/** Prior non-zero size, used when status is exited. */
|
||||
last_held_shares: number | null;
|
||||
/** Same instrument on the previous disclosure (13F or capture). */
|
||||
prior_shares: number | null;
|
||||
share_delta: number | null;
|
||||
change: 'add' | 'cut' | 'flat' | 'new' | null;
|
||||
/** Latest row is X, and this instrument never appeared in a 13F. */
|
||||
x_only: boolean;
|
||||
evidence_status?: 'ok' | 'deleted' | 'unknown';
|
||||
}
|
||||
|
||||
export function instrumentOf(notes: string | null | undefined): 'call' | 'put' | 'equity' {
|
||||
if (notes === 'call' || notes === 'put') return notes;
|
||||
return 'equity';
|
||||
}
|
||||
|
||||
export function noteFromPutCall(putCall: string | null | undefined): string | null {
|
||||
const p = (putCall || '').trim().toLowerCase();
|
||||
if (p === 'call') return 'call';
|
||||
if (p === 'put') return 'put';
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Tracked fund registry ───────────────────────────────────────────────────
|
||||
@@ -125,6 +148,100 @@ export function deleteTrackedFund(db: DatabaseSync, id: string): boolean {
|
||||
return r.changes > 0;
|
||||
}
|
||||
|
||||
export type SourcePullStatus = 'pending' | 'in_flight' | 'done' | 'backoff' | 'failed' | 'none';
|
||||
|
||||
export interface SourcePull {
|
||||
status: SourcePullStatus;
|
||||
lastAttempt: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface FundFreshness {
|
||||
xHandle: string | null;
|
||||
xPull: SourcePull;
|
||||
lastPostAt: string | null;
|
||||
lastCaptureAt: string | null;
|
||||
last13fAt: string | null;
|
||||
}
|
||||
|
||||
const EMPTY_PULL: SourcePull = { status: 'none', lastAttempt: null, error: null };
|
||||
|
||||
function asPullStatus(raw: string | null | undefined): SourcePullStatus {
|
||||
if (raw === 'pending' || raw === 'in_flight' || raw === 'done' || raw === 'backoff' || raw === 'failed') return raw;
|
||||
return 'none';
|
||||
}
|
||||
|
||||
/** Last X pull / last post / last book write for each fund (one batched read). */
|
||||
export function fundsFreshness(db: DatabaseSync, funds: TrackedFund[]): Map<string, FundFreshness> {
|
||||
const out = new Map<string, FundFreshness>();
|
||||
if (funds.length === 0) return out;
|
||||
|
||||
const handles = [...new Set(funds.map((f) => f.x_handle).filter((h): h is string => Boolean(h)))];
|
||||
const queueByHandle = new Map<string, SourcePull>();
|
||||
if (handles.length > 0) {
|
||||
const keys = handles.flatMap((h) => [`x:timeline:${h}`, `x:timeline:${h.toLowerCase()}`]);
|
||||
const rows = db.prepare(
|
||||
`SELECT key, status, last_attempt, error FROM adapter_queue WHERE key IN (${keys.map(() => '?').join(',')})`,
|
||||
).all(...keys) as Array<{ key: string; status: string; last_attempt: string | null; error: string | null }>;
|
||||
for (const r of rows) {
|
||||
const handle = r.key.slice('x:timeline:'.length);
|
||||
queueByHandle.set(handle, {
|
||||
status: asPullStatus(r.status),
|
||||
lastAttempt: r.last_attempt,
|
||||
error: r.error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const lastPostByHandle = new Map<string, string>();
|
||||
if (handles.length > 0) {
|
||||
const rows = db.prepare(
|
||||
`SELECT lower(author_handle) AS h, MAX(posted_at) AS last_post
|
||||
FROM x_cookie_posts
|
||||
WHERE lower(author_handle) IN (${handles.map(() => '?').join(',')})
|
||||
GROUP BY 1`,
|
||||
).all(...handles.map((h) => h.toLowerCase())) as Array<{ h: string; last_post: string | null }>;
|
||||
for (const r of rows) {
|
||||
if (r.last_post) lastPostByHandle.set(r.h, r.last_post);
|
||||
}
|
||||
}
|
||||
|
||||
const lastByFundSource = new Map<string, string>();
|
||||
const ids = funds.map((f) => f.id);
|
||||
const recs = db.prepare(
|
||||
`SELECT fund_id, source, MAX(created_at) AS last_created
|
||||
FROM fund_position_records
|
||||
WHERE fund_id IN (${ids.map(() => '?').join(',')})
|
||||
GROUP BY fund_id, source`,
|
||||
).all(...ids) as Array<{ fund_id: string; source: string; last_created: string }>;
|
||||
for (const r of recs) lastByFundSource.set(`${r.fund_id}:${r.source}`, r.last_created);
|
||||
|
||||
for (const f of funds) {
|
||||
const handle = f.x_handle;
|
||||
const pull = handle
|
||||
? (queueByHandle.get(handle) ?? queueByHandle.get(handle.toLowerCase()) ?? EMPTY_PULL)
|
||||
: EMPTY_PULL;
|
||||
out.set(f.id, {
|
||||
xHandle: handle,
|
||||
xPull: pull,
|
||||
lastPostAt: handle ? (lastPostByHandle.get(handle.toLowerCase()) ?? null) : null,
|
||||
lastCaptureAt: lastByFundSource.get(`${f.id}:capture`) ?? null,
|
||||
last13fAt: lastByFundSource.get(`${f.id}:13f`) ?? null,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function fundFreshness(db: DatabaseSync, fund: TrackedFund): FundFreshness {
|
||||
return fundsFreshness(db, [fund]).get(fund.id) ?? {
|
||||
xHandle: fund.x_handle,
|
||||
xPull: EMPTY_PULL,
|
||||
lastPostAt: null,
|
||||
lastCaptureAt: null,
|
||||
last13fAt: null,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Position records / Live Book ─────────────────────────────────────────────
|
||||
|
||||
function rowToRecord(row: Record<string, any>): FundPositionRecord {
|
||||
@@ -152,6 +269,7 @@ export interface NewPositionRecord {
|
||||
as_of: string;
|
||||
source: PositionSource;
|
||||
evidence_url?: string | null;
|
||||
notes?: string | null;
|
||||
id?: string; // optional stable id (e.g. derived from 13F accession + symbol) for idempotency
|
||||
}
|
||||
|
||||
@@ -160,12 +278,12 @@ export function insertPositionRecord(db: DatabaseSync, input: NewPositionRecord)
|
||||
const id = input.id ?? randomUUID();
|
||||
const created_at = new Date().toISOString();
|
||||
db.prepare(
|
||||
`INSERT INTO fund_position_records (id, fund_id, symbol, shares, value_usd, cost_basis, as_of, source, evidence_url, created_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?)`,
|
||||
`INSERT INTO fund_position_records (id, fund_id, symbol, shares, value_usd, cost_basis, as_of, source, evidence_url, notes, created_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
).run(
|
||||
id, input.fund_id, input.symbol,
|
||||
input.shares ?? null, input.value_usd ?? null, input.cost_basis ?? null,
|
||||
input.as_of, input.source, input.evidence_url ?? null, created_at,
|
||||
input.as_of, input.source, input.evidence_url ?? null, input.notes ?? null, created_at,
|
||||
);
|
||||
const row = db.prepare('SELECT * FROM fund_position_records WHERE id = ?').get(id) as Record<string, any>;
|
||||
return rowToRecord(row);
|
||||
@@ -185,32 +303,82 @@ export function listRecords(db: DatabaseSync, fundId: string): FundPositionRecor
|
||||
*/
|
||||
export function liveBook(db: DatabaseSync, fundId: string): LiveBookPosition[] {
|
||||
const rows = db.prepare(
|
||||
`SELECT fpr.*, xcp.body_text AS post_text, xcp.posted_at AS post_date
|
||||
`SELECT fpr.*, xcp.body_text AS post_text, xcp.posted_at AS post_date,
|
||||
(
|
||||
SELECT p2.shares FROM fund_position_records p2
|
||||
WHERE p2.fund_id = fpr.fund_id AND p2.symbol = fpr.symbol
|
||||
AND CASE WHEN p2.notes IN ('call','put') THEN p2.notes ELSE '' END
|
||||
= CASE WHEN fpr.notes IN ('call','put') THEN fpr.notes ELSE '' END
|
||||
AND p2.shares IS NOT NULL AND p2.shares > 0
|
||||
ORDER BY p2.as_of DESC, p2.created_at DESC
|
||||
LIMIT 1
|
||||
) AS last_held_shares,
|
||||
(
|
||||
SELECT p3.shares FROM fund_position_records p3
|
||||
WHERE p3.fund_id = fpr.fund_id AND p3.symbol = fpr.symbol
|
||||
AND CASE WHEN p3.notes IN ('call','put') THEN p3.notes ELSE '' END
|
||||
= CASE WHEN fpr.notes IN ('call','put') THEN fpr.notes ELSE '' END
|
||||
AND p3.shares IS NOT NULL AND p3.shares > 0
|
||||
AND p3.as_of < fpr.as_of
|
||||
ORDER BY p3.as_of DESC, p3.created_at DESC
|
||||
LIMIT 1
|
||||
) AS prior_shares,
|
||||
EXISTS (
|
||||
SELECT 1 FROM fund_position_records p4
|
||||
WHERE p4.fund_id = fpr.fund_id AND p4.symbol = fpr.symbol
|
||||
AND p4.source = '13f' AND p4.shares IS NOT NULL AND p4.shares > 0
|
||||
AND CASE WHEN p4.notes IN ('call','put') THEN p4.notes ELSE '' END
|
||||
= CASE WHEN fpr.notes IN ('call','put') THEN fpr.notes ELSE '' END
|
||||
) AS has_13f
|
||||
FROM fund_position_records fpr
|
||||
LEFT JOIN x_cookie_posts xcp ON (
|
||||
fpr.source = 'capture'
|
||||
AND xcp.post_id = SUBSTR(fpr.evidence_url, INSTR(fpr.evidence_url, '/status/') + 8)
|
||||
)
|
||||
JOIN (
|
||||
SELECT symbol, MAX(as_of) AS max_as_of
|
||||
SELECT symbol,
|
||||
CASE WHEN notes IN ('call','put') THEN notes ELSE '' END AS inst,
|
||||
MAX(as_of) AS max_as_of
|
||||
FROM fund_position_records
|
||||
WHERE fund_id = ?
|
||||
GROUP BY symbol
|
||||
GROUP BY symbol, CASE WHEN notes IN ('call','put') THEN notes ELSE '' END
|
||||
) latest ON latest.symbol = fpr.symbol AND latest.max_as_of = fpr.as_of
|
||||
AND CASE WHEN fpr.notes IN ('call','put') THEN fpr.notes ELSE '' END = latest.inst
|
||||
WHERE fpr.fund_id = ?
|
||||
ORDER BY fpr.as_of DESC, fpr.symbol ASC`,
|
||||
).all(fundId, fundId) as Array<Record<string, any>>;
|
||||
return rows.map((r) => ({
|
||||
symbol: r.symbol,
|
||||
shares: r.shares ?? null,
|
||||
value_usd: r.value_usd ?? null,
|
||||
cost_basis: r.cost_basis ?? null,
|
||||
as_of: r.as_of,
|
||||
source: r.source as PositionSource,
|
||||
evidence_url: r.evidence_url ?? null,
|
||||
notes: r.notes ?? null,
|
||||
post_text: r.post_text ?? null,
|
||||
}));
|
||||
return rows.map((r) => {
|
||||
const shares = r.shares ?? null;
|
||||
const exited = shares === 0;
|
||||
const prior = r.prior_shares ?? null;
|
||||
let change: LiveBookPosition['change'] = null;
|
||||
let shareDelta: number | null = null;
|
||||
if (!exited) {
|
||||
if (prior == null) {
|
||||
change = 'new';
|
||||
} else if (shares != null) {
|
||||
shareDelta = shares - prior;
|
||||
change = shareDelta > 0 ? 'add' : shareDelta < 0 ? 'cut' : 'flat';
|
||||
}
|
||||
}
|
||||
return {
|
||||
symbol: r.symbol,
|
||||
shares,
|
||||
value_usd: r.value_usd ?? null,
|
||||
cost_basis: r.cost_basis ?? null,
|
||||
as_of: r.as_of,
|
||||
source: r.source as PositionSource,
|
||||
evidence_url: r.evidence_url ?? null,
|
||||
notes: r.notes ?? null,
|
||||
post_text: r.post_text ?? null,
|
||||
status: exited ? 'exited' as const : 'held' as const,
|
||||
last_held_shares: r.last_held_shares ?? null,
|
||||
prior_shares: prior,
|
||||
share_delta: shareDelta,
|
||||
change,
|
||||
x_only: (r.source === 'capture' || r.source === 'claim') && !r.has_13f,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// ─── 13F sync (per-fund-13f-ingest, sub-task 3) ──────────────────────────────
|
||||
@@ -226,7 +394,7 @@ export function sync13FIntoRecords(db: DatabaseSync, fundId: string): number {
|
||||
| undefined;
|
||||
if (!fund) return 0;
|
||||
const rows = db.prepare(
|
||||
`SELECT symbol, shares, value_usd, reported_quarter, accession
|
||||
`SELECT symbol, shares, value_usd, reported_quarter, accession, put_call
|
||||
FROM institution_filings
|
||||
WHERE filer_cik = ? AND form = '13F-HR' AND shares IS NOT NULL`,
|
||||
).all(fund.ci_key) as Array<Record<string, any>>;
|
||||
@@ -236,24 +404,99 @@ export function sync13FIntoRecords(db: DatabaseSync, fundId: string): number {
|
||||
for (const r of rows) {
|
||||
const asOf = quarterEnd(r.reported_quarter as string);
|
||||
const evidence = r.accession ? `sec|${r.accession}` : null;
|
||||
const notes = noteFromPutCall(r.put_call as string | null);
|
||||
const inst = notes ?? '';
|
||||
if (evidence) {
|
||||
// Converge: refresh the record when the 13F source row changed.
|
||||
const upd = db.prepare(
|
||||
`UPDATE fund_position_records SET shares=?, value_usd=?, as_of=? WHERE fund_id=? AND symbol=? AND source='13f' AND evidence_url=?`,
|
||||
).run(r.shares, r.value_usd, asOf, fundId, r.symbol, evidence);
|
||||
let upd = db.prepare(
|
||||
`UPDATE fund_position_records SET shares=?, value_usd=?, as_of=?, notes=?
|
||||
WHERE fund_id=? AND symbol=? AND source='13f' AND evidence_url=?
|
||||
AND CASE WHEN notes IN ('call','put') THEN notes ELSE '' END = ?`,
|
||||
).run(r.shares, r.value_usd, asOf, notes, fundId, r.symbol, evidence, inst);
|
||||
if (upd.changes === 0 && notes) {
|
||||
// One-time: unlabeled 13F row for a name that is only options.
|
||||
upd = db.prepare(
|
||||
`UPDATE fund_position_records SET shares=?, value_usd=?, as_of=?, notes=?
|
||||
WHERE fund_id=? AND symbol=? AND source='13f' AND evidence_url=?
|
||||
AND (notes IS NULL OR notes = '')`,
|
||||
).run(r.shares, r.value_usd, asOf, notes, fundId, r.symbol, evidence);
|
||||
}
|
||||
if (upd.changes > 0) {
|
||||
updated++;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
const existing = db.prepare(
|
||||
`SELECT id FROM fund_position_records WHERE fund_id=? AND symbol=? AND source='13f' AND evidence_url IS NULL AND as_of=?`,
|
||||
).get(fundId, r.symbol, asOf) as { id?: string } | undefined;
|
||||
`SELECT id FROM fund_position_records WHERE fund_id=? AND symbol=? AND source='13f'
|
||||
AND evidence_url IS NULL AND as_of=?
|
||||
AND CASE WHEN notes IN ('call','put') THEN notes ELSE '' END = ?`,
|
||||
).get(fundId, r.symbol, asOf, inst) as { id?: string } | undefined;
|
||||
if (existing) continue;
|
||||
}
|
||||
insertPositionRecord(db, {
|
||||
fund_id: fundId, symbol: r.symbol, shares: r.shares, value_usd: r.value_usd,
|
||||
as_of: asOf, source: '13f', evidence_url: evidence,
|
||||
as_of: asOf, source: '13f', evidence_url: evidence, notes,
|
||||
});
|
||||
inserted++;
|
||||
}
|
||||
|
||||
inserted += synthesize13FExits(db, fundId, fund.ci_key);
|
||||
return inserted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Names in an earlier 13F quarter that the latest quarter omitted become
|
||||
* zero-share 13F records as of that quarter-end.
|
||||
*/
|
||||
function synthesize13FExits(db: DatabaseSync, fundId: string, filerCik: string): number {
|
||||
const latest = db.prepare(
|
||||
`SELECT MAX(reported_quarter) AS q FROM institution_filings
|
||||
WHERE filer_cik = ? AND form = '13F-HR' AND shares IS NOT NULL AND shares > 0`,
|
||||
).get(filerCik) as { q: string | null };
|
||||
if (!latest?.q) return 0;
|
||||
|
||||
const latestRows = db.prepare(
|
||||
`SELECT symbol, accession, put_call FROM institution_filings
|
||||
WHERE filer_cik = ? AND form = '13F-HR' AND reported_quarter = ?
|
||||
AND shares IS NOT NULL AND shares > 0`,
|
||||
).all(filerCik, latest.q) as Array<{ symbol: string; accession: string | null; put_call: string | null }>;
|
||||
if (latestRows.length === 0) return 0;
|
||||
|
||||
const instKey = (symbol: string, putCall: string | null | undefined) =>
|
||||
`${symbol}|${noteFromPutCall(putCall) ?? ''}`;
|
||||
const latestKeys = new Set(latestRows.map((r) => instKey(r.symbol, r.put_call)));
|
||||
const accession = latestRows.find((r) => r.accession)?.accession ?? null;
|
||||
const asOf = quarterEnd(latest.q);
|
||||
const evidence = accession ? `sec|${accession}` : null;
|
||||
|
||||
const prior = db.prepare(
|
||||
`SELECT DISTINCT symbol, put_call FROM institution_filings
|
||||
WHERE filer_cik = ? AND form = '13F-HR' AND reported_quarter < ?
|
||||
AND shares IS NOT NULL AND shares > 0`,
|
||||
).all(filerCik, latest.q) as Array<{ symbol: string; put_call: string | null }>;
|
||||
|
||||
let inserted = 0;
|
||||
for (const row of prior) {
|
||||
const notes = noteFromPutCall(row.put_call);
|
||||
if (latestKeys.has(instKey(row.symbol, row.put_call))) continue;
|
||||
const inst = notes ?? '';
|
||||
if (evidence) {
|
||||
const upd = db.prepare(
|
||||
`UPDATE fund_position_records SET shares=0, value_usd=0, as_of=?
|
||||
WHERE fund_id=? AND symbol=? AND source='13f' AND evidence_url=?
|
||||
AND CASE WHEN notes IN ('call','put') THEN notes ELSE '' END = ?`,
|
||||
).run(asOf, fundId, row.symbol, evidence, inst);
|
||||
if (upd.changes > 0) continue;
|
||||
} else {
|
||||
const existing = db.prepare(
|
||||
`SELECT id FROM fund_position_records
|
||||
WHERE fund_id=? AND symbol=? AND source='13f' AND evidence_url IS NULL AND as_of=? AND shares=0
|
||||
AND CASE WHEN notes IN ('call','put') THEN notes ELSE '' END = ?`,
|
||||
).get(fundId, row.symbol, asOf, inst) as { id?: string } | undefined;
|
||||
if (existing) continue;
|
||||
}
|
||||
insertPositionRecord(db, {
|
||||
fund_id: fundId, symbol: row.symbol, shares: 0, value_usd: 0,
|
||||
as_of: asOf, source: '13f', evidence_url: evidence, notes,
|
||||
});
|
||||
inserted++;
|
||||
}
|
||||
|
||||
@@ -520,6 +520,7 @@ CREATE TABLE IF NOT EXISTS x_cookie_posts (
|
||||
sentiment_score REAL, -- -1.0 to +1.0
|
||||
attribution TEXT, -- original author handle (may differ from author_handle for quotes/retweets)
|
||||
cached_until TEXT NOT NULL, -- 7d rolling; rows stale after this
|
||||
quoted_post_id TEXT, -- native quote target, when known
|
||||
PRIMARY KEY (post_id)
|
||||
);
|
||||
|
||||
@@ -1019,6 +1020,23 @@ CREATE TABLE IF NOT EXISTS confluence_signal_history (
|
||||
verdict TEXT -- real|false_alarm
|
||||
);
|
||||
|
||||
-- Walk-forward derived entry/exit rules (one active pair per rack).
|
||||
CREATE TABLE IF NOT EXISTS confluence_zone_rules (
|
||||
id TEXT PRIMARY KEY,
|
||||
rack_id TEXT NOT NULL REFERENCES confluence_racks(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL, -- entry | exit
|
||||
version INTEGER NOT NULL,
|
||||
derived_at TEXT NOT NULL,
|
||||
predicate_json TEXT NOT NULL,
|
||||
train_stats_json TEXT NOT NULL,
|
||||
validate_stats_json TEXT NOT NULL,
|
||||
sample_caveat TEXT NOT NULL,
|
||||
source TEXT NOT NULL, -- validated | baseline
|
||||
is_active INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_confluence_zone_rules_rack
|
||||
ON confluence_zone_rules(rack_id, kind, is_active);
|
||||
|
||||
-- ===== Price Corridor module (M24). =====
|
||||
-- Daily valuation-corridor snapshots per symbol: the observable forward/trailing
|
||||
-- P/E corridor the Corridor Method derives entry timing and implied upside from.
|
||||
|
||||
+52
-14
@@ -238,17 +238,59 @@ housekeepTimer.unref();
|
||||
// over the confluence universe, persisting rack evaluations + signal history.
|
||||
// Idempotent per (symbol, asOf, rack) — cheap to run more often than daily.
|
||||
const CONFLUENCE_EVAL_MS = 60 * 60 * 1000; // hourly tick; re-eval only when stale
|
||||
let lastZoneDeriveAt = 0;
|
||||
const ZONE_DERIVE_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
|
||||
async function confluenceTick(label: string): Promise<void> {
|
||||
const {
|
||||
runConfluenceEvaluationCycle,
|
||||
runConfluenceReplay,
|
||||
deriveAndPersistZoneRules,
|
||||
fillLearningLedger,
|
||||
LEARNING_REPLAY_DAYS,
|
||||
LEARNING_REPLAY_SYMBOLS,
|
||||
} = await import('./confluence/confluenceEngine.ts');
|
||||
const { CONFLUENCE_LEARNING_UNIVERSE } = await import('./confluence/confluenceSeed.ts');
|
||||
const replay = label === 'boot'
|
||||
? await fillLearningLedger(database, cache, 75_000)
|
||||
: await runConfluenceReplay(database, cache, {
|
||||
symbols: CONFLUENCE_LEARNING_UNIVERSE,
|
||||
symbolsPerTick: LEARNING_REPLAY_SYMBOLS,
|
||||
budgetDaysPerSymbol: LEARNING_REPLAY_DAYS,
|
||||
});
|
||||
if (replay.evaluationsStored > 0 || replay.symbolsTouched.length > 0) {
|
||||
console.log(
|
||||
`[confluence] ${label} replay stored=${replay.evaluationsStored} symbols=${replay.symbolsTouched.join(',')} ` +
|
||||
`remainingDays=${replay.remainingDays} complete=${replay.complete}`,
|
||||
);
|
||||
}
|
||||
const summary = await runConfluenceEvaluationCycle(database, cache);
|
||||
if (summary.symbolsEvaluated.length > 0) {
|
||||
console.log(
|
||||
`[confluence] ${label} evaluated=${summary.symbolsEvaluated.length} stored=${summary.evaluationsStored} ` +
|
||||
`reused=${summary.evaluationsReused} signals=${summary.signalsLogged} resolved=${summary.signalsResolved} ` +
|
||||
`corridorSnapshots=${summary.corridorSnapshots}`,
|
||||
);
|
||||
}
|
||||
const now = Date.now();
|
||||
if (now - lastZoneDeriveAt > ZONE_DERIVE_MS || lastZoneDeriveAt === 0) {
|
||||
try {
|
||||
const derived = await deriveAndPersistZoneRules(database, cache);
|
||||
if (derived.skipped) {
|
||||
console.log(`[confluence] zone rules deferred: ${derived.skipped}`);
|
||||
} else {
|
||||
lastZoneDeriveAt = now;
|
||||
console.log(`[confluence] zone rules derived source=${derived.source}`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[confluence] zone rule derive failed:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const confluenceEvalTimer = setInterval(async () => {
|
||||
try {
|
||||
const { runConfluenceEvaluationCycle } = await import('./confluence/confluenceEngine.ts');
|
||||
const summary = await runConfluenceEvaluationCycle(database, cache);
|
||||
if (summary.symbolsEvaluated.length > 0) {
|
||||
console.log(
|
||||
`[confluence] evaluated=${summary.symbolsEvaluated.length} stored=${summary.evaluationsStored} ` +
|
||||
`reused=${summary.evaluationsReused} signals=${summary.signalsLogged} resolved=${summary.signalsResolved} ` +
|
||||
`corridorSnapshots=${summary.corridorSnapshots}`,
|
||||
);
|
||||
}
|
||||
await confluenceTick('hourly');
|
||||
} catch (e) {
|
||||
console.error('[confluence] evaluation cycle failed:', e);
|
||||
}
|
||||
@@ -257,11 +299,7 @@ confluenceEvalTimer.unref();
|
||||
// First cycle shortly after boot once the EOD refresh has had a chance to land.
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
const { runConfluenceEvaluationCycle } = await import('./confluence/confluenceEngine.ts');
|
||||
const summary = await runConfluenceEvaluationCycle(database, cache);
|
||||
if (summary.symbolsEvaluated.length > 0) {
|
||||
console.log(`[confluence] boot cycle: ${summary.symbolsEvaluated.length} symbols, ${summary.evaluationsStored} evaluations stored`);
|
||||
}
|
||||
await confluenceTick('boot');
|
||||
} catch (e) {
|
||||
console.error('[confluence] boot cycle failed:', e);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Capture pipeline classification/parser tests (sub-task 4).
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { classifyPost, extractCaptures } from '../captureParser.ts';
|
||||
import { classifyPost, extractCaptures, extractRemainingBookTickers, inferResetExits } from '../captureParser.ts';
|
||||
|
||||
test('capture: total-position post with numbers', () => {
|
||||
const r = classifyPost('IREN position: 750,000 shares, avg cost $22.40, market value $28.3M.');
|
||||
@@ -19,6 +19,26 @@ test('capture: totals without explicit marker still parse via position/holdings'
|
||||
assert.ok(Math.abs(r.cost_basis! - 0.74) < 0.001);
|
||||
});
|
||||
|
||||
test('claim: follow-on add with average now prefers 1.69 over the clip print', () => {
|
||||
const r = classifyPost(
|
||||
"Quick update on this one. I added another 123,000 shares to this today at $1.62 net. Average now $1.69.",
|
||||
);
|
||||
assert.equal(r.class, 'claim');
|
||||
assert.equal(r.symbol, undefined);
|
||||
assert.equal(r.shares, 123000);
|
||||
assert.ok(Math.abs((r.cost_basis ?? 0) - 1.69) < 0.001);
|
||||
});
|
||||
|
||||
test('claim: Mike ENHA add is a claim with ticker + size + print', () => {
|
||||
const r = classifyPost(
|
||||
"Seeing some very strange trading in ENHA. So without even thinking about it, I added 115,000 shares at $1.79. I still believe in my friend @C_Angermayer's vision. Still too early to write this one off.",
|
||||
);
|
||||
assert.equal(r.class, 'claim');
|
||||
assert.equal(r.symbol, 'ENHA');
|
||||
assert.equal(r.shares, 115000);
|
||||
assert.ok(Math.abs((r.cost_basis ?? 0) - 1.79) < 0.001);
|
||||
});
|
||||
|
||||
test('claim: "added n shares at x average price"', () => {
|
||||
const r = classifyPost('Added 865,000 shares of BKKT at $3.10 average price today.');
|
||||
assert.equal(r.class, 'claim');
|
||||
@@ -35,6 +55,39 @@ test('claim: bought / sold with quantity', () => {
|
||||
assert.equal(sold.class, 'claim');
|
||||
});
|
||||
|
||||
test('full exit: sold out of $PEP is a zero-share capture', () => {
|
||||
const r = classifyPost('Sold out of $PEP this morning. Moving on.');
|
||||
assert.equal(r.class, 'capture');
|
||||
assert.equal(r.symbol, 'PEP');
|
||||
assert.equal(r.shares, 0);
|
||||
});
|
||||
|
||||
test('full exit: sold all of / closed the position / no longer hold / fully out / flat in', () => {
|
||||
assert.equal(classifyPost('Sold all of my DEO.').shares, 0);
|
||||
assert.equal(classifyPost('Sold all of my DEO.').symbol, 'DEO');
|
||||
assert.equal(classifyPost('Closed out the BMY position yesterday.').symbol, 'BMY');
|
||||
assert.equal(classifyPost('Closed out the BMY position yesterday.').shares, 0);
|
||||
assert.equal(classifyPost('We no longer hold LEN.').symbol, 'LEN');
|
||||
assert.equal(classifyPost('Fully out of $GLPI.').symbol, 'GLPI');
|
||||
assert.equal(classifyPost('Flat in $PYPL now.').symbol, 'PYPL');
|
||||
assert.equal(classifyPost('Flat in $PYPL now.').shares, 0);
|
||||
});
|
||||
|
||||
test('full exit: sold 2,000 shares of CIFR stays a claim, not an exit', () => {
|
||||
const r = classifyPost('Sold 2,000 shares of CIFR today.');
|
||||
assert.equal(r.class, 'claim');
|
||||
assert.notEqual(r.shares, 0);
|
||||
});
|
||||
|
||||
test('full exit: closed the gap / tax-bill sell / sold everything are not exits', () => {
|
||||
assert.equal(classifyPost('They closed the gap at yesterday\'s open in CIFR.').class, 'none');
|
||||
assert.equal(
|
||||
classifyPost('Sold $1.7M of income producing securities in my personal accounts this morning in order to cover the remainder of my 2025 federal tax bill.').class,
|
||||
'none',
|
||||
);
|
||||
assert.equal(classifyPost('Sold everything. Cleaning house.').class, 'none');
|
||||
});
|
||||
|
||||
test('commentary: no numbers → never a record', () => {
|
||||
const r = classifyPost('Huge week for the portfolio. Love the setup here.');
|
||||
assert.equal(r.class, 'none');
|
||||
@@ -131,3 +184,39 @@ OPEN now over 6.45M shares. Have taken it up on the weakness. RKT/Z also weak. S
|
||||
assert.equal(slnh!.class, 'claim');
|
||||
assert.equal(slnh!.shares, 100_000);
|
||||
});
|
||||
|
||||
test('book reset: what stayed lists remaining tickers', () => {
|
||||
const kept = extractRemainingBookTickers('Cleaning the book. What stayed: $CIFR $IREN $OPEN');
|
||||
assert.ok(kept);
|
||||
assert.deepEqual([...kept!].sort(), ['CIFR', 'IREN', 'OPEN']);
|
||||
});
|
||||
|
||||
test('book reset: top-10 list is not a remaining-book list', () => {
|
||||
assert.equal(extractRemainingBookTickers('Top 10 positions excluding ETFs:\n1. $CIFR\n2. $IREN\n3. $OPEN'), null);
|
||||
});
|
||||
|
||||
test('Mr T: back in $OUST at a price is a hold, not a skipped claim', () => {
|
||||
const multi = extractCaptures('back in $OUST.\n\nbought again a few days ago at $46.02. this is the exact name that gave us 40% earlier this year.');
|
||||
const oust = multi.find((c) => c.symbol === 'OUST');
|
||||
assert.ok(oust);
|
||||
assert.equal(oust!.class, 'capture');
|
||||
assert.ok(Math.abs((oust!.cost_basis ?? 0) - 46.02) < 0.001);
|
||||
});
|
||||
|
||||
test('Mr T: buyingg multi-name list materializes each ticker', () => {
|
||||
const multi = extractCaptures('buyingg $be 238.95 $bksy 31.86 $oust 46.02\n\nim gonna + more');
|
||||
assert.deepEqual(multi.map((c) => c.symbol).sort(), ['BE', 'BKSY', 'OUST']);
|
||||
assert.ok(multi.every((c) => c.class === 'capture'));
|
||||
assert.ok(Math.abs((multi.find((c) => c.symbol === 'BE')!.cost_basis ?? 0) - 238.95) < 0.001);
|
||||
});
|
||||
|
||||
test('Mr T: trim post is not a reentry hold', () => {
|
||||
const multi = extractCaptures('trimmed some $CBRS today.\n\nbought at 225 last week. sold some at 256.');
|
||||
assert.ok(!multi.some((c) => c.class === 'capture' && c.symbol === 'CBRS'));
|
||||
});
|
||||
|
||||
test('book reset: infer exits for names held but absent from the stayed list', () => {
|
||||
const exits = inferResetExits(['CIFR', 'IREN', 'DEO', 'BMY'], ['CIFR', 'IREN']);
|
||||
assert.deepEqual(exits.map((e) => e.symbol).sort(), ['BMY', 'DEO']);
|
||||
assert.ok(exits.every((e) => e.class === 'capture' && e.shares === 0));
|
||||
});
|
||||
|
||||
@@ -68,6 +68,24 @@ test('rounding: whole-share tick by default, fractional with tick 0.01', () => {
|
||||
assert.ok(Math.abs(r2.rows[0].target_qty! - 479.62) < 0.011);
|
||||
});
|
||||
|
||||
test('zero-share exits are dropped from weights; user holding gets close-to-zero delta', () => {
|
||||
const book = [
|
||||
{ symbol: 'IREN', shares: 100, value_usd: 5000, cost_basis: 40 },
|
||||
{ symbol: 'DEO', shares: 0, value_usd: 0, cost_basis: null },
|
||||
];
|
||||
const prices = { IREN: 50, DEO: 100 };
|
||||
const r = mirrorBook(book, [{ symbol: 'DEO', qty: 20, avg_cost: 80 }], prices, { base: 10000, minPositionUsd: 1 });
|
||||
assert.equal(r.rows.some((x) => x.symbol === 'IREN'), true);
|
||||
assert.ok(Math.abs(r.rows.find((x) => x.symbol === 'IREN')!.weight - 1) < 0.001);
|
||||
const deo = r.rows.find((x) => x.symbol === 'DEO');
|
||||
assert.ok(deo, 'user still holds the exited name');
|
||||
assert.equal(deo!.weight, 0);
|
||||
assert.equal(deo!.target_qty, 0);
|
||||
assert.equal(deo!.delta_qty, -20);
|
||||
assert.match(deo!.message, /delta/i);
|
||||
assert.ok(!/\byou should\b/i.test(deo!.message));
|
||||
});
|
||||
|
||||
test('ADR-0010: messages are mechanical (never advice-shaped)', () => {
|
||||
const book = [{ symbol: 'IREN', shares: 100, value_usd: 5000, cost_basis: 40 }];
|
||||
const prices = { IREN: 50 };
|
||||
|
||||
@@ -101,6 +101,10 @@ function findValue(text: string): number | undefined {
|
||||
}
|
||||
|
||||
function findCostBasis(text: string): number | undefined {
|
||||
// Stated book average wins over the print on this clip ("Average now $1.69"
|
||||
// after "added … at $1.62").
|
||||
const nowAvg = text.match(/\baverage now\s+\$?\s*([\d.,]+)/i);
|
||||
if (nowAvg) return toNum(nowAvg[1]);
|
||||
// "at $22.40 avg" / "avg cost $22.40" / "cost basis $22.40" / "@ $7" /
|
||||
// "average down to $4.42" / "at a $1.25 average"
|
||||
const m = text.match(/(?:at|avg(?: cost)?|average|cost basis|basis|bought at)[^$\d]{0,16}\$\s*([\d.,]+)/i)
|
||||
@@ -108,6 +112,60 @@ function findCostBasis(text: string): number | undefined {
|
||||
return m ? toNum(m[1]) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-exit language (position is gone), not a sized trim.
|
||||
* "Sold 2,000 shares of CIFR" is a claim and must not match.
|
||||
*/
|
||||
export function extractFullExit(text: string): ParsedCapture | null {
|
||||
const t = text || '';
|
||||
const patterns: RegExp[] = [
|
||||
/\bsold\s+out\s+of\s+\$?([A-Za-z]{1,5})\b/i,
|
||||
/\bsold\s+all\s+of\s+(?:my\s+|the\s+)?\$?([A-Za-z]{1,5})\b/i,
|
||||
/\bclosed\s+(?:out\s+)?the\s+\$?([A-Za-z]{1,5})\s+position\b/i,
|
||||
/\bno\s+longer\s+holds?\s+\$?([A-Za-z]{1,5})\b/i,
|
||||
/\bfully\s+out\s+of\s+\$?([A-Za-z]{1,5})\b/i,
|
||||
/\bflat\s+in\s+\$([A-Za-z]{1,5})\b/i,
|
||||
/\bflat\s+in\s+([A-Z]{2,5})\b/,
|
||||
];
|
||||
for (const re of patterns) {
|
||||
const m = t.match(re);
|
||||
if (!m || !isTickerCandidate(m[1])) continue;
|
||||
return { class: 'capture', symbol: m[1].toUpperCase(), shares: 0, instrument: 'equity' };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const REMAINING_BOOK_MARKERS =
|
||||
/\b(?:what stayed|cleaning the book|sold almost everything|book is now)\b/i;
|
||||
|
||||
/** Tickers named as the remaining book, or null if this is not a reset post. */
|
||||
export function extractRemainingBookTickers(text: string): Set<string> | null {
|
||||
const t = text || '';
|
||||
if (!REMAINING_BOOK_MARKERS.test(t)) return null;
|
||||
const out = new Set<string>();
|
||||
for (const m of t.matchAll(/\$([A-Za-z]{1,5})\b/g)) {
|
||||
if (isTickerCandidate(m[1])) out.add(m[1].toUpperCase());
|
||||
}
|
||||
const after = t.match(
|
||||
/\b(?:what stayed|kept|still (?:holding|long)|remain(?:ing)?|book is now)\s*:?\s*([^\n]+)/i,
|
||||
);
|
||||
if (after) {
|
||||
for (const tok of after[1].match(/\b[A-Z]{1,5}\b/g) ?? []) {
|
||||
if (isTickerCandidate(tok)) out.add(tok);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Names held before a reset post that the stayed-list omitted. */
|
||||
export function inferResetExits(heldSymbols: string[], stayed: Iterable<string>): ParsedCapture[] {
|
||||
const keep = new Set([...stayed].map((s) => s.toUpperCase()));
|
||||
return heldSymbols
|
||||
.map((s) => s.toUpperCase())
|
||||
.filter((s) => s && !keep.has(s))
|
||||
.map((symbol) => ({ class: 'capture' as const, symbol, shares: 0, instrument: 'equity' as const }));
|
||||
}
|
||||
|
||||
function findInstrument(text: string): InstrumentKind {
|
||||
// "WULF calls", "$20 Sept WULF calls", "buying calls on WULF"
|
||||
if (/\b[A-Z]{1,5}\s+calls?\b/.test(text) || /\bcalls?\s+(?:on|in)\s+/i.test(text) || /\bcall options?\b/i.test(text)) {
|
||||
@@ -214,6 +272,9 @@ export function extractCaptures(text: string): ParsedCapture[] {
|
||||
out.push(p);
|
||||
};
|
||||
|
||||
const fullExit = extractFullExit(t);
|
||||
if (fullExit) push(fullExit);
|
||||
|
||||
// Totals: "OPEN now over 6.45M shares" / "SLNH now at 3.09M shares"
|
||||
for (const m of t.matchAll(
|
||||
/\b([A-Z]{1,5})\s+now\s+(?:over|at|above|about|under|~|more than)?\s*([\d.,]+)\s*([MBKmbk])?\s*shares\b/g,
|
||||
@@ -271,14 +332,71 @@ export function extractCaptures(text: string): ParsedCapture[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
if (extractRemainingBookTickers(t)) {
|
||||
return out;
|
||||
}
|
||||
|
||||
for (const p of extractReentryCaptures(t)) push(p);
|
||||
if (out.length > 0) return out;
|
||||
|
||||
// Single-capture path
|
||||
const one = classifyPost(t);
|
||||
return one.class === 'none' ? [] : [one];
|
||||
}
|
||||
|
||||
const REENTRY_HOLD =
|
||||
/\bback in\b|\bnew position\b|\bbuyingg?\b|\bbought again\b/i;
|
||||
|
||||
/**
|
||||
* Mr T style: "back in $OUST" / "buyingg $BE 238.95 $BKSY 31.86".
|
||||
* These are current-book holds, not sized trims ("sold 2,000 shares").
|
||||
*/
|
||||
export function extractReentryCaptures(text: string): ParsedCapture[] {
|
||||
const t = text || '';
|
||||
if (!REENTRY_HOLD.test(t)) return [];
|
||||
if (/\btrimmed?\b|\bsold some\b|\bshaving\b/i.test(t) && !/\bback in\b|\bnew position\b|\bbuyingg?\b/i.test(t)) {
|
||||
return [];
|
||||
}
|
||||
const out: ParsedCapture[] = [];
|
||||
const seen = new Set<string>();
|
||||
const push = (symbol: string, cost?: number) => {
|
||||
const sym = symbol.toUpperCase();
|
||||
// Cashtags are explicit ($BE = Bloom Energy); skip only 1-letter junk.
|
||||
if (sym.length < 2 || sym.length > 5 || !/^[A-Z]+$/.test(sym)) return;
|
||||
const existing = out.find((p) => p.symbol === sym);
|
||||
if (existing) {
|
||||
if (cost !== undefined && existing.cost_basis === undefined) existing.cost_basis = cost;
|
||||
return;
|
||||
}
|
||||
seen.add(sym);
|
||||
out.push({
|
||||
class: 'capture',
|
||||
symbol: sym,
|
||||
cost_basis: cost,
|
||||
instrument: 'equity',
|
||||
});
|
||||
};
|
||||
|
||||
for (const m of t.matchAll(/\b(?:back in|new position(?:\s*[-–])?|buyingg?|bought again(?:\s+in)?)\s+\$([A-Za-z]{1,5})\b/gi)) {
|
||||
push(m[1]);
|
||||
}
|
||||
if (/\bbuyingg?\b/i.test(t)) {
|
||||
for (const m of t.matchAll(/\$([A-Za-z]{1,5})\s+(\d+(?:\.\d+)?)/g)) {
|
||||
push(m[1], Number(m[2]));
|
||||
}
|
||||
}
|
||||
const cost = findCostBasis(t);
|
||||
if (out.length === 1 && cost !== undefined && out[0].cost_basis === undefined) {
|
||||
out[0].cost_basis = cost;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Classify a post and extract structured numbers. Deterministic. */
|
||||
export function classifyPost(text: string): ParsedCapture {
|
||||
const t = text || '';
|
||||
const fullExit = extractFullExit(t);
|
||||
if (fullExit) return fullExit;
|
||||
const qty = findQty(t);
|
||||
const cost = findCostBasis(t);
|
||||
const value = findValue(t);
|
||||
@@ -306,8 +424,13 @@ export function classifyPost(text: string): ParsedCapture {
|
||||
|
||||
const base = { symbol, shares, cost_basis: cost, value_usd: value, instrument };
|
||||
|
||||
const reentry = extractReentryCaptures(t);
|
||||
if (reentry.length === 1) {
|
||||
return { ...reentry[0], shares, cost_basis: reentry[0].cost_basis ?? cost, value_usd: value };
|
||||
}
|
||||
|
||||
const claimHit = CLAIM_MARKERS.some((re) => re.test(t));
|
||||
if (claimHit && (qty !== undefined || cost !== undefined)) {
|
||||
if (claimHit && (qty !== undefined || cost !== undefined) && reentry.length === 0) {
|
||||
// Claims stay equity-oriented for v1 (delta folding is future work).
|
||||
return { class: 'claim', ...base, shares: qty };
|
||||
}
|
||||
@@ -324,13 +447,16 @@ export function classifyPost(text: string): ParsedCapture {
|
||||
// Conversational captures / option initiations: "buying ... WULF calls",
|
||||
// "bought $X", "initiating position".
|
||||
const strongAction = /\b(?:bought|added|adding|buying|buy|initiat(?:ing|ed)|new (?:position|entry)|we (?:buy|adding))\b/i.test(t);
|
||||
if (strongAction && symbol) {
|
||||
const isTrim = /\btrimmed?\b|\bsold some\b|\bshaving\b/i.test(t);
|
||||
if (strongAction && symbol && !isTrim) {
|
||||
return { class: 'capture', ...base };
|
||||
}
|
||||
|
||||
// Position list patterns: "what stayed: $X $Y $Z" / "kept: $X $Y"
|
||||
// Only a capture when the post also has a size; a bare stayed-list is
|
||||
// handled by extractRemainingBookTickers + inferResetExits at ingest.
|
||||
const listPattern = /\b(?:what stayed|kept|still (?:holding|long)|remain(?:ing)?)\s*:/i.test(t);
|
||||
if (listPattern && symbol) {
|
||||
if (listPattern && symbol && hasAccountNumbers) {
|
||||
return { class: 'capture', ...base };
|
||||
}
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ export async function fetchFund13F(db: DatabaseSync, fundId: string): Promise<Fu
|
||||
`INSERT INTO institution_filings
|
||||
(filer_cik, filer_name, symbol, form, shares, value_usd, reported_quarter, filed_at, fetched_at, accession, put_call)
|
||||
VALUES (?, ?, ?, '13F-HR', ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(filer_cik, symbol, reported_quarter, form) DO UPDATE SET
|
||||
ON CONFLICT(filer_cik, symbol, reported_quarter, form, put_call) DO UPDATE SET
|
||||
shares = excluded.shares,
|
||||
value_usd = excluded.value_usd,
|
||||
accession = excluded.accession,
|
||||
@@ -114,27 +114,27 @@ export async function fetchFund13F(db: DatabaseSync, fundId: string): Promise<Fu
|
||||
}>;
|
||||
holdingsSeen += holdings.length;
|
||||
|
||||
// Aggregate per symbol (13F lists shares + separate put/call rows for the
|
||||
// same name; uq_inst_filings allows one row per (cik, symbol, quarter, form)).
|
||||
const bySymbol = new Map<string, { shares: number; value: number; putCall: string }>();
|
||||
// Keep common stock and listed options on separate rows (same CUSIP, different putCall).
|
||||
const byKey = new Map<string, { symbol: string; shares: number; value: number; putCall: string }>();
|
||||
for (const h of holdings) {
|
||||
const symbol = cusipToSymbol(db, h.cusip, h.issuerName);
|
||||
if (!symbol) {
|
||||
unresolved.push({ cusip: h.cusip, issuerName: h.issuerName });
|
||||
continue;
|
||||
}
|
||||
const agg = bySymbol.get(symbol) ?? { shares: 0, value: 0, putCall: '' };
|
||||
const putCall = (h.putCall || '').trim();
|
||||
const key = `${symbol}\0${putCall}`;
|
||||
const agg = byKey.get(key) ?? { symbol, shares: 0, value: 0, putCall };
|
||||
agg.shares += h.sshPrnamt ?? 0;
|
||||
agg.value += h.value ?? 0;
|
||||
if (h.putCall) agg.putCall = h.putCall;
|
||||
bySymbol.set(symbol, agg);
|
||||
byKey.set(key, agg);
|
||||
}
|
||||
|
||||
for (const [symbol, agg] of bySymbol) {
|
||||
for (const agg of byKey.values()) {
|
||||
if (agg.shares <= 0 && agg.value <= 0) continue;
|
||||
upsert.run(
|
||||
fund.ci_key, null, symbol, agg.shares, agg.value, quarter, fileDate, now, accession,
|
||||
agg.putCall || null,
|
||||
fund.ci_key, null, agg.symbol, agg.shares, agg.value, quarter, fileDate, now, accession,
|
||||
agg.putCall,
|
||||
);
|
||||
upserted++;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
// re-baselines to actual equity; the `locked` option pins a fixed-size sleeve.
|
||||
//
|
||||
// ADR-0010: the ONLY strings this module can produce are MECHANICAL (arithmetic)
|
||||
// statements — "to match your mirror target, the delta is N shares (~$X)". It never
|
||||
// statements — "to match the tracked fund book, the delta is N shares (~$X)". It never
|
||||
// forms a recommending imperative ("act on this because the fund did"). The
|
||||
// ADR-0010 boundary is also enforced by the M21 Primary-Rule lint (sub-task 9).
|
||||
|
||||
@@ -87,7 +87,7 @@ function fmtUsd(value: number): string {
|
||||
function buildMessage(symbol: string, deltaQty: number, deltaValue: number): string {
|
||||
const qty = Math.abs(Math.round(deltaQty * 100) / 100);
|
||||
const verb = deltaQty >= 0 ? 'increase' : 'trim';
|
||||
return `To match your mirror target for ${symbol}, the delta is ${verb} of ${qty} shares (about $${fmtUsd(Math.abs(deltaValue))}).`;
|
||||
return `To match the tracked fund book for ${symbol}, the delta is ${verb} of ${qty} shares (about $${fmtUsd(Math.abs(deltaValue))}).`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -116,10 +116,17 @@ export function mirrorBook(
|
||||
const bookValueMap = new Map<string, number | null>(); // symbol -> fund avg cost
|
||||
for (const p of book) bookValueMap.set(p.symbol.toUpperCase(), p.cost_basis ?? null);
|
||||
|
||||
const active = book.filter((p) => p.shares == null || p.shares > 0);
|
||||
const exitedHeld = book.filter((p) => {
|
||||
if (p.shares !== 0) return false;
|
||||
const hold = actualBy.get(p.symbol.toUpperCase());
|
||||
return !!hold && hold.qty > 0;
|
||||
});
|
||||
|
||||
// Mark the fund book to current market: prefer shares × price; fall back to the
|
||||
// recorded value when we lack shares or a price.
|
||||
let bookValue = 0;
|
||||
const val = book.map((p) => {
|
||||
const val = active.map((p) => {
|
||||
const sym = p.symbol.toUpperCase();
|
||||
const price = prices[sym];
|
||||
let value: number;
|
||||
@@ -169,6 +176,29 @@ export function mirrorBook(
|
||||
(belowFloor ? excluded : rows).push(row);
|
||||
}
|
||||
|
||||
for (const p of exitedHeld) {
|
||||
const sym = p.symbol.toUpperCase();
|
||||
const hold = actualBy.get(sym)!;
|
||||
const price = prices[sym];
|
||||
const actualQty = hold.qty;
|
||||
const actualValue = typeof price === 'number' && price > 0 ? actualQty * price : 0;
|
||||
const delta = 0 - actualQty;
|
||||
const deltaVal = delta * (price ?? 0);
|
||||
rows.push({
|
||||
symbol: sym,
|
||||
weight: 0,
|
||||
fund_avg_cost: bookValueMap.get(sym) ?? null,
|
||||
target_value: 0,
|
||||
target_qty: 0,
|
||||
actual_qty: actualQty,
|
||||
actual_value: actualValue,
|
||||
delta_qty: roundTick(delta, tick),
|
||||
delta_value: deltaVal,
|
||||
message: buildMessage(sym, delta, deltaVal),
|
||||
excluded: false,
|
||||
});
|
||||
}
|
||||
|
||||
const order = (a: MirrorRow, b: MirrorRow) => b.weight - a.weight || a.symbol.localeCompare(b.symbol);
|
||||
rows.sort(order);
|
||||
excluded.sort(order);
|
||||
|
||||
@@ -141,12 +141,20 @@ export function assembleSurfaceFromCache(
|
||||
expiries = fromChains.map((r) => r.expiry);
|
||||
}
|
||||
|
||||
const asOfDay = (opts.asOf ?? new Date().toISOString()).slice(0, 10);
|
||||
let newestChain: string | null = null;
|
||||
for (const expiry of expiries) {
|
||||
const row = db.prepare(
|
||||
'SELECT MAX(ts) AS ts FROM options_chains WHERE symbol=? AND expiry=?',
|
||||
).get(symbol, expiry) as { ts: string | null } | undefined;
|
||||
if (row?.ts && (!newestChain || row.ts > newestChain)) newestChain = row.ts;
|
||||
}
|
||||
|
||||
const asOf = opts.asOf ?? newestChain ?? quote.observed_at;
|
||||
const asOfDay = asOf.slice(0, 10);
|
||||
const liveExpiries = expiries.filter((e) => e >= asOfDay).slice(0, maxExpiries);
|
||||
if (liveExpiries.length === 0) return null;
|
||||
|
||||
const rows: NormalizedOptionRow[] = [];
|
||||
let latestTs = quote.observed_at;
|
||||
|
||||
for (const expiry of liveExpiries) {
|
||||
// Latest snapshot per (strike, type) for this expiry
|
||||
@@ -184,29 +192,61 @@ export function assembleSurfaceFromCache(
|
||||
? { delta, gamma, theta, vega, vanna: null }
|
||||
: null,
|
||||
});
|
||||
if (typeof r.ts === 'string' && r.ts > latestTs) latestTs = r.ts;
|
||||
if (typeof r.ts === 'string' && (!newestChain || r.ts > newestChain)) newestChain = r.ts;
|
||||
}
|
||||
}
|
||||
|
||||
if (rows.length === 0) return null;
|
||||
|
||||
const asOf = opts.asOf ?? latestTs ?? new Date().toISOString();
|
||||
const spotHit = spotAt(db, symbol, asOf, quote);
|
||||
return {
|
||||
symbol,
|
||||
spot: quote.price,
|
||||
spot: spotHit.price,
|
||||
asOf,
|
||||
quoteAsOf: spotHit.asOf,
|
||||
chainAsOf: newestChain,
|
||||
providerId: providerId === 'synthetic' ? 'synthetic' : providerId,
|
||||
delaySeconds: meta.delaySeconds,
|
||||
oiFreshness: meta.oiFreshness,
|
||||
rows,
|
||||
provenance: {
|
||||
fetchedAt: asOf,
|
||||
fetchedAt: newestChain ?? quote.observed_at,
|
||||
sourceKind: 'yfinance',
|
||||
rawSourceId: `options-surface:${providerId}:${symbol}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Same-print window: quote and book are one snapshot if within this. */
|
||||
export const SPOT_ALIGN_MS = 45 * 60_000;
|
||||
|
||||
/** Spot at a book timestamp: live quote if aligned, else last candle at or before `atIso`. */
|
||||
export function spotAt(
|
||||
db: DatabaseSync,
|
||||
symbol: string,
|
||||
atIso: string,
|
||||
quote: { price: number; observed_at: string },
|
||||
): { price: number; asOf: string; source: 'quote' | 'candle' } {
|
||||
const at = Date.parse(atIso);
|
||||
const qAt = Date.parse(quote.observed_at);
|
||||
if (Number.isFinite(at) && Number.isFinite(qAt) && Math.abs(qAt - at) <= SPOT_ALIGN_MS) {
|
||||
return { price: quote.price, asOf: quote.observed_at, source: 'quote' };
|
||||
}
|
||||
try {
|
||||
const row = db.prepare(
|
||||
`SELECT ts, c FROM price_candles
|
||||
WHERE symbol=? AND timeframe IN ('1m','5m','1d') AND ts<=?
|
||||
ORDER BY ts DESC LIMIT 1`,
|
||||
).get(symbol, atIso) as { ts: string; c: number } | undefined;
|
||||
if (row && Number.isFinite(row.c) && row.c > 0) {
|
||||
return { price: row.c, asOf: row.ts, source: 'candle' };
|
||||
}
|
||||
} catch {
|
||||
/* tests without price_candles fall through */
|
||||
}
|
||||
return { price: quote.price, asOf: quote.observed_at, source: 'quote' };
|
||||
}
|
||||
|
||||
function num(v: unknown): number | null {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : null;
|
||||
}
|
||||
|
||||
@@ -22,6 +22,10 @@ function memDb(): DatabaseSync {
|
||||
open_interest INTEGER, volume INTEGER, ts TEXT,
|
||||
PRIMARY KEY (symbol, expiry, strike, type, ts)
|
||||
);
|
||||
CREATE TABLE price_candles (
|
||||
symbol TEXT, timeframe TEXT, ts TEXT, o REAL, h REAL, l REAL, c REAL, v REAL,
|
||||
adj_close REAL, observed_at TEXT
|
||||
);
|
||||
`);
|
||||
return db;
|
||||
}
|
||||
@@ -78,6 +82,33 @@ test('assembleSurfaceFromCache builds rows from latest chain snapshot', () => {
|
||||
const call = surface!.rows.find((r) => r.right === 'call');
|
||||
assert.equal(call!.openInterest, 200); // latest snapshot
|
||||
assert.equal(call!.greeks?.gamma, 0.02);
|
||||
assert.equal(surface!.quoteAsOf, '2026-08-06T14:00:00.000Z');
|
||||
assert.equal(surface!.chainAsOf, '2026-08-06T14:00:00.000Z');
|
||||
});
|
||||
|
||||
test('assembleSurfaceFromCache uses candle spot when quote is from another session', () => {
|
||||
const db = memDb();
|
||||
db.prepare('INSERT INTO quotes (symbol,price,observed_at) VALUES (?,?,?)').run(
|
||||
'SPY', 510, '2026-08-16T17:00:00.000Z',
|
||||
);
|
||||
db.prepare('INSERT INTO kv_cache (key,value,observed_at) VALUES (?,?,?)').run(
|
||||
'options_expiry:SPY', JSON.stringify(['2026-08-21']), '2026-08-14T20:00:00.000Z',
|
||||
);
|
||||
db.prepare(
|
||||
`INSERT INTO options_chains (symbol,expiry,strike,type,bid,ask,iv,delta,gamma,theta,vega,open_interest,volume,ts)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
).run('SPY', '2026-08-21', 500, 'call', 1, 1.1, 0.2, 0.5, 0.01, -0.1, 0.2, 100, 10, '2026-08-14T20:00:00.000Z');
|
||||
db.prepare(
|
||||
`INSERT INTO price_candles (symbol,timeframe,ts,o,h,l,c,v,adj_close,observed_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?)`,
|
||||
).run('SPY', '1d', '2026-08-14T20:00:00.000Z', 500, 502, 498, 499.25, 1, 499.25, '2026-08-14T20:00:00.000Z');
|
||||
|
||||
const surface = assembleSurfaceFromCache(db, 'SPY', { maxExpiries: 2 });
|
||||
assert.ok(surface);
|
||||
assert.equal(surface!.asOf, '2026-08-14T20:00:00.000Z');
|
||||
assert.equal(surface!.spot, 499.25);
|
||||
assert.equal(surface!.chainAsOf, '2026-08-14T20:00:00.000Z');
|
||||
assert.equal(surface!.quoteAsOf, '2026-08-14T20:00:00.000Z');
|
||||
});
|
||||
|
||||
test('composeYFinanceWithOptions routes chain kinds to options adapter', async () => {
|
||||
|
||||
@@ -46,7 +46,12 @@ export interface NormalizedOptionRow {
|
||||
export interface NormalizedOptionSurface {
|
||||
symbol: string;
|
||||
spot: number;
|
||||
/** Evaluation clock (spot / time-to-expiry). May be newer than the option book. */
|
||||
asOf: string;
|
||||
/** When the spot print was observed. */
|
||||
quoteAsOf?: string | null;
|
||||
/** When the option chain snapshot used for OI/IV was written. */
|
||||
chainAsOf?: string | null;
|
||||
providerId: OptionsProviderId;
|
||||
/** Expected quote delay in seconds (900 delayed, 0 realtime, null unknown). */
|
||||
delaySeconds: number | null;
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
DEFAULT_SOURCE_MIN_INTERVAL_MS,
|
||||
DEMAND_SET_SOFT_CAP,
|
||||
DRAIN_KIND_BUDGET,
|
||||
fetchTimeoutMs,
|
||||
isPermanentDataError,
|
||||
isRateLimitError,
|
||||
jobBackoffMs,
|
||||
@@ -30,6 +31,8 @@ import {
|
||||
MAX_JOBS_PER_DRAIN,
|
||||
parseCooldownUntil,
|
||||
rateLimitCooldownMs,
|
||||
STALE_IN_FLIGHT_MS,
|
||||
YF_STALE_IN_FLIGHT_MS,
|
||||
SCHEDULE_INTERVALS,
|
||||
sourceCooldownHitsKey,
|
||||
sourceCooldownStateKey,
|
||||
@@ -38,8 +41,11 @@ import {
|
||||
} from './sourceRatePolicy.ts';
|
||||
import {
|
||||
assertSourcesBound,
|
||||
clearVendorRateLimit,
|
||||
familyDrainBudget,
|
||||
getVendorCooldownState,
|
||||
isVendorCoolingDown,
|
||||
listRegisteredFamilies,
|
||||
noteVendorRateLimit,
|
||||
sourceToFamily,
|
||||
sourcesForFamily,
|
||||
@@ -118,6 +124,10 @@ export class AdapterQueue implements CacheScheduler {
|
||||
private _lastError: string | null = null;
|
||||
/** Per-source promise chains: serializes fetches of the same source across parallel drains. */
|
||||
private readonly _sourceChains = new Map<SourceKind, Promise<unknown>>();
|
||||
/** Header / page-view quotes jump ahead of the rest of the Yahoo pile. */
|
||||
private readonly _priorityKeys = new Set<string>();
|
||||
/** Overlapping setInterval drains must not select a second batch mid-fetch. */
|
||||
private _drainBusy = false;
|
||||
|
||||
constructor(opts: AdapterQueueOptions) {
|
||||
this._db = opts.db;
|
||||
@@ -308,6 +318,13 @@ export class AdapterQueue implements CacheScheduler {
|
||||
/** Operator / test: force clear cool-down for a source. */
|
||||
clearSourceCooldownManual(source: SourceKind | string): void {
|
||||
this.clearSourceCooldown(source);
|
||||
const family = sourceToFamily(String(source));
|
||||
if (family) {
|
||||
clearVendorRateLimit(family);
|
||||
for (const src of sourcesForFamily(family)) {
|
||||
this.clearSourceCooldown(src);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
isSymbolQuarantined(symbol: string, now = Date.now()): boolean {
|
||||
@@ -419,15 +436,36 @@ export class AdapterQueue implements CacheScheduler {
|
||||
this._db.prepare('INSERT OR REPLACE INTO adapter_queue (key,status,last_attempt,retry_count,backoff_until,error,scheduled_for) VALUES (?,?,?,?,?,?,?)').run(key, 'pending', null, 0, null, null, scheduledFor ?? null);
|
||||
}
|
||||
|
||||
/** Mark a job as next on this source (header symbol / page view). */
|
||||
async prioritize(key: CacheKey): Promise<void> {
|
||||
await this.queue(key);
|
||||
this._priorityKeys.add(key);
|
||||
}
|
||||
|
||||
async drain(): Promise<void> {
|
||||
if (!this._cache) return;
|
||||
if (this.isPaused()) return;
|
||||
if (this._drainBusy) return;
|
||||
this._drainBusy = true;
|
||||
try {
|
||||
await this.drainOnce();
|
||||
} finally {
|
||||
this._drainBusy = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async drainOnce(): Promise<void> {
|
||||
const now = Date.now();
|
||||
|
||||
// Recover hung workers: jobs left in_flight after crash/hang never complete.
|
||||
const stuckCutoff = new Date(now - 5 * 60_000).toISOString();
|
||||
// Yahoo unsticks after 45s (fetch timeout 30s). SEC 13F jobs may run 10 min.
|
||||
const yfStuckCutoff = new Date(now - YF_STALE_IN_FLIGHT_MS).toISOString();
|
||||
this._db.prepare(
|
||||
"UPDATE adapter_queue SET status='pending', error='recovered stale in_flight', retry_count=0 WHERE status='in_flight' AND (last_attempt IS NULL OR last_attempt < ?)",
|
||||
"UPDATE adapter_queue SET status='pending', error='recovered stale in_flight', retry_count=0 WHERE status='in_flight' AND key LIKE 'yfinance:%' AND (last_attempt IS NULL OR last_attempt < ?)",
|
||||
).run(yfStuckCutoff);
|
||||
const stuckCutoff = new Date(now - STALE_IN_FLIGHT_MS).toISOString();
|
||||
this._db.prepare(
|
||||
"UPDATE adapter_queue SET status='pending', error='recovered stale in_flight', retry_count=0 WHERE status='in_flight' AND key NOT LIKE 'yfinance:%' AND (last_attempt IS NULL OR last_attempt < ?)",
|
||||
).run(stuckCutoff);
|
||||
|
||||
// Prefer live marks (quote/candles) over secondary data so watchlist prices land first.
|
||||
@@ -445,24 +483,35 @@ export class AdapterQueue implements CacheScheduler {
|
||||
const nextColon = rest.indexOf(':');
|
||||
return (nextColon > 0 ? rest.slice(0, nextColon) : rest).toUpperCase();
|
||||
};
|
||||
/** Kind rank: quote=0, candles=1, symbol=2, topHoldings=3, adjustments=4, else=5 */
|
||||
const isMinuteCandle = (key: string): boolean =>
|
||||
/^yfinance:candles:[^:]+:(1m|5m)$/.test(key);
|
||||
const isOptionsSurface = (key: string): boolean =>
|
||||
/^yfinance:(chain|expiry_dates):/.test(key);
|
||||
/** Kind rank: quote=0, live 1m/5m=1, options chain=2, X=3, 13F=4, daily candles=5, … */
|
||||
const kindRank = (key: string): number => {
|
||||
if (key.startsWith('yfinance:quote:')) return 0;
|
||||
if (key.startsWith('yfinance:candles:')) return 1;
|
||||
if (key.startsWith('yfinance:symbol:')) return 2;
|
||||
if (key.startsWith('yfinance:topHoldings:')) return 3;
|
||||
if (key.startsWith('yfinance:adjustments:')) return 4;
|
||||
return 5;
|
||||
if (isMinuteCandle(key)) return 1;
|
||||
if (isOptionsSurface(key)) return 2;
|
||||
if (key.startsWith('x:timeline:')) return 3;
|
||||
if (key.startsWith('sec-fetch:fetch:')) return 4;
|
||||
if (key.startsWith('yfinance:candles:')) return 5;
|
||||
if (key.startsWith('yfinance:symbol:')) return 6;
|
||||
if (key.startsWith('yfinance:topHoldings:')) return 7;
|
||||
if (key.startsWith('yfinance:adjustments:')) return 8;
|
||||
return 9;
|
||||
};
|
||||
const jobs = this._db.prepare(
|
||||
`SELECT key, retry_count, backoff_until, scheduled_for, last_attempt FROM adapter_queue
|
||||
WHERE status IN ('pending','backoff')`,
|
||||
).all() as Array<{ key: string; retry_count: number; backoff_until?: string | null; scheduled_for?: string | null; last_attempt?: string | null }>;
|
||||
|
||||
// Tier-aware sort: portfolio symbols (T0) first, then T1, T2, T3.
|
||||
// Tier-aware sort: jumped header quotes first, then portfolio (T0), T1, T2, T3.
|
||||
// Within each tier: quote > candles > symbol > topHoldings > adjustments > other.
|
||||
// Within each kind: never-attempted first, then oldest first.
|
||||
jobs.sort((a, b) => {
|
||||
const pa = this._priorityKeys.has(a.key) ? 0 : 1;
|
||||
const pb = this._priorityKeys.has(b.key) ? 0 : 1;
|
||||
if (pa !== pb) return pa - pb;
|
||||
const ta = tierMap.get(symFromKey(a.key)) ?? 99;
|
||||
const tb = tierMap.get(symFromKey(b.key)) ?? 99;
|
||||
if (ta !== tb) return ta - tb;
|
||||
@@ -484,10 +533,37 @@ export class AdapterQueue implements CacheScheduler {
|
||||
const pendingQuotes = this._db.prepare(
|
||||
"SELECT COUNT(*) AS c FROM adapter_queue WHERE status='pending' AND key LIKE 'yfinance:quote:%'",
|
||||
).get() as { c: number };
|
||||
const demandSize = (this._db.prepare(
|
||||
'SELECT COUNT(*) AS c FROM symbol_demand WHERE in_demand=1 OR COALESCE(system_pin,0)=1',
|
||||
).get() as { c: number }).c;
|
||||
const backlogPressure = pendingQuotes.c > 25;
|
||||
const hotQuotePending = (this._db.prepare(
|
||||
`SELECT COUNT(*) AS c FROM adapter_queue q
|
||||
WHERE q.status='pending' AND q.key LIKE 'yfinance:quote:%'
|
||||
AND COALESCE((SELECT tier FROM symbol_demand d WHERE d.symbol = substr(q.key, 16)), 99) < 3`,
|
||||
).get() as { c: number }).c > 0;
|
||||
|
||||
// Soft cap is real under pressure: drop T3 Yahoo work so it cannot starve holdings.
|
||||
if (backlogPressure || demandSize > DEMAND_SET_SOFT_CAP) {
|
||||
const droppable = jobs.filter((j) => {
|
||||
if (!j.key.startsWith('yfinance:')) return false;
|
||||
if (this._priorityKeys.has(j.key)) return false;
|
||||
const sym = symFromKey(j.key);
|
||||
return (tierMap.get(sym) ?? 99) >= 3;
|
||||
});
|
||||
if (droppable.length > 0) {
|
||||
const dropped = new Set(droppable.map((j) => j.key));
|
||||
const del = this._db.prepare("DELETE FROM adapter_queue WHERE key=? AND status IN ('pending','backoff')");
|
||||
for (const j of droppable) del.run(j.key);
|
||||
for (let i = jobs.length - 1; i >= 0; i--) {
|
||||
if (dropped.has(jobs[i].key)) jobs.splice(i, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const NON_CRITICAL_YF_KINDS = new Set([
|
||||
'chain', 'shortinterest', 'dividendFundamentals',
|
||||
'topHoldings', 'expiry_dates', 'symbol', 'adjustments',
|
||||
'shortinterest', 'dividendFundamentals',
|
||||
'topHoldings', 'symbol', 'adjustments',
|
||||
]);
|
||||
|
||||
let processed = 0;
|
||||
@@ -545,12 +621,19 @@ export class AdapterQueue implements CacheScheduler {
|
||||
// below drops them because their "sym" is a fund handle, not a symbol.
|
||||
const isXTimeline = source === 'x' && kind === 'timeline';
|
||||
|
||||
const tier = tierMap.get(sym) ?? 99;
|
||||
const isHotQuote = source === 'yfinance' && kind === 'quote' && tier < 3;
|
||||
const isHotMinute = source === 'yfinance' && isMinuteCandle(job.key) && tier < 3;
|
||||
// While a holding / watched / focused quote is waiting, Yahoo only fetches
|
||||
// those live marks. Chains and T3 names cannot jump the line.
|
||||
if (hotQuotePending && source === 'yfinance' && !isHotQuote && !isHotMinute) continue;
|
||||
// Backlog pressure: defer non-critical yfinance kinds so quotes drain first.
|
||||
if (backlogPressure && source === 'yfinance' && NON_CRITICAL_YF_KINDS.has(kind)) continue;
|
||||
// Backlog pressure: skip T3 (background) symbols entirely so portfolio/alert symbols drain first.
|
||||
// x timeline jobs are exempt: their sym is a fund handle (never in tierMap), so
|
||||
// without this they'd always be dropped while the yfinance backlog persists.
|
||||
if (backlogPressure && sym && (tierMap.get(sym) ?? 99) >= 3 && !isXTimeline) continue;
|
||||
if ((hotQuotePending || backlogPressure) && source === 'yfinance' && tier >= 3) continue;
|
||||
if (backlogPressure && sym && tier >= 3 && !isXTimeline && source !== 'yfinance') continue;
|
||||
|
||||
// Source-wide cool-down: skip all jobs for this vendor until the window ends.
|
||||
if (this.isSourceCoolingDown(source, Date.now())) continue;
|
||||
@@ -562,7 +645,9 @@ export class AdapterQueue implements CacheScheduler {
|
||||
|
||||
if (family) {
|
||||
const famUsed = familyJobsThisDrain[family] ?? 0;
|
||||
if (famUsed >= familyDrainBudget(family)) {
|
||||
// Minute candles and options chains are live map/chart clicks: do not
|
||||
// park them behind the daily Yahoo quote backlog.
|
||||
if (famUsed >= familyDrainBudget(family) && !isMinuteCandle(job.key) && !isOptionsSurface(job.key)) {
|
||||
// Book for pass 2 instead of dropping: a fresh subscribe (6 yfinance
|
||||
// jobs) must be able to complete in one drain when there's room.
|
||||
overBudget.push({ job, source, kind, sym, family });
|
||||
@@ -571,6 +656,7 @@ export class AdapterQueue implements CacheScheduler {
|
||||
}
|
||||
|
||||
specs.push({ key: job.key, source, family, sym, attempt: job.retry_count + 1 });
|
||||
this._priorityKeys.delete(job.key);
|
||||
processed += 1;
|
||||
kindUsed[kind] = used + 1;
|
||||
if (family) familyJobsThisDrain[family] = (familyJobsThisDrain[family] ?? 0) + 1;
|
||||
@@ -588,11 +674,17 @@ export class AdapterQueue implements CacheScheduler {
|
||||
if (family && this.isVendorFamilyCoolingDown(family, Date.now())) continue;
|
||||
const kindBudget = DRAIN_KIND_BUDGET[kind] ?? DRAIN_KIND_BUDGET._default;
|
||||
if ((kindUsed[kind] ?? 0) >= kindBudget) continue;
|
||||
const tier = tierMap.get(sym) ?? 99;
|
||||
const isHotQuote = source === 'yfinance' && kind === 'quote' && tier < 3;
|
||||
const isHotMinute = source === 'yfinance' && isMinuteCandle(job.key) && tier < 3;
|
||||
if (hotQuotePending && source === 'yfinance' && !isHotQuote && !isHotMinute) continue;
|
||||
// Backlog pressure: defer non-critical yfinance kinds so quotes drain first.
|
||||
if (backlogPressure && source === 'yfinance' && NON_CRITICAL_YF_KINDS.has(kind)) continue;
|
||||
// Backlog pressure: skip T3 (background) symbols entirely so portfolio/alert symbols drain first.
|
||||
if (backlogPressure && sym && (tierMap.get(sym) ?? 99) >= 3) continue;
|
||||
if ((hotQuotePending || backlogPressure) && source === 'yfinance' && tier >= 3) continue;
|
||||
if (backlogPressure && sym && tier >= 3) continue;
|
||||
specs.push({ key: job.key, source, family, sym, attempt: job.retry_count + 1 });
|
||||
this._priorityKeys.delete(job.key);
|
||||
processed += 1;
|
||||
kindUsed[kind] = (kindUsed[kind] ?? 0) + 1;
|
||||
}
|
||||
@@ -658,7 +750,7 @@ export class AdapterQueue implements CacheScheduler {
|
||||
this._lastFetchAt[source] = Date.now();
|
||||
this._setStatus(key, 'in_flight');
|
||||
try {
|
||||
const FETCH_TIMEOUT_MS = 30_000;
|
||||
const FETCH_TIMEOUT_MS = fetchTimeoutMs(key);
|
||||
const fetchPromise = adapter.fetchOne(key);
|
||||
const timeoutPromise = new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`fetchOne timeout after ${FETCH_TIMEOUT_MS}ms for ${key}`)), FETCH_TIMEOUT_MS),
|
||||
@@ -1227,10 +1319,29 @@ this._db.prepare("UPDATE adapter_queue SET status='backoff', last_attempt=?, ret
|
||||
} catch { /* ignore */ }
|
||||
|
||||
const cools = this.listSourceCooldowns();
|
||||
const bySource = new Map(cools.map((c) => [c.source, c]));
|
||||
for (const family of listRegisteredFamilies()) {
|
||||
const gate = getVendorCooldownState(family);
|
||||
if (!gate.active) continue;
|
||||
for (const src of sourcesForFamily(family)) {
|
||||
const existing = bySource.get(src);
|
||||
if (!existing || gate.remainingMs > existing.remainingMs) {
|
||||
const snap: SourceCooldownSnapshot = {
|
||||
source: src,
|
||||
until: gate.until,
|
||||
remainingMs: gate.remainingMs,
|
||||
consecutiveHits: Math.max(existing?.consecutiveHits ?? 0, gate.consecutiveHits),
|
||||
active: true,
|
||||
};
|
||||
bySource.set(src, snap);
|
||||
}
|
||||
}
|
||||
}
|
||||
const mergedCools = [...bySource.values()].filter((c) => c.active);
|
||||
const controlled = this.listControlledSources();
|
||||
const notes: string[] = [];
|
||||
if (paused) notes.push('queue paused');
|
||||
for (const c of cools) {
|
||||
for (const c of mergedCools) {
|
||||
notes.push(`${c.source} cooling ${Math.ceil(c.remainingMs / 60_000)}m (hits=${c.consecutiveHits})`);
|
||||
}
|
||||
for (const c of controlled) {
|
||||
@@ -1241,8 +1352,10 @@ this._db.prepare("UPDATE adapter_queue SET status='backoff', last_attempt=?, ret
|
||||
if (candleLagMs != null && candleLagMs > 3 * 86_400_000) notes.push(`SPY candle lag ${Math.round(candleLagMs / 86_400_000)}d`);
|
||||
if (demandSize > DEMAND_SET_SOFT_CAP) notes.push(`demand set ${demandSize} > soft cap ${DEMAND_SET_SOFT_CAP}`);
|
||||
|
||||
const yfCool = cools.some((c) => c.source === 'yfinance' && c.active);
|
||||
const dataPlaneHealthy = !paused && !yfCool && (row.q ?? 0) < 150 && (candleLagMs == null || candleLagMs < 3 * 86_400_000);
|
||||
const yfCool = mergedCools.some((c) => c.source === 'yfinance' && c.active);
|
||||
const pendingQuotes = pendingByKind.quote ?? 0;
|
||||
if (pendingQuotes > 25) notes.push(`pending quotes=${pendingQuotes}`);
|
||||
const dataPlaneHealthy = !paused && !yfCool && (row.q ?? 0) < 150 && pendingQuotes < 25 && (candleLagMs == null || candleLagMs < 3 * 86_400_000);
|
||||
|
||||
return {
|
||||
queued: row.q ?? 0,
|
||||
@@ -1255,7 +1368,7 @@ this._db.prepare("UPDATE adapter_queue SET status='backoff', last_attempt=?, ret
|
||||
paused,
|
||||
counts: { pending: row.q ?? 0, in_flight: row.i ?? 0, failed: row.f ?? 0, backoff: row.b ?? 0, done: row.d ?? 0 },
|
||||
lastErrors: failedJobs,
|
||||
sourceCooldowns: cools,
|
||||
sourceCooldowns: mergedCools,
|
||||
pausedSources: controlled.filter((c) => c.paused).map((c) => c.source),
|
||||
stoppedSources: controlled.filter((c) => c.stopped).map((c) => c.source),
|
||||
pendingByKind,
|
||||
|
||||
@@ -80,6 +80,62 @@ test('after backoff expires, retries; eventually FAILED at MAX_ATTEMPTS', async
|
||||
assert.fail('did not reach failed within 10 drain cycles');
|
||||
});
|
||||
|
||||
test('drain prefers 5m candles over a pile of daily candle jobs', async () => {
|
||||
const { db, fake, queue } = setup();
|
||||
const bar = { ts: '2026-08-15T14:30:00.000Z', o: 1, h: 2, l: 1, c: 1.5, v: 10, adjClose: 1.5 };
|
||||
fake.set('yfinance:candles:NVDA:5m', [bar], 'daily_permanent');
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const sym = `S${i}`;
|
||||
fake.set(`yfinance:candles:${sym}:1d`, [bar], 'daily_permanent');
|
||||
await queue.queue(`yfinance:candles:${sym}:1d`);
|
||||
}
|
||||
await queue.queue('yfinance:candles:NVDA:5m');
|
||||
await queue.drain();
|
||||
assert.equal(statusOf(db, 'yfinance:candles:NVDA:5m').status, 'done');
|
||||
assert.ok(fake.calls.includes('yfinance:candles:NVDA:5m'));
|
||||
});
|
||||
|
||||
test('drain prefers x timeline over a pile of daily candle jobs', async () => {
|
||||
resetVendorGateForTests();
|
||||
const db = createDb({ path: ':memory:' });
|
||||
initSchema(db);
|
||||
const bar = { ts: '2026-08-15T14:30:00.000Z', o: 1, h: 2, l: 1, c: 1.5, v: 10, adjClose: 1.5 };
|
||||
const yf = new FakeSourceAdapter('yfinance');
|
||||
const x = new FakeSourceAdapter('x');
|
||||
x.set('x:timeline:TheProfInvestor', [], 'thread_7d');
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const sym = `S${i}`;
|
||||
yf.set(`yfinance:candles:${sym}:1d`, [bar], 'daily_permanent');
|
||||
}
|
||||
const adapters = new Map([
|
||||
['yfinance' as const, yf as unknown as import('../../adapters/SourceAdapter.ts').SourceFetch],
|
||||
['x' as const, x as unknown as import('../../adapters/SourceAdapter.ts').SourceFetch],
|
||||
]);
|
||||
const queue = new AdapterQueue({ db, adapters, rateLimitMs: { yfinance: 0, x: 0 } });
|
||||
const cache = createCacheRepository({ db, scheduler: queue });
|
||||
queue.cache = cache;
|
||||
for (let i = 0; i < 30; i++) await queue.queue(`yfinance:candles:S${i}:1d`);
|
||||
await queue.queue('x:timeline:TheProfInvestor');
|
||||
await queue.drain();
|
||||
assert.equal(statusOf(db, 'x:timeline:TheProfInvestor').status, 'done');
|
||||
assert.ok(x.calls.includes('x:timeline:TheProfInvestor'));
|
||||
});
|
||||
|
||||
test('drain prefers options chain over a pile of daily candle jobs', async () => {
|
||||
const { db, fake, queue } = setup();
|
||||
const bar = { ts: '2026-08-15T14:30:00.000Z', o: 1, h: 2, l: 1, c: 1.5, v: 10, adjClose: 1.5 };
|
||||
fake.set('yfinance:chain:IREN:2026-08-21', { rows: [] }, 'options_snapshot');
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const sym = `S${i}`;
|
||||
fake.set(`yfinance:candles:${sym}:1d`, [bar], 'daily_permanent');
|
||||
await queue.queue(`yfinance:candles:${sym}:1d`);
|
||||
}
|
||||
await queue.queue('yfinance:chain:IREN:2026-08-21');
|
||||
await queue.drain();
|
||||
assert.equal(statusOf(db, 'yfinance:chain:IREN:2026-08-21').status, 'done');
|
||||
assert.ok(fake.calls.includes('yfinance:chain:IREN:2026-08-21'));
|
||||
});
|
||||
|
||||
test('health() reports queued/in_flight/backoff counts', async () => {
|
||||
const { db, queue } = setup();
|
||||
await queue.queue('yfinance:quote:NVDA');
|
||||
@@ -309,6 +365,85 @@ test('stopped source schedules skip enqueue; startSource forces next_enqueue to
|
||||
assert.ok(Date.parse(row.next_enqueue) <= Date.now() + 5000);
|
||||
});
|
||||
|
||||
function seedDemand(
|
||||
db: import('node:sqlite').DatabaseSync,
|
||||
symbol: string,
|
||||
tier: number,
|
||||
): void {
|
||||
db.prepare(
|
||||
'INSERT OR REPLACE INTO symbol_demand (symbol,refcount,ticker_kind,in_demand,last_refreshed_at,system_pin,tier) VALUES (?,?,?,?,?,?,?)',
|
||||
).run(symbol, 1, 'equity', 1, null, 0, tier);
|
||||
}
|
||||
|
||||
test('drain fetches T0 quote before any T3 Yahoo job', async () => {
|
||||
const { db, fake, queue } = setup();
|
||||
fake.set('yfinance:quote:IREN', { symbol: 'IREN', price: 45 } as Quote, 'live_quote');
|
||||
fake.set('yfinance:quote:INTC', { symbol: 'INTC', price: 100 } as Quote, 'live_quote');
|
||||
seedDemand(db, 'IREN', 0);
|
||||
seedDemand(db, 'INTC', 3);
|
||||
await queue.queue('yfinance:quote:INTC');
|
||||
await queue.queue('yfinance:quote:IREN');
|
||||
await queue.drain();
|
||||
assert.equal(statusOf(db, 'yfinance:quote:IREN').status, 'done');
|
||||
assert.equal(statusOf(db, 'yfinance:quote:INTC').status, 'pending', 'T3 must wait while a hot quote is pending');
|
||||
assert.deepEqual(fake.calls, ['yfinance:quote:IREN']);
|
||||
await queue.drain();
|
||||
assert.equal(statusOf(db, 'yfinance:quote:INTC').status, 'done');
|
||||
});
|
||||
|
||||
test('prioritizeQuote jumps ahead of other pending hot quotes', async () => {
|
||||
const { db, fake, queue } = setup();
|
||||
fake.set('yfinance:quote:IREN', { symbol: 'IREN', price: 45 } as Quote, 'live_quote');
|
||||
fake.set('yfinance:quote:NVDA', { symbol: 'NVDA', price: 194 } as Quote, 'live_quote');
|
||||
seedDemand(db, 'IREN', 2);
|
||||
seedDemand(db, 'NVDA', 2);
|
||||
await queue.queue('yfinance:quote:NVDA');
|
||||
await queue.prioritize('yfinance:quote:IREN');
|
||||
await queue.drain();
|
||||
assert.equal(fake.calls[0], 'yfinance:quote:IREN');
|
||||
});
|
||||
|
||||
test('overlapping drain() does not double-select the same Yahoo job', async () => {
|
||||
const { db, queue } = setup();
|
||||
let inflight = 0;
|
||||
let maxInflight = 0;
|
||||
const slow = {
|
||||
sourceKind: 'yfinance' as const,
|
||||
async fetchOne(key: string) {
|
||||
inflight += 1;
|
||||
maxInflight = Math.max(maxInflight, inflight);
|
||||
await new Promise((r) => setTimeout(r, 40));
|
||||
inflight -= 1;
|
||||
return {
|
||||
value: { symbol: 'NVDA', price: 1 },
|
||||
ttlClass: 'live_quote' as const,
|
||||
provenance: { fetchedAt: new Date().toISOString(), sourceKind: 'yfinance' as const },
|
||||
};
|
||||
},
|
||||
};
|
||||
const { AdapterQueue: AQ } = await import('../AdapterQueue.ts');
|
||||
const q2 = new AQ({
|
||||
db,
|
||||
adapters: new Map([['yfinance' as const, slow as import('../../adapters/SourceAdapter.ts').SourceFetch]]),
|
||||
rateLimitMs: { yfinance: 0 },
|
||||
});
|
||||
q2.cache = (await import('../../cache/CacheRepository.ts')).createCacheRepository({ db, scheduler: q2 });
|
||||
await q2.queue('yfinance:quote:NVDA');
|
||||
await Promise.all([q2.drain(), q2.drain()]);
|
||||
assert.equal(maxInflight, 1);
|
||||
assert.equal(statusOf(db, 'yfinance:quote:NVDA').status, 'done');
|
||||
});
|
||||
|
||||
test('health() merges vendorGate into sourceCooldowns when queue_state is empty', async () => {
|
||||
const { queue } = setup();
|
||||
const { noteVendorRateLimit } = await import('../../services/vendorGate.ts');
|
||||
noteVendorRateLimit('yfinance');
|
||||
const h = queue.health();
|
||||
assert.ok(h.sourceCooldowns.some((c) => c.source === 'yfinance' && c.active));
|
||||
assert.ok(h.dataPlaneNotes.some((n) => /yfinance/.test(n) && /cool/.test(n)));
|
||||
assert.equal(h.dataPlaneHealthy, false);
|
||||
});
|
||||
|
||||
test('health() reports pausedSources and stoppedSources', async () => {
|
||||
const { queue } = setup();
|
||||
queue.pauseSource('fred');
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import {
|
||||
DEFAULT_FETCH_TIMEOUT_MS,
|
||||
fetchTimeoutMs,
|
||||
isPermanentDataError,
|
||||
isRateLimitError,
|
||||
isUsMarketHours,
|
||||
@@ -9,6 +11,8 @@ import {
|
||||
quoteTtlMs,
|
||||
rateLimitCooldownMs,
|
||||
RATE_LIMIT_COOLDOWN_MS,
|
||||
SEC_FETCH_TIMEOUT_MS,
|
||||
STALE_IN_FLIGHT_MS,
|
||||
} from '../sourceRatePolicy.ts';
|
||||
|
||||
test('isRateLimitError detects Yahoo Edge / 429 / bird HTTP 429', () => {
|
||||
@@ -78,6 +82,15 @@ test('jobBackoffMs stays in short window for ordinary errors', () => {
|
||||
assert.ok(jobBackoffMs(1) < rateLimitCooldownMs(1));
|
||||
});
|
||||
|
||||
test('fetchTimeoutMs gives SEC institutional jobs minutes, not 30s', () => {
|
||||
assert.equal(fetchTimeoutMs('yfinance:quote:AAPL'), DEFAULT_FETCH_TIMEOUT_MS);
|
||||
assert.equal(fetchTimeoutMs('sec-fetch:fetch:IREN'), SEC_FETCH_TIMEOUT_MS);
|
||||
assert.equal(fetchTimeoutMs('sec-sc-fetch:sc:CIFR'), SEC_FETCH_TIMEOUT_MS);
|
||||
assert.equal(fetchTimeoutMs('sec-lint-holders:holders:OPEN'), SEC_FETCH_TIMEOUT_MS);
|
||||
assert.ok(SEC_FETCH_TIMEOUT_MS >= 10 * 60_000);
|
||||
assert.ok(STALE_IN_FLIGHT_MS > SEC_FETCH_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
test('parseCooldownUntil reports active remaining window', () => {
|
||||
const now = Date.UTC(2026, 0, 1, 12, 0, 0);
|
||||
const until = new Date(now + 30_000).toISOString();
|
||||
|
||||
@@ -63,7 +63,7 @@ export const DRAIN_KIND_BUDGET: Record<string, number> = {
|
||||
topHoldings: 1,
|
||||
expiry_dates: 2,
|
||||
chain: 2,
|
||||
fetch: 1,
|
||||
fetch: 2,
|
||||
holders: 1,
|
||||
insiders: 1,
|
||||
timeline: 1,
|
||||
@@ -73,6 +73,33 @@ export const DRAIN_KIND_BUDGET: Record<string, number> = {
|
||||
/** Max jobs processed in one drain() call across all sources. */
|
||||
export const MAX_JOBS_PER_DRAIN = 25;
|
||||
|
||||
/** Default fetchOne timeout (quotes, meta, light jobs). */
|
||||
export const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
||||
|
||||
/**
|
||||
* 13F holder refresh downloads dozens of EDGAR XMLs plus a reverse-13F
|
||||
* supplement. 30s kills the job mid-run (then the work keeps hitting SEC).
|
||||
*/
|
||||
export const SEC_FETCH_TIMEOUT_MS = 10 * 60_000;
|
||||
|
||||
/**
|
||||
* Drain recovers hung in_flight rows. Must exceed the longest fetch timeout
|
||||
* or a live 13F job is reset and started a second time.
|
||||
*/
|
||||
export const STALE_IN_FLIGHT_MS = 12 * 60_000;
|
||||
|
||||
/** Yahoo fetch timeout is 30s; do not wait the SEC-sized 12m window to unstick a hung quote. */
|
||||
export const YF_STALE_IN_FLIGHT_MS = 45_000;
|
||||
|
||||
/** Per-job fetch timeout. SEC institutional jobs need minutes, not seconds. */
|
||||
export function fetchTimeoutMs(key: string): number {
|
||||
const source = key.split(':')[0] ?? '';
|
||||
if (source === 'sec-fetch' || source === 'sec-sc-fetch' || source.startsWith('sec-lint')) {
|
||||
return SEC_FETCH_TIMEOUT_MS;
|
||||
}
|
||||
return DEFAULT_FETCH_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
/** Soft cap on demand-set size; schedule skips excess beyond system pins. */
|
||||
export const DEMAND_SET_SOFT_CAP = 80;
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import {
|
||||
annotateCaptureEvidence,
|
||||
evidencePostId,
|
||||
isTweetGoneMessage,
|
||||
} from '../captureEvidence.ts';
|
||||
|
||||
test('evidencePostId extracts status id from x.com URL', () => {
|
||||
assert.equal(
|
||||
evidencePostId('https://x.com/mikealfred/status/2082901502113575384'),
|
||||
'2082901502113575384',
|
||||
);
|
||||
assert.equal(evidencePostId('sec|0002096493-26-000003'), null);
|
||||
});
|
||||
|
||||
test('isTweetGoneMessage matches bird / X not-found errors', () => {
|
||||
assert.equal(isTweetGoneMessage('Failed to read tweet: Tweet not found in response'), true);
|
||||
assert.equal(isTweetGoneMessage('HTTP 429 rate limit'), false);
|
||||
});
|
||||
|
||||
test('annotateCaptureEvidence uses cache and only looks up misses', async () => {
|
||||
const db = new DatabaseSync(':memory:');
|
||||
db.exec(`CREATE TABLE kv_cache (key TEXT PRIMARY KEY, value TEXT NOT NULL, observed_at TEXT NOT NULL)`);
|
||||
const now = Date.parse('2026-08-14T20:00:00.000Z');
|
||||
db.prepare('INSERT INTO kv_cache (key,value,observed_at) VALUES (?,?,?)').run(
|
||||
'x:post-status:111', 'ok', new Date(now).toISOString(),
|
||||
);
|
||||
const looked: string[] = [];
|
||||
const rows = [
|
||||
{ evidence_url: 'https://x.com/mikealfred/status/111' },
|
||||
{ evidence_url: 'https://x.com/mikealfred/status/2082901502113575384' },
|
||||
];
|
||||
await annotateCaptureEvidence(db, rows, async (id) => {
|
||||
looked.push(id);
|
||||
return 'deleted';
|
||||
}, now);
|
||||
assert.deepEqual(looked, ['2082901502113575384']);
|
||||
assert.equal(rows[0].evidence_status, 'ok');
|
||||
assert.equal(rows[1].evidence_status, 'deleted');
|
||||
});
|
||||
|
||||
test('annotateCaptureEvidence live:false never calls lookup', async () => {
|
||||
const db = new DatabaseSync(':memory:');
|
||||
db.exec(`CREATE TABLE kv_cache (key TEXT PRIMARY KEY, value TEXT NOT NULL, observed_at TEXT NOT NULL)`);
|
||||
const looked: string[] = [];
|
||||
const rows = [{ evidence_url: 'https://x.com/mikealfred/status/2082901502113575384' }];
|
||||
await annotateCaptureEvidence(db, rows, async (id) => {
|
||||
looked.push(id);
|
||||
return 'ok';
|
||||
}, Date.now(), { live: false });
|
||||
assert.deepEqual(looked, []);
|
||||
assert.equal(rows[0].evidence_status, 'unknown');
|
||||
});
|
||||
@@ -19,7 +19,8 @@ function makeDb(): DatabaseSync {
|
||||
);
|
||||
CREATE TABLE x_cookie_posts (
|
||||
post_id TEXT PRIMARY KEY, source TEXT, author_handle TEXT, cashtag TEXT,
|
||||
body_text TEXT, posted_at TEXT, engagement TEXT, cached_until TEXT
|
||||
body_text TEXT, posted_at TEXT, engagement TEXT, cached_until TEXT,
|
||||
quoted_post_id TEXT
|
||||
);
|
||||
`);
|
||||
db.prepare(`INSERT INTO tracked_funds (id, ci_key, fund_name, manager_name, x_handle, paywall_status, enabled, created_at)
|
||||
@@ -28,9 +29,9 @@ function makeDb(): DatabaseSync {
|
||||
return db;
|
||||
}
|
||||
|
||||
function seedPost(db: DatabaseSync, id: string, body: string, at: string): void {
|
||||
db.prepare(`INSERT INTO x_cookie_posts (post_id, source, author_handle, cashtag, body_text, posted_at)
|
||||
VALUES (?, 'x', 'mikealfred', NULL, ?, ?)`).run(id, body, at);
|
||||
function seedPost(db: DatabaseSync, id: string, body: string, at: string, quotedPostId?: string): void {
|
||||
db.prepare(`INSERT INTO x_cookie_posts (post_id, source, author_handle, cashtag, body_text, posted_at, quoted_post_id)
|
||||
VALUES (?, 'x', 'mikealfred', NULL, ?, ?, ?)`).run(id, body, at, quotedPostId ?? null);
|
||||
}
|
||||
|
||||
test('ingestFundCaptures: materializes a total-position capture with source=capture', () => {
|
||||
@@ -63,15 +64,17 @@ test('ingestFundCaptures: idempotent — same post converges, no duplicate rows'
|
||||
assert.equal(db.prepare('SELECT COUNT(*) FROM fund_position_records').get()!['COUNT(*)'], 1);
|
||||
});
|
||||
|
||||
test('ingestFundCaptures: commentary posts and claims are never materialized', () => {
|
||||
test('ingestFundCaptures: commentary is skipped; a clean add folds into the book', () => {
|
||||
const db = makeDb();
|
||||
seedPost(db, 'c1', 'Beautiful day in the mountains. Great conversations with subscribers.', '2026-08-04T23:07:20.000Z');
|
||||
seedPost(db, 'cl1', 'Added 865,000 shares of BKKT at $3.10 average price today.', '2026-08-01T10:00:00.000Z');
|
||||
|
||||
const s = ingestFundCaptures(db, 'alpine-fox-capital');
|
||||
assert.equal(s.skipped, 1); // commentary
|
||||
assert.equal(s.claims, 1); // claim detected but skipped in v1
|
||||
assert.equal(db.prepare('SELECT COUNT(*) FROM fund_position_records').get()!['COUNT(*)'], 0);
|
||||
assert.equal(s.skipped, 1);
|
||||
assert.equal(s.inserted, 1);
|
||||
const bkkt = db.prepare(`SELECT shares, cost_basis FROM fund_position_records WHERE symbol='BKKT'`).get() as any;
|
||||
assert.equal(bkkt.shares, 865000);
|
||||
assert.ok(Math.abs(bkkt.cost_basis - 3.1) < 0.001);
|
||||
});
|
||||
|
||||
test('ingestFundCaptures: trade narrative with two quantities is refused (no garbage)', () => {
|
||||
@@ -117,6 +120,102 @@ OPEN now over 6.45M shares. Have taken it up on the weakness.`,
|
||||
);
|
||||
});
|
||||
|
||||
test('ingestFundCaptures: first-disclosure add folds into an opening ENHA row', () => {
|
||||
const db = makeDb();
|
||||
seedPost(
|
||||
db,
|
||||
'2088348177665798249',
|
||||
"Seeing some very strange trading in ENHA. So without even thinking about it, I added 115,000 shares at $1.79. I still believe in my friend @C_Angermayer's vision. Still too early to write this one off.",
|
||||
'2026-08-14T19:32:59.000Z',
|
||||
);
|
||||
const s = ingestFundCaptures(db, 'alpine-fox-capital');
|
||||
assert.equal(s.inserted, 1);
|
||||
assert.equal(s.captures, 1);
|
||||
const row = db.prepare(`SELECT * FROM fund_position_records WHERE symbol='ENHA'`).get() as any;
|
||||
assert.equal(row.shares, 115000);
|
||||
assert.ok(Math.abs(row.cost_basis - 1.79) < 0.001);
|
||||
assert.equal(row.as_of, '2026-08-14');
|
||||
assert.equal(row.source, 'capture');
|
||||
});
|
||||
|
||||
test('ingestFundCaptures: same add tweet is idempotent', () => {
|
||||
const db = makeDb();
|
||||
seedPost(
|
||||
db,
|
||||
'2088348177665798249',
|
||||
"Seeing some very strange trading in ENHA. So without even thinking about it, I added 115,000 shares at $1.79.",
|
||||
'2026-08-14T19:32:59.000Z',
|
||||
);
|
||||
ingestFundCaptures(db, 'alpine-fox-capital');
|
||||
ingestFundCaptures(db, 'alpine-fox-capital');
|
||||
const n = db.prepare(`SELECT COUNT(*) AS n FROM fund_position_records WHERE symbol='ENHA'`).get() as { n: number };
|
||||
const row = db.prepare(`SELECT shares FROM fund_position_records WHERE symbol='ENHA'`).get() as { shares: number };
|
||||
assert.equal(n.n, 1);
|
||||
assert.equal(row.shares, 115000);
|
||||
});
|
||||
|
||||
test('ingestFundCaptures: second add folds onto the prior total', () => {
|
||||
const db = makeDb();
|
||||
seedPost(db, 'a1', 'Added 100,000 shares of ENHA at $1.80.', '2026-08-10T12:00:00.000Z');
|
||||
seedPost(db, 'a2', 'Added 15,000 shares of ENHA at $1.79.', '2026-08-14T19:32:59.000Z');
|
||||
ingestFundCaptures(db, 'alpine-fox-capital');
|
||||
const latest = db.prepare(
|
||||
`SELECT shares, as_of FROM fund_position_records WHERE symbol='ENHA' ORDER BY as_of DESC LIMIT 1`,
|
||||
).get() as any;
|
||||
assert.equal(latest.shares, 115000);
|
||||
assert.equal(latest.as_of, '2026-08-14');
|
||||
});
|
||||
|
||||
test('ingestFundCaptures: quote-tweet add inherits ticker from the quoted capture', () => {
|
||||
const db = makeDb();
|
||||
seedPost(
|
||||
db,
|
||||
'2088348177665798249',
|
||||
"Seeing some very strange trading in ENHA. So without even thinking about it, I added 115,000 shares at $1.79.",
|
||||
'2026-08-14T19:32:59.000Z',
|
||||
);
|
||||
seedPost(
|
||||
db,
|
||||
'2089409857296453731',
|
||||
"Quick update on this one. I added another 123,000 shares to this today at $1.62 net. Average now $1.69.",
|
||||
'2026-08-17T17:51:43.000Z',
|
||||
'2088348177665798249',
|
||||
);
|
||||
ingestFundCaptures(db, 'alpine-fox-capital');
|
||||
const latest = db.prepare(
|
||||
`SELECT shares, cost_basis, as_of FROM fund_position_records WHERE symbol='ENHA' ORDER BY as_of DESC LIMIT 1`,
|
||||
).get() as any;
|
||||
assert.equal(latest.shares, 238000);
|
||||
assert.ok(Math.abs(latest.cost_basis - 1.69) < 0.001);
|
||||
assert.equal(latest.as_of, '2026-08-17');
|
||||
});
|
||||
|
||||
test('ingestFundCaptures: sold out of $PEP writes a zero-share capture', () => {
|
||||
const db = makeDb();
|
||||
seedPost(db, 'e1', 'Sold out of $PEP this morning. Moving on.', '2026-08-02T14:00:00.000Z');
|
||||
const s = ingestFundCaptures(db, 'alpine-fox-capital');
|
||||
assert.equal(s.captures, 1);
|
||||
const row = db.prepare(`SELECT * FROM fund_position_records WHERE symbol='PEP'`).get() as any;
|
||||
assert.equal(row.shares, 0);
|
||||
assert.equal(row.source, 'capture');
|
||||
assert.equal(row.as_of, '2026-08-02');
|
||||
});
|
||||
|
||||
test('ingestFundCaptures: book-reset list exits names that were held and not named', () => {
|
||||
const db = makeDb();
|
||||
db.prepare(
|
||||
`INSERT INTO fund_position_records (id, fund_id, symbol, shares, value_usd, cost_basis, as_of, source, evidence_url, notes, created_at)
|
||||
VALUES ('r1','alpine-fox-capital','CIFR',100,1,null,'2026-06-30','13f',null,null,'2026-06-30'),
|
||||
('r2','alpine-fox-capital','DEO',50,1,null,'2026-06-30','13f',null,null,'2026-06-30')`,
|
||||
).run();
|
||||
seedPost(db, 'rst', 'Cleaning the book. What stayed: $CIFR $IREN', '2026-07-20T12:00:00.000Z');
|
||||
ingestFundCaptures(db, 'alpine-fox-capital');
|
||||
const deo = db.prepare(`SELECT * FROM fund_position_records WHERE symbol='DEO' ORDER BY as_of DESC`).all() as any[];
|
||||
assert.ok(deo.some((r) => r.shares === 0 && r.source === 'capture'));
|
||||
const cifrZero = db.prepare(`SELECT COUNT(*) AS n FROM fund_position_records WHERE symbol='CIFR' AND shares=0`).get() as { n: number };
|
||||
assert.equal(cifrZero.n, 0, 'named stayed ticker is not exited');
|
||||
});
|
||||
|
||||
test('ingestAllFundCaptures: runs across every enabled tracked fund with a handle', () => {
|
||||
const db = makeDb();
|
||||
seedPost(db, 'p1', '**Real time position update\n\nTook OPEN over 5.7M shares now. Brought average down to $4.42.', '2026-07-30T19:51:12.000Z');
|
||||
|
||||
@@ -30,6 +30,10 @@ test('kv_cache can store and read sec:cusip keys', () => {
|
||||
test('curated map resolves IREN offline', () => {
|
||||
assert.equal(curatedCusipForSymbol('IREN'), 'Q4982L109');
|
||||
assert.equal(curatedCusipForSymbol('AAPL'), '037833100');
|
||||
assert.equal(curatedCusipForSymbol('ANY'), '84841L506');
|
||||
assert.equal(curatedCusipForSymbol('SLNH'), '583543301');
|
||||
assert.equal(curatedCusipForSymbol('NUAI'), '64428N109');
|
||||
assert.equal(curatedCusipForSymbol('IRE'), '88636Y888');
|
||||
});
|
||||
|
||||
test('resolveCusipLocal seeds cache from curated map', () => {
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
// Probe X evidence URLs for tracked-fund captures. Paywalled posts still
|
||||
// resolve via cookie bird; "Tweet not found" means deleted / no longer available.
|
||||
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
export type EvidenceStatus = 'ok' | 'deleted' | 'unknown';
|
||||
|
||||
const STATUS_KEY = (id: string) => `x:post-status:${id}`;
|
||||
const OK_TTL_MS = 12 * 60 * 60_000;
|
||||
const DELETED_TTL_MS = 7 * 24 * 60 * 60_000;
|
||||
|
||||
export function evidencePostId(url: string | null | undefined): string | null {
|
||||
if (!url) return null;
|
||||
const m = /(?:x\.com|twitter\.com)\/[^/]+\/status\/(\d+)/i.exec(url);
|
||||
return m?.[1] ?? null;
|
||||
}
|
||||
|
||||
export function isTweetGoneMessage(msg: string): boolean {
|
||||
return /tweet not found|post not found|status not found|no status found|does not exist/i.test(msg);
|
||||
}
|
||||
|
||||
export interface EvidenceRow {
|
||||
evidence_url: string | null;
|
||||
evidence_status?: EvidenceStatus;
|
||||
}
|
||||
|
||||
export async function annotateCaptureEvidence(
|
||||
db: DatabaseSync,
|
||||
rows: EvidenceRow[],
|
||||
lookup: (postId: string) => Promise<EvidenceStatus>,
|
||||
now = Date.now(),
|
||||
opts: { live?: boolean } = {},
|
||||
): Promise<void> {
|
||||
const live = opts.live !== false;
|
||||
const ids = [...new Set(rows.map((r) => evidencePostId(r.evidence_url)).filter((x): x is string => !!x))];
|
||||
const statusById = new Map<string, EvidenceStatus>();
|
||||
|
||||
for (const id of ids) {
|
||||
const cached = readCachedStatus(db, id, now);
|
||||
if (cached) {
|
||||
statusById.set(id, cached);
|
||||
continue;
|
||||
}
|
||||
if (!live) {
|
||||
statusById.set(id, 'unknown');
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const status = await lookup(id);
|
||||
if (status !== 'unknown') writeCachedStatus(db, id, status, now);
|
||||
statusById.set(id, status);
|
||||
} catch {
|
||||
statusById.set(id, 'unknown');
|
||||
}
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
const id = evidencePostId(row.evidence_url);
|
||||
if (!id) continue;
|
||||
row.evidence_status = statusById.get(id) ?? 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
function readCachedStatus(db: DatabaseSync, id: string, now: number): EvidenceStatus | null {
|
||||
try {
|
||||
const row = db.prepare('SELECT value, observed_at FROM kv_cache WHERE key=?').get(STATUS_KEY(id)) as
|
||||
| { value: string; observed_at: string }
|
||||
| undefined;
|
||||
if (!row) return null;
|
||||
if (row.value !== 'ok' && row.value !== 'deleted') return null;
|
||||
const age = now - Date.parse(row.observed_at);
|
||||
const ttl = row.value === 'deleted' ? DELETED_TTL_MS : OK_TTL_MS;
|
||||
if (!Number.isFinite(age) || age > ttl) return null;
|
||||
return row.value;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function writeCachedStatus(db: DatabaseSync, id: string, status: EvidenceStatus, now: number): void {
|
||||
try {
|
||||
db.prepare('INSERT OR REPLACE INTO kv_cache (key, value, observed_at) VALUES (?,?,?)').run(
|
||||
STATUS_KEY(id),
|
||||
status,
|
||||
new Date(now).toISOString(),
|
||||
);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,12 @@
|
||||
// Local-only: no network, no rate limiting — this is a materializer, run from
|
||||
// the x schedule branch after timeline jobs are enqueued.
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
import { extractCaptures } from '../mirror/captureParser.ts';
|
||||
import {
|
||||
classifyPost,
|
||||
extractCaptures,
|
||||
extractRemainingBookTickers,
|
||||
inferResetExits,
|
||||
} from '../mirror/captureParser.ts';
|
||||
|
||||
export interface CaptureIngestStats {
|
||||
fundId: string;
|
||||
@@ -39,6 +44,7 @@ interface PostRow {
|
||||
post_id: string;
|
||||
body_text: string | null;
|
||||
posted_at: string;
|
||||
quoted_post_id?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -65,7 +71,9 @@ export function ingestFundCaptures(db: DatabaseSync, fundId: string): CaptureIng
|
||||
if (fund.enabled !== 1) { stats.disabled = true; return stats; }
|
||||
|
||||
const posts = db.prepare(
|
||||
`SELECT post_id, body_text, posted_at FROM x_cookie_posts
|
||||
`SELECT post_id, body_text, posted_at,
|
||||
${hasQuotedPostIdColumn(db) ? 'quoted_post_id' : 'NULL AS quoted_post_id'}
|
||||
FROM x_cookie_posts
|
||||
WHERE lower(author_handle) = lower(?)
|
||||
AND body_text IS NOT NULL AND body_text != ''
|
||||
ORDER BY posted_at ASC`,
|
||||
@@ -86,7 +94,21 @@ export function ingestFundCaptures(db: DatabaseSync, fundId: string): CaptureIng
|
||||
|
||||
for (const p of posts) {
|
||||
// One post can mention several names (Mike: SLNH fill + OPEN total in one tweet).
|
||||
const parsedList = extractCaptures(p.body_text ?? '');
|
||||
const parsedList = capturesForPost(db, fund.x_handle, p);
|
||||
const stayed = extractRemainingBookTickers(p.body_text ?? '');
|
||||
if (stayed) {
|
||||
const held = db.prepare(
|
||||
`SELECT fpr.symbol FROM fund_position_records fpr
|
||||
JOIN (
|
||||
SELECT symbol, MAX(as_of) AS max_as_of
|
||||
FROM fund_position_records WHERE fund_id = ? GROUP BY symbol
|
||||
) latest ON latest.symbol = fpr.symbol AND latest.max_as_of = fpr.as_of
|
||||
WHERE fpr.fund_id = ? AND (fpr.shares IS NULL OR fpr.shares > 0)`,
|
||||
).all(fundId, fundId) as Array<{ symbol: string }>;
|
||||
for (const exit of inferResetExits(held.map((h) => h.symbol), stayed)) {
|
||||
if (!parsedList.some((x) => x.symbol === exit.symbol)) parsedList.push(exit);
|
||||
}
|
||||
}
|
||||
if (parsedList.length === 0) {
|
||||
stats.skipped++;
|
||||
continue;
|
||||
@@ -94,11 +116,11 @@ export function ingestFundCaptures(db: DatabaseSync, fundId: string): CaptureIng
|
||||
|
||||
let anyMaterialized = false;
|
||||
for (const parsed of parsedList) {
|
||||
if (parsed.class === 'claim') {
|
||||
if (parsed.class === 'claim' && !isFoldableAddClaim(p.body_text ?? '', parsed.symbol, parsed.shares)) {
|
||||
stats.claims++;
|
||||
continue;
|
||||
}
|
||||
if (parsed.class !== 'capture') continue;
|
||||
if (parsed.class !== 'capture' && parsed.class !== 'claim') continue;
|
||||
if (!parsed.symbol) continue;
|
||||
|
||||
const asOf = normalizePostedAtDate(p.posted_at);
|
||||
@@ -108,17 +130,33 @@ export function ingestFundCaptures(db: DatabaseSync, fundId: string): CaptureIng
|
||||
? parsed.instrument
|
||||
: null;
|
||||
const existing = selectExisting.get(fundId, parsed.symbol, evidenceUrl) as { id?: string } | undefined;
|
||||
let shares = parsed.shares ?? null;
|
||||
if (parsed.class === 'claim') {
|
||||
// Fold the delta once. A re-run of the same tweet must not add it again.
|
||||
if (existing?.id) {
|
||||
// Keep the already-folded share count; only refresh non-qty fields.
|
||||
db.prepare(
|
||||
`UPDATE fund_position_records SET value_usd=COALESCE(?, value_usd), cost_basis=COALESCE(?, cost_basis), as_of=? WHERE id=?`,
|
||||
).run(parsed.value_usd ?? null, parsed.cost_basis ?? null, asOf, existing.id);
|
||||
stats.refreshed++;
|
||||
stats.captures++;
|
||||
anyMaterialized = true;
|
||||
continue;
|
||||
}
|
||||
const prior = latestInstrumentShares(db, fundId, parsed.symbol, instrumentNote);
|
||||
shares = (prior ?? 0) + (parsed.shares ?? 0);
|
||||
}
|
||||
|
||||
if (existing?.id) {
|
||||
updateExisting.run(
|
||||
parsed.shares ?? null, parsed.value_usd ?? null, parsed.cost_basis ?? null, asOf,
|
||||
shares, parsed.value_usd ?? null, parsed.cost_basis ?? null, asOf,
|
||||
instrumentNote, existing.id,
|
||||
);
|
||||
stats.refreshed++;
|
||||
} else {
|
||||
insertNew.run(
|
||||
crypto.randomUUID(), fundId, parsed.symbol,
|
||||
parsed.shares ?? null, parsed.value_usd ?? null, parsed.cost_basis ?? null, asOf, evidenceUrl,
|
||||
shares, parsed.value_usd ?? null, parsed.cost_basis ?? null, asOf, evidenceUrl,
|
||||
instrumentNote, new Date().toISOString(),
|
||||
);
|
||||
stats.inserted++;
|
||||
@@ -136,6 +174,81 @@ export function ingestFundCaptures(db: DatabaseSync, fundId: string): CaptureIng
|
||||
return stats;
|
||||
}
|
||||
|
||||
function hasQuotedPostIdColumn(db: DatabaseSync): boolean {
|
||||
const cols = db.prepare('PRAGMA table_info(x_cookie_posts)').all() as Array<{ name: string }>;
|
||||
return cols.some((c) => c.name === 'quoted_post_id');
|
||||
}
|
||||
|
||||
/** Quote-tweet with no ticker: inherit the symbol from the quoted capture. */
|
||||
export function inheritSymbolFromQuotedPost(
|
||||
db: DatabaseSync,
|
||||
handle: string,
|
||||
quotedPostId: string | null | undefined,
|
||||
): string | undefined {
|
||||
if (!quotedPostId) return undefined;
|
||||
const quoted = db.prepare(
|
||||
`SELECT body_text FROM x_cookie_posts WHERE post_id = ?`,
|
||||
).get(quotedPostId) as { body_text: string | null } | undefined;
|
||||
if (quoted?.body_text) {
|
||||
const fromQuote = extractCaptures(quoted.body_text).find((p) => p.symbol)?.symbol;
|
||||
if (fromQuote) return fromQuote;
|
||||
}
|
||||
const evidence = `https://x.com/${handle}/status/${quotedPostId}`;
|
||||
const row = db.prepare(
|
||||
`SELECT symbol FROM fund_position_records WHERE evidence_url = ? ORDER BY as_of DESC LIMIT 1`,
|
||||
).get(evidence) as { symbol: string } | undefined;
|
||||
return row?.symbol;
|
||||
}
|
||||
|
||||
function capturesForPost(db: DatabaseSync, handle: string, p: PostRow) {
|
||||
const parsedList = extractCaptures(p.body_text ?? '');
|
||||
const inherited = inheritSymbolFromQuotedPost(db, handle, p.quoted_post_id);
|
||||
if (!inherited) return parsedList;
|
||||
for (const parsed of parsedList) {
|
||||
if (!parsed.symbol) parsed.symbol = inherited;
|
||||
}
|
||||
if (parsedList.length === 0) {
|
||||
const one = classifyPost(p.body_text ?? '');
|
||||
if (one.class !== 'none') {
|
||||
one.symbol = inherited;
|
||||
parsedList.push(one);
|
||||
}
|
||||
}
|
||||
return parsedList;
|
||||
}
|
||||
|
||||
/** Clean single add ("I added 115,000 shares in ENHA at $1.79"). Not an order-fill story. */
|
||||
export function isFoldableAddClaim(
|
||||
text: string,
|
||||
symbol: string | undefined,
|
||||
shares: number | undefined,
|
||||
): boolean {
|
||||
if (!symbol || shares == null || shares <= 0) return false;
|
||||
const qtyMentions = (text.match(/([\d.,]+\s*[MBK]?\s*shares)/gi) ?? []).length;
|
||||
if (qtyMentions !== 1) return false;
|
||||
if (/\bfilled\b|\bshare orders?\b/i.test(text)) return false;
|
||||
if (/\bsold\b|\btrimmed?\b/i.test(text)) return false;
|
||||
return /\badded\b|\bbought\b|\bpurchased\b/i.test(text);
|
||||
}
|
||||
|
||||
function latestInstrumentShares(
|
||||
db: DatabaseSync,
|
||||
fundId: string,
|
||||
symbol: string,
|
||||
instrumentNote: string | null,
|
||||
): number | null {
|
||||
const inst = instrumentNote ?? '';
|
||||
const row = db.prepare(
|
||||
`SELECT shares FROM fund_position_records
|
||||
WHERE fund_id = ? AND symbol = ?
|
||||
AND CASE WHEN notes IN ('call','put') THEN notes ELSE '' END = ?
|
||||
ORDER BY as_of DESC, created_at DESC
|
||||
LIMIT 1`,
|
||||
).get(fundId, symbol, inst) as { shares: number | null } | undefined;
|
||||
if (!row || row.shares == null) return null;
|
||||
return row.shares;
|
||||
}
|
||||
|
||||
/** posted_at may be ISO or Twitter "Wed Jul 15 20:43:28 +0000 2026". */
|
||||
function normalizePostedAtDate(postedAt: string): string {
|
||||
if (/^\d{4}-\d{2}-\d{2}/.test(postedAt)) return postedAt.slice(0, 10);
|
||||
|
||||
@@ -17,8 +17,13 @@ export const CUSIP_TO_SYMBOL: Record<string, string> = {
|
||||
Q4982L109: 'IREN', // IREN Limited
|
||||
'46438F101': 'IBIT', // iShares Bitcoin Trust ETF
|
||||
'46438R105': 'ETHA', // iShares Ethereum Trust ETF
|
||||
'526057104': 'LEN', // Lennar Corporation
|
||||
'583543301': 'SLNH', // Soluna Holdings, Inc. (ISIN US5835433013)
|
||||
'64428N109': 'NUAI', // New Era Energy & Digital, Inc.
|
||||
'670100205': 'NVO', // Novo-Nordisk A/S
|
||||
'683712103': 'OPEN', // Opendoor Technologies Inc.
|
||||
'84841L506': 'ANY', // Sphere 3D Corp. (post Feb 2026 reverse split)
|
||||
'88636Y888': 'IRE', // Defiance Daily Target 2X Long IREN ETF
|
||||
'713448108': 'PEP', // PepsiCo, Inc.
|
||||
'75886F107': 'REGN', // Regeneron Pharmaceuticals, Inc.
|
||||
'862945300': 'ASST', // Strive, Inc.
|
||||
|
||||
@@ -374,7 +374,7 @@ export async function refreshHoldersViaReverse13f(
|
||||
INSERT INTO institution_filings
|
||||
(filer_cik, filer_name, filer_sic, symbol, form, shares, value_usd, reported_quarter, filed_at, accession, fetched_at, put_call)
|
||||
VALUES (?, ?, NULL, ?, '13F-HR', ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(filer_cik, symbol, reported_quarter, form) DO UPDATE SET
|
||||
ON CONFLICT(filer_cik, symbol, reported_quarter, form, put_call) DO UPDATE SET
|
||||
shares = excluded.shares,
|
||||
value_usd = excluded.value_usd,
|
||||
filer_name = COALESCE(excluded.filer_name, institution_filings.filer_name),
|
||||
|
||||
@@ -801,7 +801,7 @@ async function storeMatchedHoldings(
|
||||
INSERT INTO institution_filings
|
||||
(filer_cik, filer_name, filer_sic, symbol, form, shares, value_usd, reported_quarter, filed_at, accession, fetched_at, put_call)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(filer_cik, symbol, reported_quarter, form) DO UPDATE SET
|
||||
ON CONFLICT(filer_cik, symbol, reported_quarter, form, put_call) DO UPDATE SET
|
||||
shares = excluded.shares,
|
||||
value_usd = excluded.value_usd,
|
||||
filer_name = COALESCE(excluded.filer_name, institution_filings.filer_name),
|
||||
@@ -895,7 +895,9 @@ async function fetchAndStoreInstitutionalHolders(
|
||||
return tb - ta;
|
||||
});
|
||||
// Cold symbols need a bigger first pass; warm symbols only chase new filings.
|
||||
const maxDownloads = knownAccessions.size >= 50 ? 40 : 100;
|
||||
// Filing-deadline days dump hundreds of new 13F-HR accessions; 40/run
|
||||
// cannot keep Q2 coverage current when the job actually finishes.
|
||||
const maxDownloads = knownAccessions.size >= 50 ? 80 : 120;
|
||||
let downloaded = 0;
|
||||
let skippedKnown = 0;
|
||||
for (const filing of ordered) {
|
||||
|
||||
+200
-19
@@ -39,7 +39,16 @@ import { ConfluenceRepository, rackFromSlots } from '../db/confluenceRepository.
|
||||
import { runSlotBacktest, signalHistoryToStats, type ConfluenceFireEvent } from '../confluence/confluenceBacktest.ts';
|
||||
import { detectPictureChange } from '../confluence/confluenceRack.ts';
|
||||
import { runCorridorBacktest, aggregateBacktest } from '../analysis/corridorBacktest.ts';
|
||||
import { runConfluenceEvaluationCycle } from '../confluence/confluenceEngine.ts';
|
||||
import {
|
||||
runConfluenceEvaluationCycle,
|
||||
runConfluenceReplay,
|
||||
replayCoverage,
|
||||
loadActivePredicates,
|
||||
buildRecentZones,
|
||||
scoreSymbolUnderPrior,
|
||||
learningLedgerStatus,
|
||||
} from '../confluence/confluenceEngine.ts';
|
||||
import { REPLAY_LOOKBACK_DAYS } from '../confluence/confluenceSeed.ts';
|
||||
import { CorridorRepository } from '../db/corridorRepository.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -376,12 +385,14 @@ const marketRouter = router({
|
||||
const byKey = new Map(entries.map((e) => [e.key, e]));
|
||||
const val = <T>(key: string): T | null => (byKey.get(key)?.value ?? null) as T | null;
|
||||
const stale = (key: string): boolean => byKey.get(key)?.isStale ?? true;
|
||||
const fetchedAt = (key: string): string | null => byKey.get(key)?.fetchedAt ?? null;
|
||||
return {
|
||||
symbol,
|
||||
quote: val<Quote>(k.quote),
|
||||
candles: val<PriceCandle[]>(k.candles),
|
||||
sector: val<SymbolMeta>(k.sector),
|
||||
stale: { quote: stale(k.quote), candles: stale(k.candles), sector: stale(k.sector) },
|
||||
observed: { quote: fetchedAt(k.quote), candles: fetchedAt(k.candles), sector: fetchedAt(k.sector) },
|
||||
};
|
||||
}),
|
||||
|
||||
@@ -411,6 +422,7 @@ const marketRouter = router({
|
||||
candles: PriceCandle[] | null;
|
||||
sector: SymbolMeta | null;
|
||||
stale: { quote: boolean; candles: boolean; sector: boolean };
|
||||
observed: { quote: string | null; candles: string | null; sector: string | null };
|
||||
}> = [];
|
||||
|
||||
for (const symbol of symbols) {
|
||||
@@ -428,6 +440,11 @@ const marketRouter = router({
|
||||
candles: byKey.get(kCandles)?.isStale ?? true,
|
||||
sector: byKey.get(kSector)?.isStale ?? true,
|
||||
},
|
||||
observed: {
|
||||
quote: byKey.get(kQuote)?.fetchedAt ?? null,
|
||||
candles: byKey.get(kCandles)?.fetchedAt ?? null,
|
||||
sector: byKey.get(kSector)?.fetchedAt ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -617,17 +634,23 @@ const marketRouter = router({
|
||||
}),
|
||||
|
||||
candles: publicProcedure
|
||||
.input(z.object({ symbol: z.string().min(1), timeframe: z.enum(['1d', '1wk', '1mo']).default('1d') }))
|
||||
.input(z.object({ symbol: z.string().min(1), timeframe: z.enum(['1m', '5m', '1d', '1wk', '1mo']).default('1d') }))
|
||||
.query(async ({ ctx, input }) => {
|
||||
const symbol = input.symbol.toUpperCase();
|
||||
const key = `yfinance:candles:${symbol}:${input.timeframe}`;
|
||||
const entry = await ctx.cache.get<PriceCandle[]>(key);
|
||||
return { symbol, timeframe: input.timeframe, candles: (entry.value ?? []), isStale: entry.isStale };
|
||||
return {
|
||||
symbol,
|
||||
timeframe: input.timeframe,
|
||||
candles: (entry.value ?? []),
|
||||
isStale: entry.isStale,
|
||||
fetchedAt: entry.provenance?.fetchedAt ?? null,
|
||||
};
|
||||
}),
|
||||
indicators: publicProcedure
|
||||
.input(z.object({
|
||||
symbol: z.string().min(1),
|
||||
timeframe: z.enum(['1d', '1wk', '1mo']).default('1d'),
|
||||
timeframe: z.enum(['1m', '5m', '1d', '1wk', '1mo']).default('1d'),
|
||||
periods: z.object({
|
||||
ema: z.array(z.number().int()).default([9, 21, 50, 200]),
|
||||
rsi: z.number().int().default(14),
|
||||
@@ -1859,8 +1882,41 @@ const adminRouter = router({
|
||||
}),
|
||||
|
||||
xAccountsList: adminProcedure.query(({ ctx }) => {
|
||||
const rows = ctx.db.prepare("SELECT id, symbol, handle, COALESCE(label, '') AS label, created_at FROM x_accounts ORDER BY symbol, handle").all();
|
||||
return (rows ?? []) as Array<{id: string; symbol: string; handle: string; label: string; created_at: string}>;
|
||||
const rows = ctx.db.prepare("SELECT id, symbol, handle, COALESCE(label, '') AS label, created_at FROM x_accounts ORDER BY symbol, handle").all() as Array<{id: string; symbol: string; handle: string; label: string; created_at: string}>;
|
||||
const handles = [...new Set(rows.map((r) => r.handle))];
|
||||
const pullByHandle = new Map<string, { status: string; lastAttempt: string | null; error: string | null }>();
|
||||
const lastPostByHandle = new Map<string, string>();
|
||||
if (handles.length > 0) {
|
||||
const keys = handles.flatMap((h) => [`x:timeline:${h}`, `x:timeline:${h.toLowerCase()}`]);
|
||||
const jobs = ctx.db.prepare(
|
||||
`SELECT key, status, last_attempt, error FROM adapter_queue WHERE key IN (${keys.map(() => '?').join(',')})`,
|
||||
).all(...keys) as Array<{ key: string; status: string; last_attempt: string | null; error: string | null }>;
|
||||
for (const j of jobs) {
|
||||
pullByHandle.set(j.key.slice('x:timeline:'.length), {
|
||||
status: j.status,
|
||||
lastAttempt: j.last_attempt,
|
||||
error: j.error,
|
||||
});
|
||||
}
|
||||
const posts = ctx.db.prepare(
|
||||
`SELECT lower(author_handle) AS h, MAX(posted_at) AS last_post
|
||||
FROM x_cookie_posts WHERE lower(author_handle) IN (${handles.map(() => '?').join(',')})
|
||||
GROUP BY 1`,
|
||||
).all(...handles.map((h) => h.toLowerCase())) as Array<{ h: string; last_post: string | null }>;
|
||||
for (const p of posts) {
|
||||
if (p.last_post) lastPostByHandle.set(p.h, p.last_post);
|
||||
}
|
||||
}
|
||||
return rows.map((r) => {
|
||||
const pull = pullByHandle.get(r.handle) ?? pullByHandle.get(r.handle.toLowerCase()) ?? null;
|
||||
return {
|
||||
...r,
|
||||
pullStatus: pull?.status ?? null,
|
||||
pullAt: pull?.lastAttempt ?? null,
|
||||
pullError: pull?.error ?? null,
|
||||
lastPostAt: lastPostByHandle.get(r.handle.toLowerCase()) ?? null,
|
||||
};
|
||||
});
|
||||
}),
|
||||
|
||||
xAccountAdd: adminProcedure
|
||||
@@ -4604,6 +4660,8 @@ const xRouter = router({
|
||||
nextCursor: null,
|
||||
configured: true as const,
|
||||
health: null,
|
||||
lastPullAt: null,
|
||||
queuedHandles: 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4653,6 +4711,17 @@ const xRouter = router({
|
||||
|
||||
const healthRow = ctx.db.prepare('SELECT healthy, last_error, updated_at FROM x_credentials WHERE id=?').get('singleton') as { healthy?: number; last_error?: string | null } | undefined;
|
||||
|
||||
const pullKeys = handles.flatMap((h) => [`x:timeline:${h}`, `x:timeline:${h.toLowerCase()}`]);
|
||||
const pullJobs = pullKeys.length
|
||||
? ctx.db.prepare(
|
||||
`SELECT key, status, last_attempt, error FROM adapter_queue WHERE key IN (${pullKeys.map(() => '?').join(',')})`,
|
||||
).all(...pullKeys) as Array<{ key: string; status: string; last_attempt: string | null; error: string | null }>
|
||||
: [];
|
||||
const lastPullAt = pullJobs.map((j) => j.last_attempt).filter((t): t is string => Boolean(t)).sort().at(-1) ?? null;
|
||||
const queuedHandles = new Set(
|
||||
pullJobs.filter((j) => j.status === 'pending' && !j.last_attempt).map((j) => j.key.slice('x:timeline:'.length).toLowerCase()),
|
||||
).size;
|
||||
|
||||
return {
|
||||
cashtagPosts: page,
|
||||
accountPosts: [],
|
||||
@@ -4662,6 +4731,8 @@ const xRouter = router({
|
||||
healthy: healthRow.healthy === 1 ? ('healthy' as const) : ('degraded' as const),
|
||||
lastError: healthRow.last_error ?? null,
|
||||
} : null,
|
||||
lastPullAt,
|
||||
queuedHandles,
|
||||
};
|
||||
}),
|
||||
|
||||
@@ -5238,17 +5309,22 @@ const symbolsRouter = router({
|
||||
|
||||
// Tier 1 — tracked funds holding the symbol (most recent record per fund), with weight in their disclosed book.
|
||||
const trackedRows = db.prepare(
|
||||
`SELECT tf.id AS fund_id, tf.fund_name, fpr.symbol, fpr.shares, fpr.value_usd, fpr.as_of, fpr.source
|
||||
`SELECT tf.id AS fund_id, tf.fund_name, fpr.symbol, fpr.shares, fpr.value_usd,
|
||||
fpr.as_of, fpr.source, fpr.notes
|
||||
FROM fund_position_records fpr
|
||||
JOIN tracked_funds tf ON tf.id = fpr.fund_id AND tf.enabled = 1
|
||||
JOIN (
|
||||
SELECT fund_id, MAX(as_of) AS max_as_of
|
||||
SELECT fund_id,
|
||||
CASE WHEN notes IN ('call','put') THEN notes ELSE '' END AS inst,
|
||||
MAX(as_of) AS max_as_of
|
||||
FROM fund_position_records
|
||||
WHERE symbol = ?
|
||||
GROUP BY fund_id
|
||||
GROUP BY fund_id, CASE WHEN notes IN ('call','put') THEN notes ELSE '' END
|
||||
) latest ON latest.fund_id = fpr.fund_id AND latest.max_as_of = fpr.as_of
|
||||
AND CASE WHEN fpr.notes IN ('call','put') THEN fpr.notes ELSE '' END = latest.inst
|
||||
WHERE fpr.symbol = ?
|
||||
ORDER BY tf.fund_name ASC`,
|
||||
AND (fpr.shares IS NULL OR fpr.shares > 0)
|
||||
ORDER BY tf.fund_name ASC, fpr.notes ASC`,
|
||||
).all(symbol, symbol) as Array<Record<string, any>>;
|
||||
|
||||
// Weight = position value / total disclosed book value at the fund's live book (per fund).
|
||||
@@ -5268,6 +5344,7 @@ const symbolsRouter = router({
|
||||
valueUsd: r.value_usd ?? null,
|
||||
asOf: r.as_of,
|
||||
source: r.source,
|
||||
notes: (r.notes as string | null) ?? null,
|
||||
weightPct: bookValueByFund.get(r.fund_id)
|
||||
? ((r.value_usd ?? 0) / bookValueByFund.get(r.fund_id)!) * 100
|
||||
: null,
|
||||
@@ -5290,16 +5367,20 @@ const fundsRouter = router({
|
||||
/** List operator-curated tracked funds (v1: Alpine Fox). */
|
||||
list: publicProcedure
|
||||
.query(async ({ ctx }) => {
|
||||
const { listTrackedFunds } = await import('../db/fundRepository.ts');
|
||||
return listTrackedFunds(ctx.db, { includeDisabled: true });
|
||||
const { listTrackedFunds, fundsFreshness } = await import('../db/fundRepository.ts');
|
||||
const funds = listTrackedFunds(ctx.db, { includeDisabled: true });
|
||||
const fresh = fundsFreshness(ctx.db, funds);
|
||||
return funds.map((f) => ({ ...f, freshness: fresh.get(f.id) ?? null }));
|
||||
}),
|
||||
|
||||
/** Get a single tracked fund by id. */
|
||||
get: publicProcedure
|
||||
.input(z.object({ id: z.string().min(1) }))
|
||||
.query(async ({ ctx, input }) => {
|
||||
const { getTrackedFund } = await import('../db/fundRepository.ts');
|
||||
return getTrackedFund(ctx.db, input.id);
|
||||
const { getTrackedFund, fundFreshness } = await import('../db/fundRepository.ts');
|
||||
const fund = getTrackedFund(ctx.db, input.id);
|
||||
if (!fund) return null;
|
||||
return { ...fund, freshness: fundFreshness(ctx.db, fund) };
|
||||
}),
|
||||
|
||||
/** Live Book for a fund — most recent record per symbol, source-labeled. */
|
||||
@@ -5307,7 +5388,20 @@ const fundsRouter = router({
|
||||
.input(z.object({ fundId: z.string().min(1) }))
|
||||
.query(async ({ ctx, input }) => {
|
||||
const { liveBook } = await import('../db/fundRepository.ts');
|
||||
return liveBook(ctx.db, input.fundId);
|
||||
const book = liveBook(ctx.db, input.fundId);
|
||||
if (ctx.xAdapter) {
|
||||
const { annotateCaptureEvidence } = await import('../services/captureEvidence.ts');
|
||||
// Cache only on the request path. Live bird reads here were serial and
|
||||
// routinely blew the 8s client timeout, so the fund page rendered empty.
|
||||
await annotateCaptureEvidence(
|
||||
ctx.db,
|
||||
book,
|
||||
(id) => ctx.xAdapter!.readTweetStatus(id),
|
||||
Date.now(),
|
||||
{ live: false },
|
||||
);
|
||||
}
|
||||
return book;
|
||||
}),
|
||||
|
||||
/** Full append-only position timeline for a fund. */
|
||||
@@ -5330,7 +5424,11 @@ const fundsRouter = router({
|
||||
}))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const { upsertTrackedFund } = await import('../db/fundRepository.ts');
|
||||
return upsertTrackedFund(ctx.db, input);
|
||||
const fund = upsertTrackedFund(ctx.db, input);
|
||||
if (fund.enabled && fund.x_handle) {
|
||||
try { await ctx.queue.queue(`x:timeline:${fund.x_handle}`); } catch { /* hourly schedule is the fallback */ }
|
||||
}
|
||||
return fund;
|
||||
}),
|
||||
|
||||
adminSetEnabled: adminProcedure
|
||||
@@ -5530,11 +5628,90 @@ const confluenceRouter = router({
|
||||
body: (meta?.body ?? 'bull') as 'bull' | 'bear' | 'exit',
|
||||
firedAt: f.firedAt,
|
||||
qualityAtFire: f.qualityAtFire,
|
||||
verdict: f.verdict,
|
||||
priceConfirmed: f.priceConfirmed,
|
||||
};
|
||||
});
|
||||
return { fires };
|
||||
}),
|
||||
|
||||
/** Last few entry/exit windows for the open symbol, plus the active derived rules. */
|
||||
recentZones: publicProcedure
|
||||
.input(z.object({
|
||||
symbol: z.string().min(1).max(12),
|
||||
rackId: z.string().min(1).optional(),
|
||||
entries: z.number().int().min(1).max(8).optional(),
|
||||
exits: z.number().int().min(1).max(8).optional(),
|
||||
}))
|
||||
.query(async ({ ctx, input }) => {
|
||||
const repo = new ConfluenceRepository(ctx.db);
|
||||
const symbol = input.symbol.toUpperCase();
|
||||
const rackId = input.rackId ?? (repo.listSystemRacks()[0]?.id ?? null);
|
||||
if (!rackId) {
|
||||
return {
|
||||
symbol,
|
||||
rackId: null,
|
||||
entries: [],
|
||||
exits: [],
|
||||
entryRule: { label: 'n/a', source: 'baseline', derivedAt: null, sampleCaveat: 'No rack seeded.', train: null, validate: null },
|
||||
exitRule: { label: 'n/a', source: 'baseline', derivedAt: null, sampleCaveat: 'No rack seeded.', train: null, validate: null },
|
||||
symbolScore: {
|
||||
entry: { zones: 0, resolved: 0, confirmed: 0, hitRate: null },
|
||||
exit: { zones: 0, resolved: 0, confirmed: 0, hitRate: null },
|
||||
},
|
||||
learningLedger: { ready: false, filled: 0, total: 0 },
|
||||
coverage: { evaluatedDays: 0, lookbackDays: REPLAY_LOOKBACK_DAYS, replayComplete: false },
|
||||
};
|
||||
}
|
||||
const entry = await ctx.cache.get<PriceCandle[]>(`yfinance:candles:${symbol}:1d`);
|
||||
const candles = (entry?.value ?? []) as PriceCandle[];
|
||||
const zones = buildRecentZones(ctx.db, symbol, rackId, candles, input.entries ?? 3, input.exits ?? 3);
|
||||
const preds = loadActivePredicates(ctx.db, rackId);
|
||||
const coverage = replayCoverage(ctx.db, symbol, rackId, REPLAY_LOOKBACK_DAYS, candles.length);
|
||||
if (!coverage.replayComplete) {
|
||||
void runConfluenceReplay(ctx.db, ctx.cache, {
|
||||
symbols: [symbol],
|
||||
symbolsPerTick: 1,
|
||||
budgetDaysPerSymbol: 80,
|
||||
}).catch((e) => console.error('[confluence] on-read replay failed:', e));
|
||||
}
|
||||
const entryScore = scoreSymbolUnderPrior(ctx.db, symbol, rackId, candles, 'entry', preds.entry, preds.exit);
|
||||
const exitScore = scoreSymbolUnderPrior(ctx.db, symbol, rackId, candles, 'exit', preds.exit, preds.entry);
|
||||
const ledger = await learningLedgerStatus(ctx.db, ctx.cache);
|
||||
const ruleCard = (
|
||||
stored: typeof preds.entryRule,
|
||||
pred: typeof preds.entry,
|
||||
side: 'entry' | 'exit',
|
||||
) => stored ? {
|
||||
label: pred.label,
|
||||
source: stored.source,
|
||||
derivedAt: stored.derivedAt,
|
||||
sampleCaveat: stored.sampleCaveat,
|
||||
train: JSON.parse(stored.trainStatsJson),
|
||||
validate: JSON.parse(stored.validateStatsJson),
|
||||
} : {
|
||||
label: pred.label,
|
||||
source: 'baseline',
|
||||
derivedAt: null,
|
||||
sampleCaveat: ledger.ready
|
||||
? 'Learning ledger is full. A sector-checked prior has not been stored yet.'
|
||||
: 'Default window definition. Sector-checked prior is not ready (learning ledger still filling).',
|
||||
train: null,
|
||||
validate: null,
|
||||
};
|
||||
return {
|
||||
symbol,
|
||||
rackId,
|
||||
entries: zones.entries,
|
||||
exits: zones.exits,
|
||||
entryRule: ruleCard(preds.entryRule, preds.entry, 'entry'),
|
||||
exitRule: ruleCard(preds.exitRule, preds.exit, 'exit'),
|
||||
symbolScore: { entry: entryScore, exit: exitScore },
|
||||
learningLedger: { ready: ledger.ready, filled: ledger.symbols.filter((s) => s.ready).length, total: ledger.symbols.length },
|
||||
coverage,
|
||||
};
|
||||
}),
|
||||
|
||||
/** Create or update a user-owned rack. */
|
||||
saveRack: protectedProcedure
|
||||
.input(z.object({
|
||||
@@ -5662,10 +5839,14 @@ const confluenceRouter = router({
|
||||
runEvaluationNow: protectedProcedure
|
||||
.input(z.object({ symbol: z.string().min(1).max(12).optional() }))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const summary = await runConfluenceEvaluationCycle(ctx.db, ctx.cache, {
|
||||
symbols: input?.symbol ? [input.symbol] : undefined,
|
||||
const symbols = input?.symbol ? [input.symbol] : undefined;
|
||||
const replay = await runConfluenceReplay(ctx.db, ctx.cache, {
|
||||
symbols: symbols ?? undefined,
|
||||
symbolsPerTick: symbols ? 1 : 3,
|
||||
budgetDaysPerSymbol: symbols ? 200 : 40,
|
||||
});
|
||||
return summary;
|
||||
const summary = await runConfluenceEvaluationCycle(ctx.db, ctx.cache, { symbols });
|
||||
return { ...summary, replay };
|
||||
}),
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user