slice 1a-1d: DB + CacheRepository + YFinance adapter + AdapterQueue

Node 26 + node:sqlite backend (zero native deps; runtime glue adapted from
Bun-spec design, deep-module architecture unchanged). 29 tests green across
schema/CacheRepository/YFinance-parse/AdapterQueue-dedupe.
This commit is contained in:
Investor Flow Build
2026-06-29 17:28:57 -04:00
commit 1962ecc740
17 changed files with 3584 additions and 0 deletions
@@ -0,0 +1,60 @@
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { createDb, initSchema } from '../client.ts';
function freshDb() {
const d = createDb({ path: ':memory:' });
initSchema(d);
return d;
}
// Every table the locked schema (DESIGN.md §1 + §1.5 + §2.6 + deltas) must define.
const EXPECTED_TABLES = [
'users', 'sessions', 'sizing_unlocks',
'symbols', 'price_candles', 'quotes', 'options_chains', 'filings',
'institution_filings', 'insider_transactions', 'sector_map',
'macro_events', 'macro_regimes', 'price_adjustments',
'threads', 'adapter_queue', 'symbol_demand',
'watchlists', 'portfolio_holdings', 'trades', 'strategies',
'screener_filters', 'reports', 'alerts', 'trusted_accounts',
'saved_posts', 'trade_key_levels',
'llm_summaries', 'rotation_signals', 'llm_providers', 'llm_dispatch_audit',
];
test('schema creates all locked tables', () => {
const d = freshDb();
const names = (d.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all() as Array<{ name: string }>).map((r) => r.name);
for (const t of EXPECTED_TABLES) {
assert.ok(names.includes(t), `missing table: ${t}`);
}
});
test('users -> sessions FK cascades on delete', () => {
const d = freshDb();
d.prepare('INSERT INTO users (id,email,pw_hash,created_at) VALUES (?,?,?,?)').run('u1', 'a@b.co', 'hash', '2026-01-01T00:00:00Z');
d.prepare('INSERT INTO sessions (id,user_id,expires_at,created_at) VALUES (?,?,?,?)').run('s1', 'u1', '2026-12-31T00:00:00Z', '2026-01-01T00:00:00Z');
assert.equal((d.prepare('SELECT COUNT(*) AS c FROM sessions WHERE user_id=?').get('u1') as { c: number }).c, 1);
d.prepare('DELETE FROM users WHERE id=?').run('u1');
assert.equal((d.prepare('SELECT COUNT(*) AS c FROM sessions').get() as { c: number }).c, 0);
});
test('symbol_demand defaults in_demand=1, refcount=0 (demand-set gate)', () => {
const d = freshDb();
d.prepare('INSERT INTO symbol_demand (symbol,refcount,ticker_kind) VALUES (?,?,?)').run('NVDA', 0, 'equity');
const r = d.prepare('SELECT in_demand, refcount FROM symbol_demand WHERE symbol=?').get('NVDA') as { in_demand: number; refcount: number };
assert.equal(r.in_demand, 1);
assert.equal(r.refcount, 0);
});
test('price_candles PK dedupes (symbol,timeframe,ts) — backtest integrity', () => {
const d = freshDb();
const ins = d.prepare('INSERT INTO price_candles (symbol,timeframe,ts,o,h,l,c,v,adj_close,observed_at) VALUES (?,?,?,?,?,?,?,?,?,?)');
ins.run('NVDA', '1d', '2026-01-02', 130, 132, 129, 131, 1e6, 131, '2026-01-02T22:00:00Z');
assert.throws(() => ins.run('NVDA', '1d', '2026-01-02', 130, 132, 129, 131, 1e6, 131, '2026-01-02T22:00:00Z'));
});
test('schema.sql is idempotent (re-init does not error)', () => {
const d = freshDb();
// re-applying must not throw (CREATE ... IF NOT EXISTS)
assert.doesNotThrow(() => initSchema(d));
});
+66
View File
@@ -0,0 +1,66 @@
// Investor Flow — SQLite DB client (node:sqlite). DESIGN.md §1 schema seam.
//
// Runtime adaptation (reversible): design specified Bun + bun:sqlite. Implemented on
// Node 26 native TS (type-stripping) + node:sqlite (built-in) so the backend has ZERO
// native dependencies. Deep-module architecture (CacheRepository, etc.) is unchanged.
import { DatabaseSync } from 'node:sqlite';
import { readFileSync, mkdirSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = dirname(fileURLToPath(import.meta.url));
const SCHEMA_PATH = join(__dirname, 'schema.sql');
const DEFAULT_DB_DIR = join(__dirname, '..', '..', 'data');
const DEFAULT_DB_PATH = join(DEFAULT_DB_DIR, 'investor-flow.db');
export interface DbOptions {
/** SQLite file path. Defaults to $IFLOW_DB_PATH or <server>/data/investor-flow.db. Use ':memory:' for tests. */
path?: string;
readonly?: boolean;
}
/** Open a SQLite connection with foreign keys + WAL enabled. */
export function createDb(opts: DbOptions = {}): DatabaseSync {
const path = opts.path ?? process.env.IFLOW_DB_PATH ?? DEFAULT_DB_PATH;
if (!opts.readonly && path !== ':memory:') {
mkdirSync(dirname(path), { recursive: true });
}
const database = new DatabaseSync(path, {
readOnly: opts.readonly ?? false,
enableForeignKeyConstraints: true,
});
// WAL is persistent on the file; setting per-connection is harmless and ensures it.
try {
database.exec('PRAGMA journal_mode = WAL;');
} catch {
/* readOnly or disallowed — ignore */
}
return database;
}
/** Apply the full schema (CREATE TABLE IF NOT EXISTS). Idempotent. */
export function initSchema(database: DatabaseSync): void {
const sql = readFileSync(SCHEMA_PATH, 'utf8');
database.exec(sql);
}
let _db: DatabaseSync | null = null;
/** Process-wide singleton connection (auto-creates + inits schema on first call). */
export function db(): DatabaseSync {
if (!_db) {
_db = createDb();
initSchema(_db);
}
return _db;
}
/** CLI entry: `node src/db/client.ts` initializes the on-disk database. */
const isMain = process.argv[1] ? resolve(process.argv[1]) === fileURLToPath(import.meta.url) : false;
if (isMain) {
const d = createDb();
initSchema(d);
const path = process.env.IFLOW_DB_PATH ?? DEFAULT_DB_PATH;
const rows = d.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all() as Array<{ name: string }>;
console.log(`[investor-flow] schema initialized at ${path} (${rows.length} tables)`);
d.close();
}
+349
View File
@@ -0,0 +1,349 @@
-- Investor Flow — SQLite schema. Engineering contract: DESIGN.md §1 (Tier A/B/C/D),
-- §1.5 (LLM provenance), §2.6 (ticker-data dedupe), + locked review deltas (price_adjustments, trade_key_levels).
-- Tiers: A shared market cache (ownerId NULL) | B shared content | C per-user (ownerId NOT NULL) | D system.
-- Conventions: timestamps ISO8601 TEXT; observed_at/fetched_at provenance per P4.
-- Idempotent: CREATE ... IF NOT EXISTS. Safe to re-run on every boot.
PRAGMA foreign_keys = ON;
-- ===== Tier D — System / Identity =====
CREATE TABLE IF NOT EXISTS users (
id TEXT PRIMARY KEY, -- uuid
email TEXT UNIQUE NOT NULL,
pw_hash TEXT NOT NULL, -- scrypt(verifiable); design specified argon2id, adapted to built-in scrypt (reversible)
totp_secret TEXT, -- 2FA secret, encrypted at rest (slice 2)
is_2fa_enabled INTEGER NOT NULL DEFAULT 0,
oauth_subject TEXT, -- nullable; social OAuth link (slice 2)
oauth_provider TEXT, -- 'github' | 'google' | NULL
complexity TEXT NOT NULL DEFAULT 'beginner', -- beginner|intermediate|advanced
risk_tolerance TEXT NOT NULL DEFAULT 'moderate',
convexity_posture TEXT NOT NULL DEFAULT 'off', -- off|covered_income|cash_secured_entry|insurance_sleeve|leaps_conviction
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS sessions (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at TEXT NOT NULL,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS sizing_unlocks (
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
unlock TEXT NOT NULL, -- 'tier_a_star' | 'tier_a' | ('covered_income'|...)
earned_at TEXT NOT NULL,
evidence TEXT, -- stats that justified unlock
PRIMARY KEY (user_id, unlock)
);
-- ===== Tier A — Shared market cache (no ownerId) =====
CREATE TABLE IF NOT EXISTS symbols (
symbol TEXT PRIMARY KEY,
name TEXT,
sector TEXT,
industry TEXT,
exchange TEXT,
ticker_kind TEXT NOT NULL, -- equity|crypto|etf|index (gates module applicability)
peers TEXT, -- JSON array
updated_at TEXT
);
CREATE TABLE IF NOT EXISTS price_candles (
symbol TEXT NOT NULL,
timeframe TEXT NOT NULL, -- '1m'|'5m'|'1h'|'1d'|'1w'
ts TEXT NOT NULL,
o REAL, h REAL, l REAL, c REAL, v REAL,
adj_close REAL, -- continuous split+dividend-adjusted close for backtests
observed_at TEXT NOT NULL, -- when fetched (provenance)
PRIMARY KEY (symbol, timeframe, ts)
);
CREATE TABLE IF NOT EXISTS quotes (
symbol TEXT PRIMARY KEY,
price REAL,
bid REAL,
ask REAL,
change REAL,
change_pct REAL,
iv REAL, -- implied vol when available
observed_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS options_chains (
symbol TEXT NOT NULL,
expiry TEXT NOT NULL,
strike REAL NOT NULL,
type TEXT NOT NULL, -- call|put
bid REAL, ask REAL, iv REAL,
delta REAL, gamma REAL, theta REAL, vega REAL, rho REAL,
open_interest INTEGER, volume INTEGER,
ts TEXT NOT NULL, -- snapshot timestamp
PRIMARY KEY (symbol, expiry, strike, type, ts)
);
CREATE TABLE IF NOT EXISTS filings (
accession TEXT PRIMARY KEY, -- SEC accession number
form TEXT NOT NULL, -- 10-K|10-Q|8-K|13F-HR|13D|13G|4|...
symbol TEXT NOT NULL,
filed_at TEXT NOT NULL,
period TEXT,
primary_doc_url TEXT,
items TEXT, -- JSON: list of items (8-K materiality heuristics)
fetched_at TEXT NOT NULL,
raw_payload TEXT -- cached full XBRL/parsed doc
);
CREATE TABLE IF NOT EXISTS institution_filings (
filer_cik TEXT NOT NULL, -- institution's CIK
filer_name TEXT,
filer_sic TEXT, -- SIC code -> classifies MM/broker-dealer (ADR Q2)
symbol TEXT NOT NULL,
form TEXT NOT NULL, -- 13F-HR|13D|13G
shares REAL, -- shares reported
value_usd REAL,
reported_quarter TEXT NOT NULL, -- calendar quarter positions are "as of"
filed_at TEXT NOT NULL,
fetched_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS insider_transactions (
form4_id TEXT NOT NULL, -- accession + offset
symbol TEXT NOT NULL,
insider_name TEXT NOT NULL,
insider_role TEXT, -- officer|director|ten_pct_holder
tx_date TEXT NOT NULL,
tx_code TEXT NOT NULL, -- P|S|M|A|... (SEC codes)
tx_type TEXT NOT NULL, -- buy|sell|exercise|grant|vesting
shares REAL,
price REAL,
is_10b5_1 INTEGER NOT NULL DEFAULT 0, -- 1 = under 10b5-1 plan (Routine)
classification TEXT NOT NULL, -- informed_buy|informed_sell|routine
filed_at TEXT NOT NULL,
fetched_at TEXT NOT NULL,
PRIMARY KEY (form4_id, symbol, insider_name, tx_date)
);
CREATE TABLE IF NOT EXISTS sector_map (
symbol TEXT NOT NULL,
sector TEXT NOT NULL,
as_of TEXT NOT NULL,
rs_ratio REAL, -- relative strength vs SPX
rs_slope REAL, -- slope over N weeks (incipient detection)
breadth_thrust REAL, -- % members beating SPX
PRIMARY KEY (symbol, sector, as_of)
);
CREATE TABLE IF NOT EXISTS macro_events (
event_id TEXT PRIMARY KEY, -- e.g. '2026-06-30-FOMC'
name TEXT NOT NULL,
event_ts TEXT NOT NULL,
impact TEXT, -- high|medium|low
forecast TEXT,
previous TEXT,
actual TEXT, -- populated after release
source TEXT -- 'FRED'|'econoday'|...
);
CREATE TABLE IF NOT EXISTS macro_regimes (
as_of TEXT PRIMARY KEY,
regime TEXT NOT NULL, -- trending_up|trending_down|range_bound|structurally_shifting
signals TEXT -- JSON: yield curve, inflation, rate path, liquidity, breadth
);
CREATE TABLE IF NOT EXISTS price_adjustments (
symbol TEXT NOT NULL,
ex_date TEXT NOT NULL, -- date the price line shifted
type TEXT NOT NULL, -- 'split'|'dividend'
ratio REAL NOT NULL, -- split ratio (3.0 = 3-for-1) or dividend amount ($/share)
PRIMARY KEY (symbol, ex_date, type)
);
-- ===== Tier B — Shared content =====
CREATE TABLE IF NOT EXISTS threads (
source TEXT NOT NULL, -- x|reddit
source_id TEXT NOT NULL, -- tweet id / reddit post id
symbol TEXT NOT NULL,
author TEXT NOT NULL,
body TEXT,
ts TEXT NOT NULL,
sentiment TEXT, -- bullish|bearish|neutral (computed)
llm_summary_hash TEXT, -- cross-ref to llm_summaries
fetched_at TEXT NOT NULL,
PRIMARY KEY (source, source_id)
);
CREATE TABLE IF NOT EXISTS adapter_queue (
key TEXT PRIMARY KEY, -- e.g. "yfinance:quote:NVDA"
status TEXT NOT NULL, -- pending|in_flight|done|backoff
last_attempt TEXT,
retry_count INTEGER NOT NULL DEFAULT 0,
backoff_until TEXT
);
CREATE TABLE IF NOT EXISTS symbol_demand (
symbol TEXT PRIMARY KEY,
refcount INTEGER NOT NULL DEFAULT 0,
ticker_kind TEXT NOT NULL,
in_demand INTEGER NOT NULL DEFAULT 1, -- gate flag for the adapter queue
last_refreshed_at TEXT
);
-- ===== Tier C — Per-user (ownerId NOT NULL) =====
CREATE TABLE IF NOT EXISTS watchlists (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
name TEXT NOT NULL,
symbols TEXT NOT NULL, -- JSON array
created_at TEXT NOT NULL,
sort_order INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS portfolio_holdings (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
symbol TEXT NOT NULL,
qty REAL NOT NULL,
avg_cost REAL NOT NULL,
acquired_at TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'open' -- open|closed
);
CREATE TABLE IF NOT EXISTS trades (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
trade_id TEXT NOT NULL,
symbol TEXT NOT NULL,
date TEXT NOT NULL,
status TEXT NOT NULL, -- planned|active|closed
environment TEXT, -- trending|ranging|transitioning
tier TEXT NOT NULL, -- A_STAR|A|B|C (Conviction Tier)
strategy_id TEXT, -- ref to strategies (nullable)
confluence_score INTEGER,
risk_pct REAL,
position_size REAL, -- computed by SizingEngine
entry_price REAL,
stop_loss REAL,
targets TEXT, -- JSON [{price, reason}]
justifications TEXT, -- JSON {entryWhy, stopWhy, targetWhy}
checklist_completed INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS strategies (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
name TEXT NOT NULL,
regime_gate TEXT NOT NULL, -- JSON: gate config
setup TEXT NOT NULL, -- JSON: entry/exit checklist w/ params
risk_policy TEXT NOT NULL, -- JSON: sizing rules
created_at TEXT NOT NULL,
is_public INTEGER NOT NULL DEFAULT 0 -- shareable read-only (later phase)
);
CREATE TABLE IF NOT EXISTS screener_filters (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
name TEXT NOT NULL,
expressions TEXT NOT NULL, -- JSON: filter expression set
universe_kind TEXT NOT NULL, -- watchlist|portfolio|broader_sector
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS reports (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
scope TEXT NOT NULL, -- symbol|watchlist|portfolio|rotation|sizing_year|risk_posture
range_start TEXT,
range_end TEXT,
format TEXT NOT NULL, -- html|markdown|csv|json
payload_ref TEXT NOT NULL, -- file path in local reports dir
generated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS alerts (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
symbol TEXT,
alert_type TEXT NOT NULL, -- informed_buy|informed_sell|new_13da|rotation_incipient|regime_shift|conviction_unlock|...
params TEXT NOT NULL, -- JSON thresholds
last_fired TEXT,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS trusted_accounts (
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
source TEXT NOT NULL, -- x|reddit
handle TEXT NOT NULL,
PRIMARY KEY (owner_id, source, handle)
);
CREATE TABLE IF NOT EXISTS saved_posts (
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
note TEXT,
linked_trade_id TEXT REFERENCES trades(id) ON DELETE SET NULL,
saved_at TEXT NOT NULL,
PRIMARY KEY (owner_id, source, source_id)
);
CREATE TABLE IF NOT EXISTS trade_key_levels (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
symbol TEXT NOT NULL,
type TEXT NOT NULL, -- swing_high|swing_low|supply|demand|support|resistance
price REAL NOT NULL,
timeframe TEXT, -- '1H'|'4H'|'1D'|...
strength TEXT, -- low|medium|high
created_at TEXT NOT NULL
);
-- ===== Shared LLM cache (output deterministic by prompt-hash; not per-user) =====
CREATE TABLE IF NOT EXISTS llm_summaries (
hash TEXT PRIMARY KEY, -- hash(prompt + source-payload + complexity)
feature TEXT NOT NULL, -- filing_summary|post_summary|dashboard_rollup|explainer|qa|sizing_explain|macro_commentary|...
complexity TEXT NOT NULL,
output TEXT NOT NULL,
cited_sources TEXT, -- JSON refs to cached rows (provenance)
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS rotation_signals (
id TEXT PRIMARY KEY,
sector TEXT NOT NULL,
detected_at TEXT NOT NULL,
phase TEXT NOT NULL, -- accumulation|expansion|distribution|markdown
confidence REAL,
price_resolved INTEGER NOT NULL DEFAULT 0, -- follows through within N=4 weeks?
inst_resolved INTEGER NOT NULL DEFAULT 0, -- 13F/insider flow later confirms?
resolved_at TEXT,
verdict TEXT -- real|false_alarm (gamma two-stage)
);
-- ===== Section 1.5 — LLM provenance (ADR-0006). Operator-managed, NOT user-editable. =====
CREATE TABLE IF NOT EXISTS llm_providers (
id TEXT PRIMARY KEY, -- 'local_ollama' | 'local_vllm' | 'zen_glm_5_2' | ...
base_url TEXT NOT NULL,
is_local INTEGER NOT NULL DEFAULT 0, -- 1 if localhost/127.0.0.1/::1/LAN subnet
review_adr_ref TEXT, -- ADR-XXXX or note documenting provenance review date
enabled INTEGER NOT NULL DEFAULT 0 -- only one with enabled=1 by default (local)
);
CREATE TABLE IF NOT EXISTS llm_dispatch_audit (
id TEXT PRIMARY KEY,
feature TEXT NOT NULL,
payload_class TEXT NOT NULL, -- public_safe|sensitive
provider_id TEXT,
status TEXT NOT NULL, -- sent_local|sent_external|blocked_sensitive_external
ts TEXT NOT NULL
);
-- ===== Indexes =====
CREATE INDEX IF NOT EXISTS idx_pc_symbol_tf_ts ON price_candles(symbol, timeframe, ts);
CREATE INDEX IF NOT EXISTS idx_quote_obs ON quotes(observed_at);
CREATE INDEX IF NOT EXISTS idx_filing_symbol_form ON filings(symbol, form);
CREATE INDEX IF NOT EXISTS idx_instfil_sym_q ON institution_filings(symbol, reported_quarter);
CREATE INDEX IF NOT EXISTS idx_insider_sym_date ON insider_transactions(symbol, tx_date);
CREATE INDEX IF NOT EXISTS idx_threads_symbol_ts ON threads(symbol, ts);
CREATE INDEX IF NOT EXISTS idx_trades_owner ON trades(owner_id);
CREATE INDEX IF NOT EXISTS idx_alerts_owner_sym ON alerts(owner_id, symbol);
CREATE INDEX IF NOT EXISTS idx_signals_sector ON rotation_signals(sector);