transnet/investor-flow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling

Browse Source 
- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips
- Error capture: every failed/backoff attempt logged to queue_errors with the
  full stack trace; admin queue page expands a failed job to stream it
- Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs)
- Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop
  (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes
- Startup recovery: interrupted in_flight jobs reset to pending on boot
- fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the
  company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty
- Migration: add queue_errors, queue_schedules, queue_state tables plus error/
  scheduled_for columns on adapter_queue (idempotent ALTER on startup)

Verified: full sec-fetch backfill now succeeds for all watched symbols
(NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.).
503 backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Investor Flow Build
2026-07-12 12:55:59 -04:00
co-authored by Claude
parent 55f07e6b42
commit ca385c1960
10 changed files with 1345 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/server/src/db/schema.sql
+161 -2
View File
@@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS users (
pw_hash TEXT NOT NULL, -- scrypt(verifiable); design specified argon2id, adapted to built-in scrypt (reversible)
totp_secret TEXT, -- 2FA secret, encrypted at rest (slice 2)
is_2fa_enabled INTEGER NOT NULL DEFAULT 0,
is_admin INTEGER NOT NULL DEFAULT 0, -- 1 = operator (admin CLI/route access, Slice 25); never user-editable
oauth_subject TEXT, -- nullable; social OAuth link (slice 2)
oauth_provider TEXT, -- 'github' | 'google' | NULL
complexity TEXT NOT NULL DEFAULT 'beginner', -- beginner|intermediate|advanced
@@ -180,12 +181,44 @@ CREATE TABLE IF NOT EXISTS threads (
PRIMARY KEY (source, source_id)
);
CREATE TABLE IF NOT EXISTS admin_audit (
id TEXT PRIMARY KEY, -- uuid
actor TEXT NOT NULL, -- admin user id (or 'cli' when none)
action TEXT NOT NULL, -- users.reset-password|users.gdpr-export|queue.reset-backoff|...
target TEXT, -- target user id / sourceKind / null
ts TEXT NOT NULL, -- ISO timestamp
meta TEXT -- optional JSON blob (never secrets)
);
CREATE TABLE IF NOT EXISTS adapter_queue (
key TEXT PRIMARY KEY, -- e.g. "yfinance:quote:NVDA"
status TEXT NOT NULL, -- pending|in_flight|done|backoff
status TEXT NOT NULL, -- pending|in_flight|done|backoff|failed
last_attempt TEXT,
retry_count INTEGER NOT NULL DEFAULT 0,
backoff_until TEXT
backoff_until TEXT,
error TEXT, -- last error message (for failed/backoff)
scheduled_for TEXT -- ISO timestamp; job won't run before this
);
CREATE TABLE IF NOT EXISTS queue_errors (
id INTEGER PRIMARY KEY AUTOINCREMENT,
queue_key TEXT NOT NULL,
attempt INTEGER NOT NULL,
error_message TEXT,
error_stack TEXT,
ts TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS queue_schedules (
source_kind TEXT PRIMARY KEY,
interval_ms INTEGER NOT NULL,
last_enqueued TEXT,
next_enqueue TEXT
);
CREATE TABLE IF NOT EXISTS queue_state (
key TEXT PRIMARY KEY, -- 'paused'
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS symbol_demand (
@@ -206,6 +239,8 @@ CREATE TABLE IF NOT EXISTS watchlists (
sort_order INTEGER NOT NULL DEFAULT 0
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_watchlists_owner_name ON watchlists(owner_id, name);
CREATE TABLE IF NOT EXISTS portfolio_holdings (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
@@ -216,6 +251,9 @@ CREATE TABLE IF NOT EXISTS portfolio_holdings (
status TEXT NOT NULL DEFAULT 'open' -- open|closed
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_portfolio_owner_symbol
ON portfolio_holdings(owner_id, symbol);
CREATE TABLE IF NOT EXISTS trades (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
@@ -306,6 +344,28 @@ CREATE TABLE IF NOT EXISTS trade_key_levels (
);
-- ===== Shared LLM cache (output deterministic by prompt-hash; not per-user) =====
-- ===== Slice 22 — Emotion Logger: per-trade emotion snapshots =====
CREATE TABLE IF NOT EXISTS trade_executions (
id TEXT PRIMARY KEY,
trade_id TEXT NOT NULL REFERENCES trades(id) ON DELETE CASCADE,
status TEXT NOT NULL DEFAULT 'active', -- active|closed
started_at TEXT NOT NULL,
completed_at TEXT,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS emotion_logs (
id TEXT PRIMARY KEY,
trade_execution_id TEXT NOT NULL REFERENCES trade_executions(id) ON DELETE CASCADE,
timestamp INTEGER NOT NULL,
price_at_event REAL, -- price at moment of emotion event
emotion TEXT NOT NULL, -- e.g. 'fear', 'greed', 'FOMO', 'regret'
note TEXT, -- optional user note
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_emotion_logs_trade ON emotion_logs(trade_execution_id);
CREATE TABLE IF NOT EXISTS llm_summaries (
hash TEXT PRIMARY KEY, -- hash(prompt + source-payload + complexity)
feature TEXT NOT NULL, -- filing_summary|post_summary|dashboard_rollup|explainer|qa|sizing_explain|macro_commentary|...
@@ -391,3 +451,102 @@ CREATE INDEX IF NOT EXISTS idx_threads_symbol_ts ON threads(symbol, ts);
CREATE INDEX IF NOT EXISTS idx_trades_owner ON trades(owner_id);
CREATE INDEX IF NOT EXISTS idx_alerts_owner_sym ON alerts(owner_id, symbol);
CREATE INDEX IF NOT EXISTS idx_signals_sector ON rotation_signals(sector);
-- ===== Slice 18 — Risk Engine: Gentle-halt circuit breaker state =====
CREATE TABLE IF NOT EXISTS halt_state (
user_id TEXT PRIMARY KEY,
halted_until TEXT NOT NULL, -- ISO8601; 24h cooldown window
triggered_by TEXT NOT NULL, -- e.g. 'max_drawdown_tolerance_breach'
ts TEXT NOT NULL -- ISO8601; when the halt was recorded
);
-- ===== Slice 17 — AlertEngine: individual alert event instances =====
CREATE TABLE IF NOT EXISTS alert_events (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
type TEXT NOT NULL, -- informed_buy|informed_sell|...
severity TEXT NOT NULL, -- info|warning|critical
title TEXT NOT NULL,
description TEXT NOT NULL,
symbol TEXT,
created_at TEXT NOT NULL,
acknowledged INTEGER NOT NULL DEFAULT 0,
dedup_key TEXT NOT NULL,
payload TEXT NOT NULL DEFAULT '{}', -- JSON
throttle_key TEXT -- for rate limiting: "user:type:symbol:YYYYMMDDHH"
);
CREATE INDEX IF NOT EXISTS idx_alert_events_user ON alert_events(user_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_alert_events_dedup ON alert_events(dedup_key);
CREATE INDEX IF NOT EXISTS idx_alert_events_throttle ON alert_events(throttle_key);
-- ===== Slice 12 — Strategy Lab + Backtest =====
CREATE TABLE IF NOT EXISTS strategies (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
name TEXT NOT NULL,
components TEXT NOT NULL DEFAULT '[]', -- JSON array of StrategyComponent
unlocked INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS backtest_results (
id TEXT PRIMARY KEY,
strategy_id TEXT NOT NULL REFERENCES strategies(id) ON DELETE CASCADE,
symbol TEXT NOT NULL,
timeframe TEXT NOT NULL DEFAULT '1d',
equity_curve TEXT NOT NULL DEFAULT '[]', -- JSON array
exit_reasons TEXT NOT NULL DEFAULT '[]', -- JSON array
sample_size INTEGER NOT NULL,
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_strategies_owner ON strategies(owner_id);
CREATE INDEX IF NOT EXISTS idx_backtest_strategy ON backtest_results(strategy_id);
-- ===== Slice 13 — Saved Filter Sets =====
CREATE TABLE IF NOT EXISTS saved_filters (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
name TEXT NOT NULL,
expression TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'watchlist',
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_saved_filters_user ON saved_filters(user_id);
-- ===== Slice 19 — Options Convexity Sleeve Unlock State =====
CREATE TABLE IF NOT EXISTS options_unlock (
user_id TEXT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
state INTEGER NOT NULL DEFAULT 0,
last_unlock_at TEXT,
understanding INTEGER NOT NULL DEFAULT 0 -- demonstrated understanding flag
);
-- ===== Slice 20 — Macro Regime History =====
CREATE TABLE IF NOT EXISTS regime_history (
id TEXT PRIMARY KEY,
date TEXT NOT NULL,
regime TEXT NOT NULL,
confidence INTEGER NOT NULL,
factors TEXT NOT NULL DEFAULT '[]', -- JSON array
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_regime_history_date ON regime_history(date DESC);
-- ===== Slice 21 — Thesis Monitoring =====
CREATE TABLE IF NOT EXISTS theses (
id TEXT PRIMARY KEY,
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
symbol TEXT NOT NULL,
statement TEXT NOT NULL,
invalidation_criteria TEXT NOT NULL DEFAULT '[]', -- JSON array
status TEXT NOT NULL DEFAULT 'intact',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_theses_user ON theses(user_id);
CREATE INDEX IF NOT EXISTS idx_theses_symbol ON theses(symbol);