feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling
- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips - Error capture: every failed/backoff attempt logged to queue_errors with the full stack trace; admin queue page expands a failed job to stream it - Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs) - Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes - Startup recovery: interrupted in_flight jobs reset to pending on boot - fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty - Migration: add queue_errors, queue_schedules, queue_state tables plus error/ scheduled_for columns on adapter_queue (idempotent ALTER on startup) Verified: full sec-fetch backfill now succeeds for all watched symbols (NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.). 503 backend tests pass. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
55f07e6b42
commit
ca385c1960
@@ -44,12 +44,24 @@ export function initSchema(database: DatabaseSync): void {
|
||||
database.exec(sql);
|
||||
}
|
||||
|
||||
/** Idempotent migrations for existing databases (new columns, tables). */
|
||||
function runMigrations(db: DatabaseSync): void {
|
||||
const migrations: string[] = [
|
||||
`ALTER TABLE adapter_queue ADD COLUMN error TEXT`,
|
||||
`ALTER TABLE adapter_queue ADD COLUMN scheduled_for TEXT`,
|
||||
];
|
||||
for (const sql of migrations) {
|
||||
try { db.exec(sql); } catch { /* column already exists */ }
|
||||
}
|
||||
}
|
||||
|
||||
let _db: DatabaseSync | null = null;
|
||||
/** Process-wide singleton connection (auto-creates + inits schema on first call). */
|
||||
export function db(): DatabaseSync {
|
||||
if (!_db) {
|
||||
_db = createDb();
|
||||
initSchema(_db);
|
||||
runMigrations(_db);
|
||||
// Ensure the anonymous user exists for unauthenticated watchlist/portfolio usage.
|
||||
try {
|
||||
_db.prepare("INSERT OR IGNORE INTO users (id, email, pw_hash, created_at) VALUES (?, ?, ?, ?)")
|
||||
|
||||
Reference in New Issue
Block a user