feat(queue): adapter queue overhaul — pause/resume, error capture, scheduling
- Pause/resume persisted in queue_state; drain loop checks isPaused() and skips - Error capture: every failed/backoff attempt logged to queue_errors with the full stack trace; admin queue page expands a failed job to stream it - Retry controls: retryJob(key), retrySource(kind), clearDone(olderThanMs) - Per-source scheduling: queue_schedules table + 30s enqueueDueSchedules loop (seed defaults sec-fetch 24h, yfinance 5min); admin UI lists/adds/deletes - Startup recovery: interrupted in_flight jobs reset to pending on boot - fix(edgar): archive URLs use the filer CIK (accession-number prefix), not the company CIK — resolves Cloudflare 429 that left SEC backfills sparse/empty - Migration: add queue_errors, queue_schedules, queue_state tables plus error/ scheduled_for columns on adapter_queue (idempotent ALTER on startup) Verified: full sec-fetch backfill now succeeds for all watched symbols (NVDA 14,675 institution filings, CIFR 274 insider txns, TSLA 6,011, etc.). 503 backend tests pass. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
55f07e6b42
commit
ca385c1960
@@ -4,6 +4,7 @@
|
||||
import { randomUUID, randomBytes, scryptSync, timingSafeEqual, createHmac } from 'node:crypto';
|
||||
import type { DatabaseSync } from 'node:sqlite';
|
||||
import type { CacheRepository } from '../cache/CacheRepository.ts';
|
||||
import type { AdapterQueue } from '../queue/AdapterQueue.ts';
|
||||
|
||||
export const SESSION_COOKIE = 'iflow_session';
|
||||
export const OAUTH_STATE_COOKIE = 'iflow_oauth_state';
|
||||
@@ -19,6 +20,7 @@ const SESSION_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
|
||||
export interface Context {
|
||||
db: DatabaseSync;
|
||||
cache: CacheRepository;
|
||||
queue: AdapterQueue;
|
||||
resHeaders: Headers; // mutable; procedures append Set-Cookie here (applied to Response by the fetch adapter)
|
||||
userId: string | null; // resolved from the session cookie; null = unauthenticated
|
||||
cookies: Record<string, string>; // parsed request cookies (session + oauth state)
|
||||
@@ -100,12 +102,13 @@ export function verifyPassword(pw: string, stored: string): boolean {
|
||||
}
|
||||
|
||||
// ----- context factory: 1f wires real db+cache; tests inject in-memory -----
|
||||
export function makeCreateContext(opts: { db: DatabaseSync; cache: CacheRepository }) {
|
||||
export function makeCreateContext(opts: { db: DatabaseSync; cache: CacheRepository; queue: AdapterQueue }) {
|
||||
return ({ req, resHeaders }: CreateContextOpts): Context => {
|
||||
const cookies = parseCookies(req);
|
||||
return {
|
||||
db: opts.db,
|
||||
cache: opts.cache,
|
||||
queue: opts.queue,
|
||||
resHeaders,
|
||||
cookies,
|
||||
userId: resolveSessionUserId(opts.db, req),
|
||||
|
||||
Reference in New Issue
Block a user