feat: per-source pause/stop controls for queue admin

Add operator pause/resume/stop/start for individual adapter sources, persisted in queue_state. Pause freezes draining and retains jobs; stop clears pending/backoff jobs and blocks re-enqueue until start. Wires admin tRPC endpoints plus a clean two-row Actions layout in the queue admin page.
2026-08-12 22:39:23 -04:00
parent 0bc7b636e5
commit f7efea2178
7 changed files with 388 additions and 40 deletions
@@ -2,6 +2,7 @@
 import { useEffect, useState, useCallback } from "react";
 import { AdminLayout } from "@/components/AdminLayout";
 import { api } from "@/lib/trpc";
+import { InfoTooltip } from "@/components/InfoTooltip";

 interface QueueItem {
  source: string;
@@ -23,6 +24,8 @@ interface QueueStatus {
  last_error: string | undefined;
  counts: Record<string, number>;
  lastErrors: Array<{ key: string; error: string | null }>;
+  pausedSources?: string[];
+  stoppedSources?: string[];
 }

 interface ErrorLogEntry {
@@ -71,6 +74,18 @@ function formatTimestamp(ts: string | null): string {
  }
 }

+const SOURCE_DESCRIPTIONS: Record<string, string> = {
+  yfinance: "Yahoo Finance market data: quotes, candles, options chains, symbol metadata and fundamentals for every demand symbol.",
+  "sec-fetch": "SEC EDGAR equity filings (13F/SC 13G) + institutional holders for eligible equity tickers.",
+  "sec-sc-fetch": "SEC SC filings (13D/G) for the current SC watchlist.",
+  "sec-tickers": "SEC company-ticker mapping refresh (daily).",
+  "x": "X/Twitter timelines via the local bird CLI: tracked fund-manager posts and x_accounts feeds that back position captures.",
+  fred: "FRED macro series warm-up (yields, CPI, employment) used by the macro regime engine.",
+  "finra-si": "FINRA short-interest biweekly snapshot.",
+  nasdaq: "NASDAQ short-interest feed.",
+  reddit: "Reddit market sentiment posts for the tracked subreddits.",
+};
+
 function msToHuman(ms: number): string {
  if (ms < 60000) return `${Math.round(ms / 1000)}s`;
  if (ms < 3600000) return `${Math.round(ms / 60000)}m`;
@@ -113,6 +128,7 @@ export default function QueuePage() {

  const [resettingSource, setResettingSource] = useState<string | null>(null);
  const [resetResult, setResetResult] = useState<Record<string, number>>({});
+  const [controllingSource, setControllingSource] = useState<{ source: string; action: string } | null>(null);

  const [newScheduleKind, setNewScheduleKind] = useState("sec-fetch");
  const [newScheduleInterval, setNewScheduleInterval] = useState("86400000");
@@ -160,9 +176,11 @@ export default function QueuePage() {
  }, []);

  useEffect(() => {
-    loadData();
-    const interval = setInterval(loadData, 5000);
-    return () => { clearInterval(interval); };
+    // Kick off the initial fetch from a callback so setState happens async,
+    // then poll every 5s (react-hooks/set-state-in-effect).
+    const boot = setTimeout(() => { void loadData(); }, 0);
+    const interval = setInterval(() => { void loadData(); }, 5000);
+    return () => { clearTimeout(boot); clearInterval(interval); };
  }, [loadData]);

  const handlePause = async () => {
@@ -297,6 +315,36 @@ export default function QueuePage() {
    }
  };

+  const handleSourceControl = async (source: string, action: "pause" | "resume" | "stop" | "start") => {
+    setControllingSource({ source, action });
+    setError(null);
+    try {
+      switch (action) {
+        case "pause":
+          await api.admin.queueSourcePause(source);
+          setSuccess(`${source} paused — jobs frozen, queue retained`);
+          break;
+        case "resume":
+          await api.admin.queueSourceResume(source);
+          setSuccess(`${source} resumed`);
+          break;
+        case "stop":
+          const stopRes = await api.admin.queueSourceStop(source);
+          setSuccess(`${source} stopped — ${stopRes.cleared} job(s) cleared, re-enqueue blocked`);
+          break;
+        case "start":
+          await api.admin.queueSourceStart(source);
+          setSuccess(`${source} started — re-queuing current demand`);
+          break;
+      }
+      await loadData();
+    } catch (e) {
+      setError(e instanceof Error ? e.message : `Failed to ${action} ${source}`);
+    } finally {
+      setControllingSource(null);
+    }
+  };
+
  const handleRetryJob = async (key: string) => {
    try {
      await api.admin.queueRetryJob(key);
@@ -379,6 +427,10 @@ export default function QueuePage() {

  const sortedSources = Object.entries(sources).sort(([a], [b]) => a.localeCompare(b));

+  const sourceControl: Record<string, "running" | "paused" | "stopped"> = {};
+  for (const s of status?.stoppedSources ?? []) sourceControl[s] = "stopped";
+  for (const s of status?.pausedSources ?? []) if (!sourceControl[s]) sourceControl[s] = "paused";
+
  if (loading) {
    return <AdminLayout><Spinner /></AdminLayout>;
  }
@@ -462,18 +514,23 @@ export default function QueuePage() {
              <div className="flex items-center gap-4 px-4 pt-3 border-b border-[#1a1b28]">
                <span className="text-[10px] uppercase tracking-wide text-fg-faint flex items-center gap-1.5">
                  <span className="h-1.5 w-1.5 rounded-full bg-accent" /> Pending
+                  <InfoTooltip>Jobs waiting to be picked up by the next drain cycle.</InfoTooltip>
                </span>
                <span className="text-[10px] uppercase tracking-wide text-fg-faint flex items-center gap-1.5">
                  <span className="h-1.5 w-1.5 rounded-full bg-[#34d399]" /> In-flight
+                  <InfoTooltip>Jobs currently being fetched. Per source they run one at a time to respect vendor rate limits.</InfoTooltip>
                </span>
                <span className="text-[10px] uppercase tracking-wide text-fg-faint flex items-center gap-1.5">
                  <span className="h-1.5 w-1.5 rounded-full bg-fg-muted/40" /> Done
+                  <InfoTooltip>Jobs completed successfully and cached. Cleared via the Clear Done button (24h).</InfoTooltip>
                </span>
                <span className="text-[10px] uppercase tracking-wide text-fg-faint flex items-center gap-1.5">
                  <span className="h-1.5 w-1.5 rounded-full bg-[#fb923c]" /> Backoff
+                  <InfoTooltip>A vendor rate-limit or cool-down. Jobs stall and retry automatically once the window passes.</InfoTooltip>
                </span>
                <span className="text-[10px] uppercase tracking-wide text-fg-faint flex items-center gap-1.5">
                  <span className="h-1.5 w-1.5 rounded-full bg-danger" /> Failed
+                  <InfoTooltip>Permanent failures (quarantine / max retries exceeded). See the Failed Jobs log below for details and manual retry.</InfoTooltip>
                </span>
              </div>

@@ -487,20 +544,42 @@ export default function QueuePage() {
              <table className="w-full text-xs mt-3">
                <thead>
                  <tr className="border-b border-line-strong text-left text-fg-faint uppercase tracking-wide text-[10px]">
-                    <th className="pb-2 pr-4 pl-4 font-medium w-48">Source</th>
-                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-accent">Pending</span></th>
-                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-up">In-flight</span></th>
-                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-fg-muted/60">Done</span></th>
-                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-[#fb923c]">Backoff</span></th>
-                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-danger">Failed</span></th>
-                    <th className="pb-2 pr-4 font-medium">Last Attempt</th>
-                    <th className="pb-2 font-medium w-48 text-center">Actions</th>
+                    <th className="pb-2 pr-4 pl-4 font-medium w-48">
+                      <span className="inline-flex items-center gap-1">
+                        Source
+                        <InfoTooltip>Vendor data plane. Pause freezes draining but keeps jobs; Stop clears pending jobs and blocks enqueue until Start.</InfoTooltip>
+                      </span>
+                    </th>
+                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-accent inline-flex items-center gap-1">Pending<InfoTooltip>Jobs awaiting the next drain cycle.</InfoTooltip></span></th>
+                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-up inline-flex items-center gap-1">In-flight<InfoTooltip>Jobs actively being fetched right now.</InfoTooltip></span></th>
+                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-fg-muted/60 inline-flex items-center gap-1">Done<InfoTooltip>Completed and cached successfully.</InfoTooltip></span></th>
+                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-[#fb923c] inline-flex items-center gap-1">Backoff<InfoTooltip>Rate-limited; auto-retries after cool-down.</InfoTooltip></span></th>
+                    <th className="pb-2 pr-4 font-medium text-right"><span className="text-danger inline-flex items-center gap-1">Failed<InfoTooltip>Permanent failures requiring manual retry.</InfoTooltip></span></th>
+                    <th className="pb-2 pr-4 font-medium"><span className="inline-flex items-center gap-1">Last Attempt<InfoTooltip>Most recent time any job for this source was attempted.</InfoTooltip></span></th>
+                    <th className="pb-2 font-medium w-72 text-center">
+                      <span className="inline-flex items-center gap-1 justify-center">
+                        Actions
+                        <InfoTooltip>Per-source controls: Pause/Resume freeze draining; Stop clears pending jobs and blocks re-enqueue; Start re-queues demand. Retry forces backoff/failed jobs back to pending.</InfoTooltip>
+                      </span>
+                    </th>
                  </tr>
                </thead>
                <tbody className="divide-y divide-[#1a1b28]">
-                  {sortedSources.map(([source, agg]) => (
-                    <tr key={source} className={`hover:bg-surface/50 transition-colors ${agg.hasBackoff ? "bg-orange-950/10" : ""}`}>
-                      <td className="py-2.5 pr-4 pl-4 font-mono text-fg">{source}</td>
+                  {sortedSources.map(([source, agg]) => {
+                    const ctrl = sourceControl[source] ?? "running";
+                    return (
+                    <tr key={source} className={`hover:bg-surface/50 transition-colors ${agg.hasBackoff ? "bg-orange-950/10" : ""} ${ctrl === "stopped" ? "bg-red-950/10" : ""} ${ctrl === "paused" ? "bg-orange-950/5" : ""}`}>
+                      <td className="py-2.5 pr-4 pl-4 font-mono text-fg">
+                        <span className="inline-flex items-center gap-1.5">
+                          {source}
+                          <InfoTooltip>{SOURCE_DESCRIPTIONS[source] ?? `Adapter data source: ${source}.`}</InfoTooltip>
+                        </span>
+                        {ctrl !== "running" && (
+                          <span className={`ml-2 inline-block rounded px-1.5 py-0.5 text-[9px] font-semibold uppercase tracking-wide ${ctrl === "stopped" ? "bg-red-950/50 text-danger" : "bg-orange-950/50 text-[#fb923c]"}`}>
+                            {ctrl}
+                          </span>
+                        )}
+                      </td>
                      <td className={`py-2.5 pr-4 font-mono text-right ${agg.pending > 0 ? "text-accent" : "text-fg-faint"}`}>{agg.pending}</td>
                      <td className={`py-2.5 pr-4 font-mono text-right ${agg.inFlight > 0 ? "text-up" : "text-fg-faint"}`}>{agg.inFlight}</td>
                      <td className="py-2.5 pr-4 font-mono text-right text-fg-muted">{agg.done}</td>
@@ -508,32 +587,81 @@ export default function QueuePage() {
                      <td className={`py-2.5 pr-4 font-mono text-right ${agg.failed > 0 ? "text-danger" : "text-fg-faint"}`}>{agg.failed}</td>
                      <td className="py-2.5 pr-4 text-[10px] text-fg-faint whitespace-nowrap font-mono">{formatTimestamp(agg.lastAttempt)}</td>
                      <td className="py-2.5 text-center">
-                        <div className="flex items-center justify-center gap-1.5">
-                          {agg.hasBackoff && (
-                            <button
-                              onClick={() => handleResetBackoff(source)}
-                              disabled={resettingSource === source}
-                              className="inline-flex items-center gap-1 rounded border border-orange-500/40 bg-orange-950/30 px-2 py-1 text-[10px] font-medium text-[#fb923c] hover:bg-orange-950/60 transition-colors disabled:opacity-40 disabled:cursor-not-allowed"
-                            >
-                              {resettingSource === source ? (
-                                <span className="h-3 w-3 border border-[#fb923c] border-t-transparent rounded-full animate-spin" />
-                              ) : null}
-                              Retry
-                            </button>
-                          )}
-                          {agg.failed > 0 && (
-                            <button
-                              onClick={() => handleResetBackoff(source)}
-                              disabled={resettingSource === source}
-                              className="inline-flex items-center gap-1 rounded border border-red-500/40 bg-red-950/30 px-2 py-1 text-[10px] font-medium text-danger hover:bg-red-950/60 transition-colors disabled:opacity-40 disabled:cursor-not-allowed"
-                            >
-                              Retry All
-                            </button>
+                        <div className="flex flex-col items-center gap-1.5 min-w-[140px]">
+                          <div className="flex items-center justify-center gap-1.5">
+                            {ctrl === "stopped" ? (
+                              <button
+                                onClick={() => handleSourceControl(source, "start")}
+                                disabled={controllingSource?.source === source}
+                                className="inline-flex items-center justify-center gap-1 rounded border border-green-500/40 bg-green-950/30 px-2 py-1 text-[10px] font-medium text-up hover:bg-green-950/60 transition-colors disabled:opacity-40 disabled:cursor-not-allowed min-w-[88px]"
+                              >
+                                {controllingSource?.source === source && controllingSource.action === "start" ? (
+                                  <span className="h-3 w-3 border border-up border-t-transparent rounded-full animate-spin" />
+                                ) : null}
+                                Start
+                              </button>
+                            ) : (
+                              <>
+                                <button
+                                  onClick={() => handleSourceControl(source, ctrl === "paused" ? "resume" : "pause")}
+                                  disabled={controllingSource?.source === source}
+                                  className={`inline-flex items-center justify-center gap-1 rounded border px-2 py-1 text-[10px] font-medium transition-colors disabled:opacity-40 disabled:cursor-not-allowed min-w-[64px] ${
+                                    ctrl === "paused"
+                                      ? "border-green-500/40 bg-green-950/30 text-up hover:bg-green-950/60"
+                                      : "border-orange-500/40 bg-orange-950/30 text-[#fb923c] hover:bg-orange-950/60"
+                                  }`}
+                                >
+                                  {controllingSource?.source === source && (controllingSource.action === "pause" || controllingSource.action === "resume") ? (
+                                    <span className="h-3 w-3 border border-current border-t-transparent rounded-full animate-spin" />
+                                  ) : null}
+                                  {ctrl === "paused" ? "Resume" : "Pause"}
+                                </button>
+                                <button
+                                  onClick={() => handleSourceControl(source, "stop")}
+                                  disabled={controllingSource?.source === source}
+                                  className="inline-flex items-center justify-center gap-1 rounded border border-red-500/40 bg-red-950/30 px-2 py-1 text-[10px] font-medium text-danger hover:bg-red-950/60 transition-colors disabled:opacity-40 disabled:cursor-not-allowed min-w-[64px]"
+                                  title="Clear pending/backoff jobs and block re-enqueue until Start"
+                                >
+                                  {controllingSource?.source === source && controllingSource.action === "stop" ? (
+                                    <span className="h-3 w-3 border border-danger border-t-transparent rounded-full animate-spin" />
+                                  ) : null}
+                                  Stop
+                                </button>
+                              </>
+                            )}
+                          </div>
+                          {(agg.hasBackoff || agg.failed > 0) && (
+                            <div className="flex items-center justify-center gap-1.5">
+                              {agg.hasBackoff && (
+                                <button
+                                  onClick={() => handleResetBackoff(source)}
+                                  disabled={resettingSource === source}
+                                  className="inline-flex items-center justify-center gap-1 rounded border border-orange-500/40 bg-orange-950/30 px-2 py-1 text-[10px] font-medium text-[#fb923c] hover:bg-orange-950/60 transition-colors disabled:opacity-40 disabled:cursor-not-allowed min-w-[64px]"
+                                  title="Requeue backoff jobs so they retry immediately"
+                                >
+                                  {resettingSource === source ? (
+                                    <span className="h-3 w-3 border border-[#fb923c] border-t-transparent rounded-full animate-spin" />
+                                  ) : null}
+                                  Retry
+                                </button>
+                              )}
+                              {agg.failed > 0 && (
+                                <button
+                                  onClick={() => handleResetBackoff(source)}
+                                  disabled={resettingSource === source}
+                                  className="inline-flex items-center justify-center gap-1 rounded border border-red-500/40 bg-red-950/30 px-2 py-1 text-[10px] font-medium text-danger hover:bg-red-950/60 transition-colors disabled:opacity-40 disabled:cursor-not-allowed min-w-[64px]"
+                                  title="Requeue this source's failed jobs"
+                                >
+                                  Retry All
+                                </button>
+                              )}
+                            </div>
                          )}
                        </div>
                      </td>
                    </tr>
-                  ))}
+                    );
+                  })}
                </tbody>
              </table>

@@ -31,7 +31,7 @@ export function InfoTooltip({ children }: InfoTooltipProps) {
        ?
      </button>
      {open && (
-        <div className="absolute z-50 left-0 top-5 w-72 p-3 rounded-lg border border-line bg-surface-raised shadow-lg text-xs text-fg-muted leading-relaxed">
+        <div className="absolute z-50 left-0 top-5 w-72 p-3 rounded-lg border border-line bg-surface-raised shadow-lg text-xs text-fg-muted leading-relaxed normal-case">
          {children}
        </div>
      )}
@@ -1305,6 +1305,10 @@ export const api = {
    queueLogs: (key: string, limit: number) => trpcQuery<ErrorLogEntry[]>("admin.queueLogs", { key, limit }),
    queueRetryJob: (key: string) => trpcMutate<{ ok: boolean }>("admin.queueRetryJob", { key }),
    queueRetrySource: (sourceKind: string) => trpcMutate<{ cleared: number }>("admin.queueRetrySource", { sourceKind }),
+    queueSourcePause: (source: string) => trpcMutate<{ ok: boolean; source: string }>("admin.queueSourcePause", { source }),
+    queueSourceResume: (source: string) => trpcMutate<{ ok: boolean; source: string }>("admin.queueSourceResume", { source }),
+    queueSourceStop: (source: string) => trpcMutate<{ ok: boolean; source: string; cleared: number }>("admin.queueSourceStop", { source }),
+    queueSourceStart: (source: string) => trpcMutate<{ ok: boolean; source: string }>("admin.queueSourceStart", { source }),
    auditLog: (limit = 50, offset = 0, actor?: string, action?: string) =>
      trpcQuery<Array<{ id: string; actor: string; action: string; target: string | null; ts: string; meta: string | null }>>("admin.auditLog", { limit, offset, ...(actor ? { actor } : {}), ...(action ? { action } : {}) }),
    queueLint: (symbol: string, kind: 'sec-lint-holders' | 'sec-lint-insiders') =>
@@ -1726,6 +1730,10 @@ export interface QueueStatus {
  lastErrors: Array<{ key: string; error: string | null }>;
  /** Active vendor cool-downs after 429 (ADR-0009). */
  sourceCooldowns?: SourceCooldown[];
+  /** Operator-paused sources (drain frozen, jobs retained). */
+  pausedSources?: string[];
+  /** Operator-stopped sources (drain frozen, jobs cleared, enqueue blocked). */
+  stoppedSources?: string[];
 }

 export interface ScheduleEntry {