cortex/frontend/
metrics.rs

1// Copyright 2015-2025 Deyan Ginev. See the LICENSE
2// file at the top-level directory of this distribution.
3//
4// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
5// This file may not be copied, modified, or distributed
6// except according to those terms.
7
8//! Prometheus **`/metrics`** — operational gauges for scraping (Arm 8 observability).
9//! **Token-gated** via the [`Actor`] guard, so it is not public; Prometheus scrapes it with
10//! `?token=<token>` (the guard also accepts the `X-Cortex-Token` header). Deliberately limited to
11//! **current-state gauges** read on each scrape — connection-pool saturation, background-job
12//! backlog, active admin sessions, registered corpora/services, the dispatcher worker fleet's
13//! size + in-flight backlog, and the **pending-conversion backlog** (`cortex_tasks_todo`, the one
14//! full-table count — bounded ~tens-to-hundreds of ms even at arXiv scale).
15//!
16//! It does **not** instrument the hot paths (no dispatcher changes) and does **not** run the
17//! `/healthz` ZMQ/filesystem probes (those are slow and that endpoint's job). Real-time
18//! **counters** (request rates, per-event tallies via the `metrics` crate) need hot-path
19//! instrumentation and are a follow-on — this gives the operationally-critical saturation/backlog
20//! signals today, cheaply.
21
22use rocket::http::ContentType;
23use rocket::{Route, State};
24
25use crate::backend::DbPool;
26use crate::frontend::actor::Actor;
27use crate::models::{Corpus, Service, Session, Task, WorkerMetadata};
28
29/// Appends one Prometheus gauge (HELP + TYPE + value lines) to `out`.
30fn gauge(out: &mut String, name: &str, help: &str, value: impl std::fmt::Display) {
31  out.push_str(&format!(
32    "# HELP {name} {help}\n# TYPE {name} gauge\n{name} {value}\n"
33  ));
34}
35
36/// `GET /metrics` — Prometheus exposition of current-state gauges. **Token-gated** (the [`Actor`]
37/// guard; scrape with `?token=`). Pool gauges are always emitted (in-memory); DB-derived gauges are
38/// best-effort — on a pool/db hiccup they are omitted (and `cortex_db_reachable` is `0`) rather
39/// than reporting a wrong value.
40#[get("/metrics")]
41pub fn metrics(_caller: Actor, pool: &State<DbPool>) -> (ContentType, String) {
42  let mut out = String::new();
43
44  out.push_str("# HELP cortex_build_info CorTeX build information.\n");
45  out.push_str("# TYPE cortex_build_info gauge\n");
46  out.push_str(&format!(
47    "cortex_build_info{{version=\"{}\"}} 1\n",
48    env!("CARGO_PKG_VERSION")
49  ));
50
51  // Connection pool — in-memory, always available even if the database is unreachable.
52  let state = pool.state();
53  gauge(
54    &mut out,
55    "cortex_pool_max",
56    "Maximum size of the frontend connection pool.",
57    pool.max_size(),
58  );
59  gauge(
60    &mut out,
61    "cortex_pool_connections",
62    "Connections currently established (idle + in use).",
63    state.connections,
64  );
65  gauge(
66    &mut out,
67    "cortex_pool_idle",
68    "Idle, immediately-available pooled connections.",
69    state.idle_connections,
70  );
71  gauge(
72    &mut out,
73    "cortex_pool_in_use",
74    "Pooled connections currently checked out (saturation signal).",
75    state.connections.saturating_sub(state.idle_connections),
76  );
77
78  // DB-derived gauges: best-effort over one checkout. db_reachable doubles as the "trust the gauges
79  // below" flag.
80  match pool.get() {
81    Ok(mut connection) => {
82      gauge(
83        &mut out,
84        "cortex_db_reachable",
85        "1 if the database is reachable, else 0.",
86        1,
87      );
88      gauge(
89        &mut out,
90        "cortex_corpora_total",
91        "Registered corpora.",
92        Corpus::all(&mut connection).map_or(0, |corpora| corpora.len()),
93      );
94      gauge(
95        &mut out,
96        "cortex_services_total",
97        "Registered services.",
98        Service::all(&mut connection).map_or(0, |services| services.len()),
99      );
100      gauge(
101        &mut out,
102        "cortex_jobs_active",
103        "Background jobs currently queued or running.",
104        crate::jobs::list_recent(&mut connection, true, 1000).len(),
105      );
106      gauge(
107        &mut out,
108        "cortex_jobs_failed_recent",
109        "Background jobs that ended in `failed` within the last 24h (rolling window).",
110        crate::jobs::count_recent_with_status(&mut connection, "failed", 24),
111      );
112      gauge(
113        &mut out,
114        "cortex_jobs_interrupted_recent",
115        "Background jobs `interrupted` within the last 24h — stale-reaped (W-4) or restart orphans \
116         (rolling window).",
117        crate::jobs::count_recent_with_status(&mut connection, "interrupted", 24),
118      );
119      gauge(
120        &mut out,
121        "cortex_sessions_active",
122        "Active (unexpired) admin sessions.",
123        Session::active(&mut connection).map_or(0, |sessions| sessions.len()),
124      );
125      if let Ok((workers, in_flight)) = WorkerMetadata::fleet_summary(&mut connection) {
126        gauge(
127          &mut out,
128          "cortex_workers_total",
129          "Workers active in the last ~10 minutes (dispatched or returned a task) — the actively-converting fleet, not all registered rows. 0 when no dispatcher is running.",
130          workers,
131        );
132        gauge(
133          &mut out,
134          // Metric name kept stable for existing scrapers/dashboards. Now a TRUE current gauge:
135          // tasks in-flight at the *active* workers, so it reads 0 on an idle deployment
136          // instead of the old misleading cumulative-lifetime gap (KNOWN_ISSUES P-3).
137          "cortex_workers_in_flight_total",
138          "Tasks in-flight (dispatched, not yet returned) at the active workers — real current in-flight work, 0 on an idle deployment.",
139          in_flight,
140        );
141      }
142      // Pending-conversion backlog: TODO tasks not yet dispatched to any worker — the headline "is
143      // the fleet keeping up?" signal (`workers_in_flight` above is the *leased* backlog; this is
144      // the *unleased* one, otherwise invisible to a scrape). One count over `tasks` — the
145      // costliest gauge here (~tens-to-hundreds of ms at arXiv scale), but operationally
146      // critical; degrades to 0 on a query error, like its siblings.
147      gauge(
148        &mut out,
149        "cortex_tasks_todo",
150        "Tasks awaiting conversion (status TODO, not yet dispatched) — the pending-work backlog.",
151        Task::count_todo(&mut connection),
152      );
153    },
154    Err(_) => gauge(
155      &mut out,
156      "cortex_db_reachable",
157      "1 if the database is reachable, else 0.",
158      0,
159    ),
160  }
161
162  (ContentType::Plain, out)
163}
164
165/// The `/metrics` route.
166pub fn routes() -> Vec<Route> { routes![metrics] }
cortex/frontend/metrics.rs

cortex/frontend/
metrics.rs
