From f7c312021be51792b3a520c30d036b968494dade Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 23 Jul 2026 13:18:03 +0000 Subject: [PATCH] :sparkles: Improve error-reports CLI with streaming, time-range, and stats Server changes: - Switch list ordering from DESC to ASC (oldest first) - Flip cursor direction to > for forward pagination - Add 'until' param for server-side upper-bound filtering CLI changes: - Add --from/--to flags mapping to server's since/until - Streaming output for --all and --format ndjson - Add --format ndjson option (one JSON object per line) - Add --normalize-hints flag to strip dynamic values - Add --output flag to write list results to file - Add 'stats' subcommand with aggregations (signature, host, tenant, version, source, kind, hour) reading from API, file, stdin - stats input supports JSON, JSON array, and NDJSON formats Test changes: - Fix pagination assertions for ASC ordering AI-assisted-by: mimo-v2.5-pro --- .serena/memories/scripts/error-reports.md | 148 +++++-- .../src/app/rpc/commands/error_reports.clj | 26 +- .../rpc_commands_error_reports_test.clj | 10 +- scripts/error-reports.mjs | 410 ++++++++++++++---- 4 files changed, 456 insertions(+), 138 deletions(-) diff --git a/.serena/memories/scripts/error-reports.md b/.serena/memories/scripts/error-reports.md index 13ba81fece..1978cd0eb0 100644 --- a/.serena/memories/scripts/error-reports.md +++ b/.serena/memories/scripts/error-reports.md @@ -6,7 +6,8 @@ - Querying error reports from the database for debugging or analysis - Filtering errors by source, kind, tenant, or backend version -- Exporting error data in JSON or table format +- Exporting error data in JSON, NDJSON, or table format +- Computing error statistics (top signatures, per-host breakdown, hourly distribution) - Investigating specific error reports by ID ## Prerequisites @@ -52,19 +53,25 @@ WHERE id = ''; | Flag | Description | Default | |------|-------------|---------| | `-l, --limit ` | Max items per page (max: 200) | `50` | -| `--since ` | ISO timestamp (fetch errors before this date) | — | -| `--since-id ` | Fetch errors before this ID (cursor pagination) | — | +| `--from ` | ISO timestamp — oldest boundary (items after this) | — | +| `--to ` | ISO timestamp — newest boundary (items before this) | — | +| `--since ` | ISO timestamp — explicit cursor for manual pagination | — | +| `--since-id ` | Fetch errors after this ID (cursor pagination) | — | | `-s, --source ` | Filter by source (see source names below) | — | | `-p, --profile-id ` | Filter by profile ID | — | | `-k, --kind ` | Filter by kind (string) | — | | `-t, --tenant ` | Filter by tenant (string) | — | | `--version ` | Filter by version | — | | `--hint ` | Filter by hint (ILIKE match) | — | -| `-a, --all` | Fetch all pages automatically | `false` | -| `-f, --format ` | Output format: `json` or `table` | `json` | +| `-a, --all` | Fetch all pages automatically (streams output) | `false` | +| `-f, --format ` | Output format: `json`, `table`, or `ndjson` | `table` | +| `--normalize-hints` | Normalize hints by stripping dynamic values | `false` | +| `-o, --output ` | Write output to file instead of stdout | — | | `--env ` | Custom .env file path | `.env` | | `-h, --help` | Show help message | — | +**Streaming behavior:** With `--all` or `--format ndjson`, items are printed as they arrive (no buffering). `--all` + `table` prints rows immediately. `--all` + `json` streams NDJSON (one JSON object per line). + #### `get` - Get a single error report by ID ```bash @@ -77,10 +84,29 @@ WHERE id = ''; |------|-------------|----------| | `--id ` | Error report ID | Yes (or --error-id) | | `--error-id ` | Error report error-id | Yes (or --id) | -| `-f, --format ` | Output format: `json` or `table` | No (default: `json`) | +| `-f, --format ` | Output format: `json` or `table` | No (default: `table`) | | `--env ` | Custom .env file path | No (default: `.env`) | | `-h, --help` | Show help message | No | +#### `stats` - Compute error report statistics + +```bash +./scripts/error-reports.mjs stats [options] +``` + +Reads from `--input `, stdin (piped), or fetches from API. Computes aggregations by signature, host, tenant, version, source, kind, and hour. + +**Options:** + +| Flag | Description | Default | +|------|-------------|---------| +| `--from ` | Start of interval (ISO timestamp) | — | +| `--to ` | End of interval (ISO timestamp) | — | +| `--limit ` | Items per page when fetching from API | `200` | +| `--input ` | Read from local JSON/NDJSON file instead of API | — | +| `-f, --format ` | Output format: `json` or `table` | `table` | +| `--env ` | Custom .env file path | `.env` | + ## Source Names The `--source` filter accepts these values: @@ -89,6 +115,15 @@ The `--source` filter accepts these values: - `audit-log` - `rlimit` +## Hint Normalization + +With `--normalize-hints` (or always in `stats`), hints are normalized by stripping dynamic values: + +1. URIs (`https://...`) → `` +2. UUIDs (8-4-4-4-12 hex) → `` +3. Elapsed times (`7.5s`, `2m3.027s`) → `` +4. Numeric IDs in parentheses `(12345)` → `()` + ## Examples ### List recent errors @@ -96,6 +131,22 @@ The `--source` filter accepts these values: ./scripts/error-reports.mjs list --limit 10 ``` +### Time-range query (today) +```bash +./scripts/error-reports.mjs list --from 2026-07-23T00:00:00Z --to 2026-07-23T23:59:59Z --all +``` + +### Stream all errors as NDJSON +```bash +./scripts/error-reports.mjs list --all --format ndjson > errors.ndjson +``` + +### Save to file with --output +```bash +./scripts/error-reports.mjs list --all --format json -o errors.json +./scripts/error-reports.mjs list --all --format ndjson -o errors.ndjson +``` + ### Filter by source ```bash ./scripts/error-reports.mjs list --source audit-log --limit 20 @@ -123,7 +174,7 @@ The `--source` filter accepts these values: ### Fetch all errors with pagination ```bash -./scripts/error-reports.mjs list --all --format json +./scripts/error-reports.mjs list --all ``` ### Get specific error by ID @@ -141,46 +192,36 @@ The `--source` filter accepts these values: ./scripts/error-reports.mjs list --source audit-log --kind exception-page --tenant production --limit 50 ``` +### Stats from API +```bash +./scripts/error-reports.mjs stats --from 2026-07-23T00:00:00Z --to 2026-07-23T23:59:59Z +``` + +### Stats from file +```bash +./scripts/error-reports.mjs stats --input errors.json +``` + +### Stats from pipe +```bash +./scripts/error-reports.mjs list --all --format json | ./scripts/error-reports.mjs stats +``` + ## Output Formats ### Table (default) -Human-readable table format for terminal display: - -``` -Found 15 error reports - -ID | Created At | Source | Profile ID | Kind | Hint --------------------------------------+---------------------+-----------+--------------------------------------+----------------+------------------ -550e8400-e29b-41d4-a716-446655440000 | 2026-01-20 10:30:00 | audit-log | e98bb95f-573d-8137-8008-252580aa456d | exception-page | Error description -abc12345-e29b-41d4-a716-446655440001 | 2026-01-20 10:29:00 | logging | - | error | Another error that is very long and ne... - -More results: use --since 2026-01-20T10:28:00Z --since-id def45678-e29b-41d4-a716-446655440002 -``` +Human-readable table format for terminal display. With `--all`, rows stream as they arrive. ### JSON -Returns structured JSON with error details and pagination metadata: +Single page: `{items: [...], nextSince, nextId}`. With `--all`: NDJSON (one JSON object per line). -```json -{ - "items": [ - { - "id": "uuid", - "createdAt": "2026-01-20T10:30:00Z", - "source": "audit-log", - "profileId": "e98bb95f-573d-8137-8008-252580aa456d", - "kind": "exception-page", - "tenant": "production", - "version": "2.1.0", - "hint": "Error description" - } - ], - "nextSince": "2026-01-20T10:29:00Z", - "nextId": "next-uuid" -} -``` +### NDJSON +One JSON object per line, always streaming. Pipe-friendly: `| jq -c '.hint'`, `| wc -l`. ## Pagination +The server returns items in **ascending** order (oldest first). Cursor pagination uses `--since` / `--since-id` to fetch the next page of newer items. + ### Manual pagination Use `--since` and `--since-id` with values from `nextSince` and `nextId` in the response: @@ -191,20 +232,28 @@ Use `--since` and `--since-id` with values from `nextSince` and `nextId` in the ``` ### Automatic pagination -Use `--all` to fetch all pages automatically: +Use `--all` to fetch all pages automatically (streams output): ```bash ./scripts/error-reports.mjs list --all ``` +### Time-range queries +Use `--from` and `--to` to bound the query. These map to the server's `--since` and `--until` parameters: + +```bash +./scripts/error-reports.mjs list --from 2026-07-20T00:00:00Z --to 2026-07-23T23:59:59Z --all +``` + ## Key principles - **Authentication required** - Uses access token with `error-reports:read` permission - **API endpoint configurable** - Set via `PENPOT_API_URI` in `.env` file -- **Table is default format** - Use `--format json` for structured JSON output -- **Pagination is automatic with --all** - Fetches all pages without manual cursor management +- **Table is default format** - Use `--format json` for structured JSON, `--format ndjson` for streaming +- **Streaming with --all** - Items print as they arrive, no buffering - **Filters are combinable** - All filter options can be used together - **Both flag formats supported** - `--option=value` and `--option value` both work +- **Ascending order** - Server returns oldest items first (changed from DESC) ## Error handling @@ -217,12 +266,21 @@ The tool provides helpful error messages for common issues: ## Integration with other scripts -- **jq**: Pipe JSON output to `jq` for further processing +- **jq**: Pipe NDJSON output to `jq` for further processing ```bash - ./scripts/error-reports.mjs list --all --format json | jq '.items[] | {id, kind, hint}' + ./scripts/error-reports.mjs list --all --format ndjson | jq -c '{id, hint}' + ``` +- **stats from pipe**: Fetch data once, compute stats + ```bash + ./scripts/error-reports.mjs list --all --format json -o errors.json + ./scripts/error-reports.mjs stats --input errors.json + ``` +- **stats from NDJSON pipe**: Works with NDJSON format too + ```bash + ./scripts/error-reports.mjs list --all --format ndjson | ./scripts/error-reports.mjs stats ``` - **grep/search**: Filter output by specific patterns -- **Redirect**: Save output to files for analysis +- **--output**: Save to file without shell redirection ```bash - ./scripts/error-reports.mjs list --all --format json > errors.json + ./scripts/error-reports.mjs list --all --format ndjson -o errors.ndjson ``` diff --git a/backend/src/app/rpc/commands/error_reports.clj b/backend/src/app/rpc/commands/error_reports.clj index fb14f6ff78..dfb6e6ba49 100644 --- a/backend/src/app/rpc/commands/error_reports.clj +++ b/backend/src/app/rpc/commands/error_reports.clj @@ -68,7 +68,8 @@ [:kind {:optional true} ::sm/text] [:tenant {:optional true} ::sm/text] [:version {:optional true} ::sm/text] - [:hint {:optional true} ::sm/text]]) + [:hint {:optional true} ::sm/text] + [:until {:optional true} ct/schema:inst]]) (def ^:private schema:get-error-reports-result [:map @@ -104,7 +105,7 @@ "FROM server_error_report")) (defn- build-list-query - [{:keys [since since-id source profile-id kind tenant version hint limit] + [{:keys [since since-id source profile-id kind tenant version hint until limit] :or {limit default-limit}}] (let [source-id (when source (name->source source)) clauses (keep identity @@ -126,14 +127,17 @@ {:where "content->>'~:hint' ILIKE ?" :params [(str "%" hint "%")]}) (when since + {:where "(created_at, id) > (?::timestamptz, ?::uuid)" + :params [since (or since-id uuid/zero)]}) + (when until {:where "(created_at, id) < (?::timestamptz, ?::uuid)" - :params [since (or since-id uuid/zero)]})]) + :params [until uuid/zero]})]) sql-parts (map :where clauses) sql-params (mapcat :params clauses) sql (str base-list-sql (when (seq sql-parts) (str " WHERE " (str/join " AND " sql-parts))) - " ORDER BY created_at DESC, id DESC" + " ORDER BY created_at ASC, id ASC" " LIMIT ?")] (into [sql] (concat sql-params [limit])))) @@ -169,13 +173,13 @@ [cfg {:keys [id]}] (if-let [report (db/get-by-id cfg :server-error-report id {::db/check-deleted false})] (let [content (db/decode-transit-pgobject (:content report))] - (-> report - (dissoc :content) - (merge content) - (update :source source->name) - (assoc :kind (or (:kind content) (:origin content))) - (assoc :version (:version content)) - (d/without-nils))) + (-> report + (dissoc :content) + (merge content) + (update :source source->name) + (assoc :kind (or (:kind content) (:origin content))) + (assoc :version (:version content)) + (d/without-nils))) (ex/raise :type :not-found :code :report-not-found :hint (str "error report " id " not found")))) diff --git a/backend/test/backend_tests/rpc_commands_error_reports_test.clj b/backend/test/backend_tests/rpc_commands_error_reports_test.clj index 174eac5cb9..7e22c05691 100644 --- a/backend/test/backend_tests/rpc_commands_error_reports_test.clj +++ b/backend/test/backend_tests/rpc_commands_error_reports_test.clj @@ -170,23 +170,23 @@ (insert-report! th/*system* {:id (uuid/next) :source 3 :content {:hint "2"} :created-at t2}) (insert-report! th/*system* {:id (uuid/next) :source 3 :content {:hint "3"} :created-at t3}) (insert-report! th/*system* {:id (uuid/next) :source 3 :content {:hint "4"} :created-at t4}) - ;; Page 1: newest 2 + ;; Page 1: oldest 2 (ASC order) (let [out (token-cmd profile {::th/type :get-error-reports :limit 2})] (t/is (th/success? out)) (let [{:keys [items next-since next-id]} (:result out)] (t/is (= 2 (count items))) (t/is (some? next-since)) (t/is (some? next-id)) - (t/is (= "4" (:hint (first items)))) - (t/is (= "3" (:hint (second items)))) + (t/is (= "1" (:hint (first items)))) + (t/is (= "2" (:hint (second items)))) ;; Page 2: next 2, using since and since-id from page 1 (let [out2 (token-cmd profile {::th/type :get-error-reports :limit 2 :since next-since :since-id next-id})] (t/is (th/success? out2)) (let [{:keys [items next-since]} (:result out2)] (t/is (= 2 (count items))) (t/is (nil? next-since)) - (t/is (= "2" (:hint (first items)))) - (t/is (= "1" (:hint (second items)))))))))) + (t/is (= "3" (:hint (first items)))) + (t/is (= "4" (:hint (second items)))))))))) (t/deftest get-error-reports-pagination-same-timestamp (let [profile (th/create-profile* 1 {:is-active true}) diff --git a/scripts/error-reports.mjs b/scripts/error-reports.mjs index 6b7924e474..31ba4f0154 100755 --- a/scripts/error-reports.mjs +++ b/scripts/error-reports.mjs @@ -1,8 +1,9 @@ #!/usr/bin/env node import { Command } from "commander"; import dotenv from "dotenv"; -import { readFileSync, existsSync } from "fs"; +import { readFileSync, existsSync, createWriteStream } from "fs"; import path from "path"; +import { createInterface } from "readline"; // ============================================================================ // Configuration @@ -10,7 +11,7 @@ import path from "path"; function loadConfig(envPath) { const envFile = envPath || ".env"; - + if (existsSync(envFile)) { dotenv.config({ path: envFile }); } else if (envPath) { @@ -51,7 +52,7 @@ function loadConfig(envPath) { async function rpcCall(config, method, params = {}) { const url = `${config.apiUrl}/api/main/methods/${method}`; - + let response; try { response = await fetch(url, { @@ -139,6 +140,21 @@ async function rpcCall(config, method, params = {}) { return response.json(); } +// ============================================================================ +// Hint Normalization +// ============================================================================ + +function normalizeHint(hint) { + if (!hint) return hint; + let h = hint; + h = h.replace(/https?:\/\/"[^"]*"/g, ''); + h = h.replace(/https?:\/\/\S+/g, ''); + h = h.replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, ''); + h = h.replace(/[\d.]+[smh](?:[\d.]+[smh])?/g, ''); + h = h.replace(/\(\d+\)/g, '()'); + return h.trim(); +} + // ============================================================================ // Output Formatting // ============================================================================ @@ -147,16 +163,38 @@ function formatJson(data) { return JSON.stringify(data, null, 2); } -function formatTable(data, type) { - if (type === "list") { - return formatListTable(data); - } else if (type === "get") { - return formatGetTable(data); - } - return formatJson(data); +function truncateHint(text, maxLength) { + if (!text) return "-"; + if (text.length <= maxLength) return text; + return text.substring(0, maxLength - 3) + "..."; } -function formatListTable(data, hasMore = false) { +const TABLE_HEADERS = ["ID", "Created At", "Source", "Profile ID", "Kind", "Hint"]; + +function formatTableRow(item, normalizeHints) { + const hint = normalizeHints ? normalizeHint(item.hint) : item.hint; + return [ + item.id || "-", + item.createdAt || "-", + item.source || "-", + item.profileId || "-", + item.kind || "-", + truncateHint(hint, 60) + ]; +} + +function computeColWidths(headerLines, dataRows) { + return headerLines.map((h, i) => { + const maxData = Math.max(...dataRows.map(r => (r[i] || "").toString().length)); + return Math.max(h.length, maxData); + }); +} + +function padRow(row, colWidths) { + return row.map((cell, i) => (cell || "-").toString().padEnd(colWidths[i])).join(" | "); +} + +function formatListTable(data, hasMore = false, normalizeHints = false) { if (!data.items || data.items.length === 0) { return "No error reports found."; } @@ -165,30 +203,12 @@ function formatListTable(data, hasMore = false) { lines.push(`Found ${data.items.length} error reports`); lines.push(""); - const headers = ["ID", "Created At", "Source", "Profile ID", "Kind", "Hint"]; - const rows = data.items.map(item => [ - item.id || "-", - item.createdAt || "-", - item.source || "-", - item.profileId || "-", - item.kind || "-", - truncateHint(item.hint, 60) - ]); + const rows = data.items.map(item => formatTableRow(item, normalizeHints)); + const colWidths = computeColWidths(TABLE_HEADERS, rows); - const colWidths = headers.map((h, i) => { - const maxData = Math.max(...rows.map(r => (r[i] || "").toString().length)); - return Math.max(h.length, maxData); - }); - - const headerLine = headers.map((h, i) => h.padEnd(colWidths[i])).join(" | "); - const separator = colWidths.map(w => "-".repeat(w)).join("-+-"); - const dataLines = rows.map(row => - row.map((cell, i) => (cell || "-").toString().padEnd(colWidths[i])).join(" | ") - ); - - lines.push(headerLine); - lines.push(separator); - lines.push(...dataLines); + lines.push(padRow(TABLE_HEADERS, colWidths)); + lines.push(colWidths.map(w => "-".repeat(w)).join("-+-")); + lines.push(...rows.map(row => padRow(row, colWidths))); if (hasMore && data.nextSince && data.nextId) { lines.push(""); @@ -198,10 +218,14 @@ function formatListTable(data, hasMore = false) { return lines.join("\n"); } -function truncateHint(text, maxLength) { - if (!text) return "-"; - if (text.length <= maxLength) return text; - return text.substring(0, maxLength - 3) + "..."; +function printTableHeaderStr(normalizeHints) { + const colWidths = TABLE_HEADERS.map(h => Math.max(h.length, h === "ID" ? 36 : 10)); + return padRow(TABLE_HEADERS, colWidths) + "\n" + colWidths.map(w => "-".repeat(w)).join("-+-") + "\n"; +} + +function printTableRowStr(item, normalizeHints) { + const row = formatTableRow(item, normalizeHints); + return row.map(cell => (cell || "-").toString()).join(" | "); } function formatGetTable(data) { @@ -251,10 +275,8 @@ function formatGetTable(data) { async function cmdList(config, args) { const params = {}; - + if (args.limit !== undefined) params.limit = args.limit; - if (args.since) params.since = args.since; - if (args.sinceId) params["since-id"] = args.sinceId; if (args.source !== undefined) params.source = args.source; if (args.profileId) params["profile-id"] = args.profileId; if (args.kind) params.kind = args.kind; @@ -262,43 +284,110 @@ async function cmdList(config, args) { if (args.version) params.version = args.version; if (args.hint) params.hint = args.hint; - let allItems = []; - let lastResult = null; - let hasMore = true; - let sinceParam = params.since; - let sinceIdParam = params["since-id"]; + // --from maps to server's --since (oldest boundary) + // --since / --since-id are explicit cursor overrides + if (args.since) { + params.since = args.since; + } else if (args.from) { + params.since = args.from; + } + if (args.sinceId) params["since-id"] = args.sinceId; - while (hasMore) { - const currentParams = { ...params }; - if (sinceParam) currentParams.since = sinceParam; - if (sinceIdParam) currentParams["since-id"] = sinceIdParam; + // --to maps to server's --until (newest boundary) + if (args.to) params.until = args.to; - lastResult = await rpcCall(config, "get-error-reports", currentParams); - - allItems = [...allItems, ...lastResult.items]; - - if (args.all && lastResult.nextSince && lastResult.nextId) { - sinceParam = lastResult.nextSince; - sinceIdParam = lastResult.nextId; + // Output target + const out = args.output + ? createWriteStream(args.output) + : process.stdout; + const write = (text) => new Promise((resolve) => out.write(text, resolve)); + + const streaming = args.all || args.format === "ndjson"; + let itemCount = 0; + + if (streaming) { + // Streaming mode: print items as they arrive + if (args.format === "table") { + await write(printTableHeaderStr(args.normalizeHints)); + } + + let sinceParam = params.since; + let sinceIdParam = params["since-id"]; + let hasMore = true; + + while (hasMore) { + const currentParams = { ...params }; + if (sinceParam) currentParams.since = sinceParam; + if (sinceIdParam) currentParams["since-id"] = sinceIdParam; + + const result = await rpcCall(config, "get-error-reports", currentParams); + + for (const item of result.items) { + itemCount++; + const outItem = args.normalizeHints ? { ...item, hint: normalizeHint(item.hint) } : item; + if (args.format === "table") { + await write(printTableRowStr(outItem, false) + "\n"); + } else { + await write(JSON.stringify(outItem) + "\n"); + } + } + + if (result.nextSince && result.nextId) { + sinceParam = result.nextSince; + sinceIdParam = result.nextId; + } else { + hasMore = false; + } + } + } else { + // Single page: buffer and print + let allItems = []; + let lastResult = null; + let hasMore = true; + let sinceParam = params.since; + let sinceIdParam = params["since-id"]; + + while (hasMore) { + const currentParams = { ...params }; + if (sinceParam) currentParams.since = sinceParam; + if (sinceIdParam) currentParams["since-id"] = sinceIdParam; + + lastResult = await rpcCall(config, "get-error-reports", currentParams); + allItems = [...allItems, ...lastResult.items]; + + if (args.all && lastResult.nextSince && lastResult.nextId) { + sinceParam = lastResult.nextSince; + sinceIdParam = lastResult.nextId; + } else { + hasMore = false; + } + } + + if (args.normalizeHints) { + for (const item of allItems) { + item.hint = normalizeHint(item.hint); + } + } + + itemCount = allItems.length; + const output = { items: allItems }; + + if (lastResult) { + output.nextSince = lastResult.nextSince; + output.nextId = lastResult.nextId; + } + + if (args.format === "table") { + const hasMore = lastResult.nextSince && lastResult.nextId; + await write(formatListTable(output, hasMore, false) + "\n"); } else { - hasMore = false; + await write(formatJson(output) + "\n"); } } - const output = { - items: allItems - }; - - if (!args.all && lastResult) { - output.nextSince = lastResult.nextSince; - output.nextId = lastResult.nextId; - } - - if (args.format === "table") { - const hasMore = !args.all && lastResult.nextSince && lastResult.nextId; - console.log(formatListTable(output, hasMore)); - } else { - console.log(formatJson(output)); + if (args.output) { + out.end(); + process.stderr.write(`Wrote ${itemCount} items to ${args.output}\n`); } } @@ -307,12 +396,161 @@ async function cmdGet(config, args) { const result = await rpcCall(config, "get-error-report", params); if (args.format === "table") { - console.log(formatTable(result, "get")); + console.log(formatGetTable(result)); } else { console.log(formatJson(result)); } } +async function cmdStats(config, args) { + let items; + + if (args.input) { + // Read from file (supports JSON, JSON array, and NDJSON) + const raw = readFileSync(args.input, "utf-8"); + items = parseItems(raw); + } else if (!process.stdin.isTTY) { + // Read from stdin (supports JSON, JSON array, and NDJSON) + const raw = await readStdin(); + items = parseItems(raw); + } else { + // Fetch from API + const params = {}; + if (args.limit !== undefined) params.limit = args.limit; + if (args.from) params.since = args.from; + if (args.to) params["until"] = args.to; + + items = []; + let sinceParam = params.since; + let sinceIdParam = undefined; + let hasMore = true; + + while (hasMore) { + const currentParams = { ...params }; + if (sinceParam) currentParams.since = sinceParam; + if (sinceIdParam) currentParams["since-id"] = sinceIdParam; + + const result = await rpcCall(config, "get-error-reports", currentParams); + items = [...items, ...result.items]; + + if (result.nextSince && result.nextId) { + sinceParam = result.nextSince; + sinceIdParam = result.nextId; + } else { + hasMore = false; + } + } + } + + if (items.length === 0) { + console.log("No error reports found."); + return; + } + + // Normalize all hints for grouping + for (const item of items) { + item._normHint = normalizeHint(item.hint) || "(empty)"; + } + + // Aggregations + const byHint = {}; + const byHost = {}; + const byTenant = {}; + const byVersion = {}; + const bySource = {}; + const byKind = {}; + const byHour = {}; + const profiles = new Set(); + + for (const item of items) { + count(byHint, item._normHint); + count(byHost, item.host || item.tenant || "(unknown)"); + count(byTenant, item.tenant || "(unknown)"); + count(byVersion, item.version || "(unknown)"); + count(bySource, item.source || "(unknown)"); + count(byKind, item.kind || "(none)"); + + const hour = item.createdAt ? item.createdAt.substring(11, 13) : "??"; + count(byHour, hour); + + if (item.profileId) profiles.add(item.profileId); + } + + if (args.format === "json") { + console.log(formatJson({ + total: items.length, + uniqueProfiles: profiles.size, + byHint: sortDesc(byHint), + byHost: sortDesc(byHost), + byTenant: sortDesc(byTenant), + byVersion: sortDesc(byVersion), + bySource: sortDesc(bySource), + byKind: sortDesc(byKind), + byHour: sortDesc(byHour) + })); + } else { + console.log(`=== Error Stats ===`); + console.log(`Total: ${items.length}`); + console.log(`Unique profiles: ${profiles.size}`); + console.log(""); + + printTable("By Signature (normalized hint)", byHint, items.length); + printTable("By Source", bySource, items.length); + printTable("By Kind", byKind, items.length); + printTable("By Host", byHost, items.length); + printTable("By Tenant", byTenant, items.length); + printTable("By Version", byVersion, items.length); + printTable("By Hour (UTC)", byHour, items.length); + } +} + +function readStdin() { + return new Promise((resolve) => { + let data = ""; + const rl = createInterface({ input: process.stdin }); + rl.on("line", (line) => { data += line + "\n"; }); + rl.on("close", () => resolve(data)); + }); +} + +function parseItems(raw) { + try { + const data = JSON.parse(raw); + return Array.isArray(data) ? data : data.items || []; + } catch { + // NDJSON: one JSON object per line + return raw.split("\n") + .filter(line => line.trim()) + .map(line => JSON.parse(line)); + } +} + +function count(map, key) { + map[key] = (map[key] || 0) + 1; +} + +function sortDesc(map) { + return Object.entries(map) + .sort((a, b) => b[1] - a[1]) + .map(([key, count]) => ({ key, count })); +} + +function printTable(title, map, total) { + const entries = Object.entries(map).sort((a, b) => b[1] - a[1]); + const maxCount = entries[0]?.[1] || 1; + const barWidth = 30; + + console.log(`${title}:`); + for (const [key, n] of entries) { + const pct = ((100 * n) / total).toFixed(1); + const barLen = Math.max(1, Math.round((n / maxCount) * barWidth)); + const bar = "█".repeat(barLen); + const label = key.length > 60 ? key.substring(0, 57) + "..." : key; + console.log(` ${String(n).padStart(5)} (${pct.padStart(5)}%) ${bar} ${label}`); + } + console.log(""); +} + // ============================================================================ // CLI Setup with Commander // ============================================================================ @@ -328,16 +566,20 @@ program .command("list") .description("List error reports with pagination and filters") .option("-l, --limit ", "Max items per page (default: 50, max: 200)", (value) => parseInt(value, 10), 50) - .option("--since ", "ISO timestamp (fetch errors before this date)") - .option("--since-id ", "Fetch errors before this ID (cursor pagination)") + .option("--from ", "ISO timestamp — oldest boundary (fetches items after this)") + .option("--to ", "ISO timestamp — newest boundary (fetches items before this)") + .option("--since ", "ISO timestamp — explicit cursor for manual pagination") + .option("--since-id ", "Fetch errors after this ID (cursor pagination)") .option("-s, --source ", "Filter by source (legacy-v1, legacy-v2, logging, audit-log, rlimit)") .option("-p, --profile-id ", "Filter by profile ID") .option("-k, --kind ", "Filter by kind (string)") .option("-t, --tenant ", "Filter by tenant (string)") .option("--version ", "Filter by version") .option("--hint ", "Filter by hint (ILIKE match)") - .option("-a, --all", "Fetch all pages automatically", false) - .option("-f, --format ", "Output format (json|table)", "table") + .option("-a, --all", "Fetch all pages automatically (streams output)", false) + .option("-f, --format ", "Output format (json|table|ndjson)", "table") + .option("--normalize-hints", "Normalize hints by stripping dynamic values", false) + .option("-o, --output ", "Write output to file instead of stdout") .option("--env ", "Custom .env file path") .action(async (options) => { const config = loadConfig(options.env); @@ -356,4 +598,18 @@ program await cmdGet(config, options); }); +program + .command("stats") + .description("Compute error report statistics") + .option("--from ", "Start of interval (ISO timestamp)") + .option("--to ", "End of interval (ISO timestamp)") + .option("--limit ", "Items per page (default: 200)", (value) => parseInt(value, 10), 200) + .option("--input ", "Read from local JSON/NDJSON file instead of API") + .option("-f, --format ", "Output format (json|table)", "table") + .option("--env ", "Custom .env file path") + .action(async (options) => { + const config = loadConfig(options.env); + await cmdStats(config, options); + }); + program.parse();