#!/bin/bash # OTel Exporter Config & Gateway Hosts Management CLI — requires curl, jq # --- Defaults --- DEFAULT_CM_BASE_URL="http://localhost:7180" DEFAULT_CM_USER="admin" DEFAULT_CM_PASS="admin" CM_API_STALENESS="v41" CM_POLL_INTERVAL=5 CM_POLL_TIMEOUT=600 PARAM_EXPORTER="otelcol_gateway_external_metrics_exporter" PARAM_PROCESSOR="otelcol_processors" PARAM_SERVICE="otelcol_gateway_metrics_service" PARAM_EXTENSION="otelcol_extensions" PARAM_CONNECTORS="otelcol_connectors" PARAM_EXPORTER_LOGS="otelcol_gateway_external_logs_exporter" PARAM_SERVICE_LOGS="otelcol_gateway_logs_service" FANOUT_CONNECTOR="routing/metrics-fanout" GATEWAY_LB_CONNECTOR="roundrobin/gateway-lb-metrics" GATEWAY_LB_PIPELINE="metrics/gateway-lb" ROUTING_PIPELINE="metrics/routing" CUSTOMER_PIPELINE_RECEIVER="[${FANOUT_CONNECTOR}]" FANOUT_GATEWAY_TARGET="$GATEWAY_LB_PIPELINE" SIG_EXPORTER_PARAM="$PARAM_EXPORTER" SIG_SERVICE_PARAM="$PARAM_SERVICE" SIG_PREFIX="metrics" SIG_SYSTEM_EXPORTER_PREFIX='^prometheusremotewrite/gateway-' select_signal() { case "$1" in metrics) SIG_EXPORTER_PARAM="$PARAM_EXPORTER" SIG_SERVICE_PARAM="$PARAM_SERVICE" SIG_PREFIX="metrics" SIG_SYSTEM_EXPORTER_PREFIX='^prometheusremotewrite/gateway-' FANOUT_CONNECTOR="routing/metrics-fanout" GATEWAY_LB_CONNECTOR="roundrobin/gateway-lb-metrics" GATEWAY_LB_PIPELINE="metrics/gateway-lb" ROUTING_PIPELINE="metrics/routing" ;; logs) SIG_EXPORTER_PARAM="$PARAM_EXPORTER_LOGS" SIG_SERVICE_PARAM="$PARAM_SERVICE_LOGS" SIG_PREFIX="logs" SIG_SYSTEM_EXPORTER_PREFIX='^dbus/gateway-' FANOUT_CONNECTOR="routing/logs-fanout" GATEWAY_LB_CONNECTOR="roundrobin/gateway-lb-logs" GATEWAY_LB_PIPELINE="logs/gateway-lb" ROUTING_PIPELINE="logs/routing" ;; *) echo "❌ Internal error: unknown signal '$1'" >&2 return 1 ;; esac CUSTOMER_PIPELINE_RECEIVER="[${FANOUT_CONNECTOR}]" FANOUT_GATEWAY_TARGET="$GATEWAY_LB_PIPELINE" } # Expand the --signal flag ($SIGNAL) into the SIGNALS array. Empty/both => both # families, in a fixed order (metrics first) so batched writes are deterministic. build_signals() { case "${SIGNAL:-}" in ""|both) SIGNALS=(metrics logs) ;; metrics) SIGNALS=(metrics) ;; logs) SIGNALS=(logs) ;; *) echo "❌ Error: invalid --signal '$SIGNAL' (expected metrics|logs|both)." exit 1 ;; esac } # --- Prerequisite / customer-detection params --- # CM returns these UPPERCASE; write lowercase (CM is case-insensitive on write), read case-insensitively. PARAM_DMP_METRICS="enable_observability_metrics_dmp" PARAM_DMP_LOGS="enable_observability_real_time_jobs" PARAM_ALTUS_ACCOUNT="telemetry_altus_account" PARAM_REMOTE_WRITE_URL="otelcol_remote_write_url" ALTUS_ACCOUNT_NAME="dummy" DUMMY_REMOTE_WRITE_URL="http://localhost:11111" # Placeholder CDP private key for the dummy altus account. NOT A SECRET: a throwaway # PKCS#8 RSA key so the cdpauth/gateway extension can PEM-decode and build on an # air-gapped cluster (a literal like "dummy" fails pem.Decode()). Safe to commit. DUMMY_CDP_PRIVATE_KEY='-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDu6at6z4IOxV6l\nbI1vrgN1zMYfXfg6gKFqNhq5A0W19WAbuW0vh4KVjXkzqA3OfOXJdQVXoIrFOyBN\n7IYr80k/w7DibHlgbtfNYp8r2/DHWCpSckDlxrQqROCqrwL5PTwqVUum0GPb5oEL\n2Mo5wOpO3sHKGSzuYysZ8+HHBHZF8t+tOM7Mc/Yp0cTNc0rcLECIA+3FAmWSLAWW\nDh6rKb2l8Cp+DLFxIM3t+yQFMtmnTGOFEX75+eZS2WyrntQikeONkD5/ZK5G1Oer\nbKb72S8iXAb77vAHJnBHlbevwtRal6Ax7l/3EzuTQt63X1BV2lBrGviqsGPvCuGe\nJffuGjGjAgMBAAECggEAFiJ4ZzPuKPUt8XtEV/+ncBS37wmzPk79kCFmyhOyCTYs\nddUbobwseTOhcjY/NTggHVE7XKTR8Teq+gfltGZhYnLRzWCJwZ6eCCLQ0Wzv+rVs\nkl/l1a6vsXzZKHx3dOUF+9FpkBEdjK7nRNQ3LtufObhspuSR45zfeOjorETPD716\nGqYiDU4BYAH4UQ5C0pWR6eXYTfx0bnjjDpeGocwtmuMn4FPv/R0jwPVj68PYBGEa\nxFxOPKVULHbNq8GbcL+BKNZaooSAAV+7/+hwMbL19iUbS6cOEM9Gn+2QqeM0hOj9\naMs/F8ukC+v19VEgKAIaOy5XnnJKTJHCUcSurcst0QKBgQD3YTDSMmS4qL4fDHPR\nePiwQc2rZnSf0ylsSEObAsr0/Ea4gTan2vjoXsn0wciRSIRLGcW7DG8UsWst7zlv\ngRp3WFHGju+DvbtzvBnifQm9jMJbANx1GeNNJzxdN5kYdMaR7mG2CV0wX0NyF7uL\nC8WyCr41L80j2X0ymDCP8KkgEQKBgQD3PPLFVIZubrqS5kBxmeIFxheVwGon+O1a\n5B9yb1qyv/YpaeVLMgB7HhRqCZmckK6fmUGipnDDbkZyEMV4xOYbzEDYRUSGQABt\nlywkmPyAPsdvG+/JKfYDBom3XG7cwhAdz0R1k2XClzYe0PAAILAPOY5nUrJ60In8\nMxCTIv4qcwKBgQD1mg7IgbOyCgZH+Sr7q2jsRUiWhTAAhbs4gD5upcUumVD1WYQT\n/rjkq0dCsyoX47FEhp/jZyLZnpPUlQ+m/rxyBMObqMlvOG3lJ7ljPdTv1dIYW7CE\nqmEbdnrd6ZcbysRimhN0kbNtlQb3PUy7bcKnUzcowv6eOw2rr6F3Ju4ioQKBgASv\neQJYWbvaCVsjVfFRM0FcwbGq1dgwFX91d0sZ5SQiEy4xvZCxP+UVatmfmPOookaQ\nBvhoIxbxDhC3VKa3RI0eT9RJa8UHjXqGNOX0qERd7DWk1+R+ERV5mD6LV+aDHv8R\noICB7JnFd9cZqpK2C+Ad/R7kKFG/hf3rkdLYBO0VAoGAIaXHTdCT2ixSdzhlI7QG\nzMoaovekM8+2PuBJWvtTrt2uQTYFS8TVZ8KbvUxM+cpOS6Jz1el/1IJBkVYZsXxK\n/zrSQ7xMUf/CkFzvkXkhQZ583z72yf8cNPY+nPNAnrhv+0tkINboV27xelBi2lRn\n2AMnuao5x5Lqaz5yo1H4Tx0=\n-----END PRIVATE KEY-----' # --- TelemetryPublisher (TP) params --- # The logs signal (real-time-jobs / RTM) flows through the mgmt TelemetryPublisher role; # a placeholder cluster needs the role present and its safety valve seeded, or the whole # logs pipeline is non-functional even with the DMP logs flag enabled. TP_ROLE_TYPE="TELEMETRYPUBLISHER" TP_ROLE_CONFIG_GROUP="mgmt-TELEMETRYPUBLISHER-BASE" PARAM_TP_SAFETY_VALVE="telemetrypublisher_safety_valve" # Dummy safety-valve content (real newline between the two properties; jq JSON-encodes it). DUMMY_TP_SAFETY_VALVE=$'telemetry.upload.job.logs=true\ntelemetry.altus.url=https://dbusapi.sigma-dev.cloudera.com' DEBUG=false NORMALIZED_CONFIG="" # set by normalize_yaml_indentation; cleaned on exit trap 'rm -f "$NORMALIZED_CONFIG"' EXIT log() { [[ "$DEBUG" == true ]] && echo "$*" >&2; return 0; } # Pipe help text through a pager when available _pager() { if command -v less >/dev/null 2>&1; then less -R elif command -v more >/dev/null 2>&1; then more else cat fi } # ============================================================================== # HELP FUNCTIONS # ============================================================================== CMD=$(basename "$0") # Common connection options repeated in every SYNOPSIS block. _common_synopsis=' [--url ] [--user ] [--pass ] [--debug]' _print_common_opts() { printf ' --url (string) Cloudera Manager base URL. Default: %s --user (string) Cloudera Manager username for API authentication. Default: %s --pass (string) Cloudera Manager password for API authentication. Default: %s --debug (boolean) Turn on debug logging. Prints step-by-step progress messages. --refresh (boolean) After the command writes config, apply any resulting stale config to the running roles: a no-downtime deploy-client-configs-and-refresh for stale cluster services, and a restart of the Management Service only if it is genuinely STALE. If nothing is stale (e.g. plain gateway/host-level edits) this is a no-op. The same apply step is available on demand via the '"'"'refresh-config'"'"' command. ' "$DEFAULT_CM_BASE_URL" "$DEFAULT_CM_USER" "$DEFAULT_CM_PASS" } usage() { cat < [parameters] Use ${CMD} --help for information on a specific command. OPTIONS --url (string) Cloudera Manager base URL used for all API calls. Default: $DEFAULT_CM_BASE_URL --user (string) Cloudera Manager username for API authentication. Default: $DEFAULT_CM_USER --pass (string) Cloudera Manager password for API authentication. Default: $DEFAULT_CM_PASS --debug (boolean) Turn on debug logging. Progress messages are printed for each step (YAML validation, API payloads, pipeline changes). Without this flag only errors are printed. --help | -h Display this help message and exit. AVAILABLE COMMANDS Gateway Host Commands: o get-gateway-hosts o list-hosts o add-gateway-host o remove-gateway-host Onboarding: o setup-prerequisites (self-managed/airgapped: seed dummy altus creds, DMP flags, dummy remote_write_url, optional gateway hosts. add-exporter then auto-disables thanos/DMP.) Apply / Refresh: o refresh-config (detect stale CM config and apply it: deploy+refresh stale cluster services, restart mgmt if STALE. Same as passing --refresh on a command, but runnable on demand.) Exporter Commands: o get-metric-configs o list-exporters o add-exporter o update-exporter o remove-exporter Processor Commands: o list-processors o add-processor o update-processor o remove-processor Extension Commands: o list-extensions o add-extension o update-extension o remove-extension EOF exit 0 } usage_get_gateway_hosts() { { cat < [--host ...] ${_common_synopsis} OPTIONS --host (string) Hostname to add as a gateway node. Repeat this option to add multiple hosts in a single call. EOF _print_common_opts cat < [--host ...] ${_common_synopsis} OPTIONS --host (string) Hostname to remove from the gateway nodes list. Repeat this option to remove multiple hosts in a single call. EOF _print_common_opts cat < otelcol_gateway_external_metrics_exporter + otelcol_gateway_metrics_service (fan-out via routing/metrics-fanout -> roundrobin/gateway-lb-metrics) logs -> otelcol_gateway_external_logs_exporter + otelcol_gateway_logs_service (dedicated fan-out via routing/logs-fanout -> roundrobin/gateway-lb-logs) With --signal both, the same --file exporter is defined and wired into each family, and both are written in a single batch. Telemetry is duplicated (fan-out) to every destination: the internal SaaS gateway path keeps round-robin load balancing across the system gateway exporters, while each customer exporter receives 100% of the gateway signal via the signal's fan-out connector. This command also maintains otelcol_connectors (adds routing/metrics-fanout and a dedicated roundrobin/gateway-lb-metrics connector) and wires the service pipelines: metrics/routing (consumes roundrobin, exports to the fan-out connector), metrics/gateway-lb (fan-out -> roundrobin/gateway-lb-metrics), and repoints metrics/gateway- to receive from roundrobin/gateway-lb-metrics. This avoids a cyclic connector dependency. The following rules are enforced before any change is made: o File must use spaces only — tab characters are not allowed. o Root-level keys must be bare exporter names (key: with no value on the same line). Child properties must be indented below them. o Exporter names starting with prometheusremotewrite/gateway- are reserved for system use and will be rejected. o Exporter names that already exist in the config will be rejected. Use update-exporter to modify an existing exporter. o At least one OTel gateway host (otelcol_gateway_hosts) must be configured — the exporter pipelines run on the gateway, so this is mandatory. Add one with add-gateway-host (or setup-prerequisites --host) before adding an exporter. o If the DMP/thanos path is not fully configured (DMP metrics or real-time-jobs flag off, or a missing/dummy remote_write_url, or missing/dummy altus credentials), the SaaS fan-out target (metrics/gateway-lb) is dropped from routing/metrics-fanout so metrics flow only to the customer exporter(s) — this avoids export 4xx/5xx from a non-functional DMP endpoint. A fully-configured SaaS path is kept. SYNOPSIS add-exporter --file [--signal metrics|logs|both] [--setup-prerequisites] ${_common_synopsis} OPTIONS --file (string) Path to the YAML file containing one or more exporter block(s). Each root-level key in the file is treated as an exporter name. --signal (string) Which telemetry signal to wire the exporter into: metrics, logs, or both. Defaults to both when omitted. --setup-prerequisites (flag) Opt-in: before adding the exporter, seed self-managed / airgapped prerequisites — but NON-DESTRUCTIVELY: only values that are missing or disabled are added (dummy altus account, enable DMP metrics/logs flags, dummy remote_write_url). Existing real values are never overwritten. OFF by default. If the DMP path ends up not fully real, the thanos/DMP fan-out is then disabled automatically. (Gateway hosts are separate: use add-gateway-host or setup-prerequisites.) EOF _print_common_opts cat < ${_common_synopsis} OPTIONS --file (string) Path to the YAML file containing updated exporter block(s). Each root-level key must match an exporter that already exists in CM. EOF _print_common_opts cat < [--exporter ...] [--signal metrics|logs|both] ${_common_synopsis} OPTIONS --exporter (string) Name of the exporter to remove. Repeat this option to remove multiple exporters in a single call. --signal (string) Which telemetry signal to unwire the exporter from: metrics, logs, or both. Defaults to both when omitted. EOF _print_common_opts cat </). The exporter pipeline must already exist (run add-exporter first). Mode C — Link existing (--processor + --exporter) Links an already-defined processor to one or more exporter pipelines. No file is needed. --signal selects the telemetry family (default: both). With --signal both, the processor is linked into the exporter's pipeline in EACH family where the exporter exists; a family where the exporter is absent is skipped. It is an error only if the exporter matches no pipeline in any selected family. SYNOPSIS add-processor --file [--exporter ...] [--signal metrics|logs|both] add-processor --processor [--processor ...] --exporter [--exporter ...] [--signal metrics|logs|both] OPTIONS --file (string) Path to the YAML file containing one or more processor block(s). Cannot be combined with --processor. --processor (string) Name of an already-defined processor to link to a pipeline. Requires at least one --exporter. Cannot be combined with --file. Repeat to link multiple processors in a single call. --exporter (string) Name of the exporter whose pipeline should include this processor. Repeat to link to multiple exporter pipelines. --signal (string) Telemetry family to link into: metrics, logs, or both (default: both). EOF _print_common_opts cat < ${_common_synopsis} OPTIONS --file (string) Path to the YAML file containing updated processor block(s). Each root-level key must match a processor that already exists in CM. EOF _print_common_opts cat < [--processor ...] [--signal metrics|logs|both] remove-processor --processor --exporter [--exporter ...] [--signal metrics|logs|both] OPTIONS --processor (string) Name of the processor to remove or unlink. Repeat to target multiple processors in a single call. --exporter (string) When provided, only unlinks the processor from this exporter's pipeline. The definition in otelcol_processors is kept. Repeat to unlink from multiple exporter pipelines. --signal (string) Telemetry family to act on: metrics, logs, or both (default: both). EOF _print_common_opts cat < ${_common_synopsis} OPTIONS --file (string) Path to the YAML file containing one or more extension block(s). Root-level keys are treated as extension names. EOF _print_common_opts cat < ${_common_synopsis} OPTIONS --file (string) Path to the YAML file containing updated extension block(s). Each root-level key must match an extension that already exists in CM. EOF _print_common_opts cat < [--extension ...] ${_common_synopsis} OPTIONS --extension (string) Name of the extension to remove. Repeat to remove multiple extensions in a single call. EOF _print_common_opts cat </dev/null } send_batch() { local payload="$1" local tmp_file tmp_file=$(mktemp) local http_code http_code=$(curl -k -s -o "$tmp_file" -w "%{http_code}" -u "$CM_USER:$CM_PASS" \ -X POST "$CM_BASE_URL/api/v15/batch" \ -H "Content-Type: application/json" \ -d "$payload") if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then echo "✅ Success! Configuration update applied. (HTTP $http_code)" else echo "❌ Failed. (HTTP $http_code)" cat "$tmp_file" rm -f "$tmp_file" exit 1 fi rm -f "$tmp_file" } # Normalize file indentation to 2 spaces per level; sets NORMALIZED_CONFIG. normalize_yaml_indentation() { local file="$1" local content content=$(cat "$file") # ── Detect mixed tabs + spaces (leading whitespace) ────────────────────── local tab_lines space_lines tab_lines=$(printf '%s\n' "$content" | grep -c $'^\t' || true) space_lines=$(printf '%s\n' "$content" | grep -c '^ ' || true) if [[ "$tab_lines" -gt 0 && "$space_lines" -gt 0 ]]; then echo "❌ Error: Mixed tab and space indentation detected in '$file'." echo " Please use spaces only (2 or 4 spaces per level)." exit 1 fi # ── Convert pure-tab indentation → 2 spaces per tab ───────────────────── if [[ "$tab_lines" -gt 0 ]]; then log "ℹ️ Converting tab indentation → 2 spaces per level." content=$(printf '%s\n' "$content" | awk '{ line = $0; tabs = 0 while (substr(line, tabs+1, 1) == "\t") tabs++ rest = substr(line, tabs+1) spaces = tabs * 2 printf "%*s%s\n", spaces, "", rest }') fi local min_indent=0 while IFS= read -r line; do [[ -z "${line// }" ]] && continue local indent indent=$(printf '%s' "$line" | awk '{n=0; while(substr($0,n+1,1)==" ") n++; print n}') if [[ $indent -gt 0 ]]; then if [[ $min_indent -eq 0 || $indent -lt $min_indent ]]; then min_indent=$indent fi fi done <<< "$content" # ── No indentation found at all → flat YAML, cannot determine hierarchy ── if [[ $min_indent -eq 0 ]]; then echo "❌ Error: No indentation found in config file." echo " The immediate child properties of each root key must be indented" echo " with 2 spaces, 4 spaces, or tabs." echo "" echo " Example (2-space):" echo " prometheusremotewrite/my-exporter:" echo " endpoint: \"https://...\"" echo " retry_on_failure:" echo " enabled: true" exit 1 fi # ── Already 2-space ─────────────────────────────────────────────────────── if [[ $min_indent -eq 2 ]]; then NORMALIZED_CONFIG=$(mktemp /tmp/otelcli_norm.XXXXXX) printf '%s\n' "$content" > "$NORMALIZED_CONFIG" return fi # ── Validate all indentation is a consistent multiple of min_indent ─────── local bad_line="" while IFS= read -r line; do [[ -z "${line// }" ]] && continue local indent indent=$(printf '%s' "$line" | awk '{n=0; while(substr($0,n+1,1)==" ") n++; print n}') if [[ $indent -gt 0 && $((indent % min_indent)) -ne 0 ]]; then bad_line="$line" break fi done <<< "$content" if [[ -n "$bad_line" ]]; then echo "❌ Error: Inconsistent indentation detected." echo " Expected multiples of ${min_indent} space(s), but found: '${bad_line}'" echo " Please fix the indentation and retry." exit 1 fi # ── Re-scale to 2 spaces per level ─────────────────────────────────────── log "ℹ️ Normalizing indentation from ${min_indent}-space → 2-space per level." content=$(printf '%s\n' "$content" | awk -v unit="$min_indent" '{ line = $0; spaces = 0 while (substr(line, spaces+1, 1) == " ") spaces++ rest = substr(line, spaces+1) new_spaces = int(spaces / unit) * 2 printf "%*s%s\n", new_spaces, "", rest }') NORMALIZED_CONFIG=$(mktemp /tmp/otelcli_norm.XXXXXX) printf '%s\n' "$content" > "$NORMALIZED_CONFIG" } validate_yaml_indentation() { local file="$1" local existing_root_indent="$2" log "🔎 Normalizing and validating YAML indentation..." # Normalize first (tabs → 2-space, 4-space → 2-space, etc.) normalize_yaml_indentation "$file" # All further validation works on the normalized copy local norm="$NORMALIZED_CONFIG" if grep -q ' $' "$norm"; then log "⚠️ Warning: Config file has trailing spaces on some lines." fi FILE_ROOT_INDENT=$(grep -m1 '[^ ]' "$norm" | sed 's/^\( *\).*/\1/' | wc -c) FILE_ROOT_INDENT=$((FILE_ROOT_INDENT - 1)) if [[ "$FILE_ROOT_INDENT" -ne "$existing_root_indent" ]]; then echo "❌ Error: Indentation mismatch." echo " Existing config uses $existing_root_indent space(s) at root level." echo " Your file uses $FILE_ROOT_INDENT space(s) at root level (after normalization)." exit 1 fi local indent_errors=0 while IFS= read -r line; do [[ -z "${line// }" ]] && continue local line_indent stripped line_indent=$(echo "$line" | sed 's/^\( *\).*/\1/' | wc -c) line_indent=$((line_indent - 1)) stripped=$(echo "$line" | sed 's/^[[:space:]]*//') if [[ "$line_indent" -lt "$FILE_ROOT_INDENT" ]]; then echo "❌ Error: Line under-indented: '$line'" indent_errors=$((indent_errors + 1)) elif [[ "$line_indent" -eq "$FILE_ROOT_INDENT" ]]; then # Root-level line must be a bare key (exporter/processor name) — no value after ':' if ! echo "$stripped" | grep -qE '^[^[:space:]]+:[[:space:]]*$'; then echo "❌ Error: Child property at root indentation level (must be indented under key name):" echo " '$line'" indent_errors=$((indent_errors + 1)) fi fi done < "$norm" if [[ "$indent_errors" -gt 0 ]]; then echo "❌ $indent_errors indentation error(s) found. Please fix and retry." exit 1 fi # Detect scalar-valued key followed by a MORE-indented line (invalid nesting). # Sequence markers ("- ") advance the effective indent so list-item siblings # (which align after the dash) are not mistakenly flagged. local _pyi=-1 _pys=false _pyl="" while IFS= read -r line; do [[ -z "${line// }" ]] && continue local _ci _sv _eff _content _ci=$(echo "$line" | awk '{n=0; while(substr($0,n+1,1)==" ") n++; print n}') _sv=$(echo "$line" | sed 's/^[[:space:]]*//') # Strip leading "- " sequence marker(s) and advance the effective indent # to the column where the map content actually begins. _content="$_sv" _eff="$_ci" while [[ "$_content" == "- "* || "$_content" == "-" ]]; do if [[ "$_content" == "-" ]]; then _content="" _eff=$((_eff + 1)) else _content="${_content#- }" _eff=$((_eff + 2)) fi done if [[ "$_pys" == true && "$_eff" -gt "$_pyi" ]]; then echo "❌ Error: Indentation inconsistency detected." echo " '${_pyl}' has a value but the next line is more indented:" echo " '${line}'" echo " Properties at the same level must use consistent indentation." exit 1 fi # Has scalar value: key: value — but not block scalars (|, >) or bare keys (key:) if echo "$_content" | grep -qE '^[^:]+:[[:space:]]+[^|>[:space:]]'; then _pys=true else _pys=false fi _pyi="$_eff" _pyl="$line" done < "$norm" log "✅ YAML indentation valid (normalized to 2-space)." } # Parse all exporter names from a config string — prints one name per line to stdout. # Bash 3-compatible (no nameref). Callers capture with a while read loop. parse_exporter_names() { local config="$1" local root_indent="$2" while IFS= read -r line; do [[ -z "${line// }" ]] && continue local line_indent line_indent=$(echo "$line" | sed 's/^\( *\).*/\1/' | wc -c) line_indent=$((line_indent - 1)) if [[ "$line_indent" -eq "$root_indent" ]] && echo "$line" | grep -q ':'; then local name name=$(echo "$line" | sed 's/:[[:space:]]*$//' | sed 's/^[[:space:]]*//') [[ -n "$name" ]] && echo "$name" fi done <<< "$config" } # Remove a named YAML block by name from a config string remove_named_block() { local config="$1" local target_name="$2" local root_indent="$3" echo "$config" | awk \ -v target="$target_name" \ -v rindent="$root_indent" \ 'BEGIN { skip=0 } { indent = 0 line = $0 while (substr(line, indent+1, 1) == " ") indent++ # A key at root indent toggles skip; comment lines never toggle it if (indent == rindent && $0 !~ /^[[:space:]]*$/ && substr(line, indent+1, 1) != "#") { name = $0 gsub(/^[[:space:]]+/, "", name) gsub(/:[[:space:]]*$/, "", name) skip = (name == target) ? 1 : 0 } if (!skip) print $0 }' } # Remove an exporter name from the service pipeline exporters list remove_from_service() { local service_val="$1" local name="$2" local escaped escaped=$(printf '%s\n' "$name" | sed 's/[[\.*^$()+?{|]/\\&/g') echo "$service_val" \ | sed "s|, ${escaped}||g" \ | sed "s|${escaped}, ||g" \ | sed "s|\[${escaped}\]|\[\]|g" } is_system_exporter() { echo "$1" | grep -q "$SIG_SYSTEM_EXPORTER_PREFIX" } is_system_extension() { echo "$1" | grep -q '^cdpauth/thanos-' } # Indent of the first named child under a wrapper key (e.g. "processors:"). get_child_indent() { local config="$1" local wrapper_key="$2" echo "$config" | awk -v key="$wrapper_key" ' $0 ~ "^[[:space:]]*" key "[[:space:]]*$" { found=1; next } found && /^[[:space:]]+[^[:space:]]/ { spaces = 0 line = $0 while (substr(line, spaces+1, 1) == " ") spaces++ if (substr(line, spaces+1, 1) == "#") next print spaces exit } ' } # Auth extension references (auth.authenticator: ) declared in a config file. parse_authenticator_refs() { grep -E '^[[:space:]]*authenticator:[[:space:]]*[^[:space:]#]' "$1" 2>/dev/null \ | sed -E 's/^[[:space:]]*authenticator:[[:space:]]*//; s/[[:space:]]+#.*$//; s/[[:space:]]*$//' \ | tr -d "\"'" \ | sort -u } # True if NAME is defined as a child key of WRAPPER in CONFIG. # Matches the whole name: a plain 'grep -qF' on the raw YAML would accept any # prefix, so 'attributes/partne' would be found in 'attributes/partner'. component_defined() { local name="$1" config="$2" wrapper="$3" root names [[ -z "$config" || "$config" == "null" ]] && return 1 root=$(get_child_indent "$config" "$wrapper") [[ -z "$root" ]] && return 1 # Capture first, then match: grep -q closing a live pipe early would SIGPIPE the # producer (non-zero under pipefail). A herestring has no upstream process to kill. names=$(parse_exporter_names "$config" "$root") grep -qxF "$name" <<< "$names" } # True if an extension NAME is defined in the otelcol_extensions value. extension_defined() { component_defined "$1" "$2" "extensions:" } # Indent every non-empty line of a block by N spaces. indent_block() { local spaces="$1" local block="$2" local pad="" i=0 while [[ $i -lt $spaces ]]; do pad="${pad} "; i=$((i+1)); done while IFS= read -r line; do if [[ -z "$line" ]]; then echo ""; else echo "${pad}${line}"; fi done <<< "$block" } # Derive OTel pipeline name from an exporter name, for the active signal. # metrics: prometheusremotewrite/priv → metrics/prometheusremotewrite-priv # logs: prometheusremotewrite/priv → logs/prometheusremotewrite-priv exporter_to_pipeline() { echo "${SIG_PREFIX}/$(echo "$1" | tr '/' '-')" } # Check if a named pipeline block exists in the service YAML. pipeline_exists() { local service_val="$1" local pipeline_name="$2" echo "$service_val" | grep -qF "${pipeline_name}:" } # Return 0 if VALUE is an element of the bracketed list FIELD (e.g. processors) # within the named pipeline block, else 1. Scoped to that single pipeline so a # link in a different pipeline does not produce a false positive. pipeline_field_contains() { local service_val="$1" local pipeline_name="$2" local field="$3" local value="$4" local list list=$(echo "$service_val" | awk \ -v pl="${pipeline_name}:" \ -v field="${field}:" \ 'BEGIN { in_p=0 } index($0, pl) && in_p==0 { in_p=1; next } in_p && index($0, field) { if (match($0, /\[.*\]/)) { print substr($0, RSTART+1, RLENGTH-2) } in_p=0 } in_p && /exporters:/ { in_p=0 }') local IFS=',' local item for item in $list; do item="${item#"${item%%[![:space:]]*}"}" item="${item%"${item##*[![:space:]]}"}" [[ "$item" == "$value" ]] && return 0 done return 1 } # Emit "\t" for every entry of every pipeline's processors # list in a service YAML value. Splits the bracketed list so lookups match whole # names. Pass one signal's service value at a time; the caller concatenates. service_processor_links() { echo "$1" | awk ' /^[[:space:]]+(metrics|logs)\/[^[:space:]]+:[[:space:]]*$/ { pl = $0 sub(/^[[:space:]]+/, "", pl) sub(/:[[:space:]]*$/, "", pl) next } pl != "" && /^[[:space:]]+processors:/ { if (match($0, /\[[^]]*\]/)) { n = split(substr($0, RSTART+1, RLENGTH-2), items, ",") for (i = 1; i <= n; i++) { gsub(/^[[:space:]]+|[[:space:]]+$/, "", items[i]) if (items[i] != "") print pl "\t" items[i] } } }' } # Add a value to a bracketed list field inside a named pipeline block. # Handles both empty ( [] ) and non-empty ( [a, b] ) lists. # If the field key does not exist yet in the pipeline, it is inserted # before the exporters: line (used when adding the first processor). add_to_pipeline_field() { local service_val="$1" local pipeline_name="$2" local field="$3" local name="$4" local safe_name safe_name=$(printf '%s' "$name" | sed 's/[&\]/\\&/g') echo "$service_val" | awk \ -v pipeline="${pipeline_name}:" \ -v field="$field:" \ -v name="$safe_name" \ 'BEGIN { in_pipe=0; found_field=0 } { if (index($0, pipeline) && in_pipe==0) { in_pipe=1 } if (in_pipe) { trimmed = $0 gsub(/^[[:space:]]+/, "", trimmed) # Found the target field — append the name to its list if (index(trimmed, field) == 1) { if ($0 ~ /\[\]/) { sub(/\[\]/, "[" name "]") } else { sub(/\]$/, ", " name "]") } found_field=1 in_pipe=0 } # Reached exporters: without having seen the field — insert it first if (!found_field && index(trimmed, "exporters:") == 1) { indent_str = $0 sub(/[^ ].*/, "", indent_str) print indent_str field " [" name "]" in_pipe=0 } } print }' } # Remove a value from a bracketed list field inside a named pipeline block. remove_from_pipeline_field() { local service_val="$1" local pipeline_name="$2" local field="$3" local name="$4" local escaped escaped=$(printf '%s\n' "$name" | sed 's/[[\.*^$()+?{|]/\\&/g') echo "$service_val" | awk \ -v pipeline="${pipeline_name}:" \ -v field="$field:" \ -v esc="$escaped" \ 'BEGIN { in_pipe=0 } { if (index($0, pipeline) && in_pipe==0) { in_pipe=1 } if (in_pipe) { trimmed = $0 gsub(/^[[:space:]]+/, "", trimmed) if (index(trimmed, field) == 1) { gsub(", " esc, "") gsub(esc ", ", "") gsub("\\[" esc "\\]", "[]") in_pipe=0 } } print }' } default_connectors_yaml() { cat <<'EOF' connectors: roundrobin: loadbalancing: EOF } get_connectors_or_default() { local connectors_val="$1" if [[ -z "$connectors_val" || "$connectors_val" == "null" ]]; then default_connectors_yaml else printf '%s\n' "$connectors_val" fi } # Ensure a bare connector key (e.g. roundrobin/gateway-lb-metrics:) exists under # the connectors: header. Idempotent; preserves existing keys/blocks. ensure_connector_key() { local connectors_val="$1" local key="$2" if echo "$connectors_val" | grep -qE "^[[:space:]]*${key}:[[:space:]]*$"; then printf '%s\n' "$connectors_val" return fi local indent child indent=$(echo "$connectors_val" | grep -m1 '[^ ]' | sed 's/^\( *\).*/\1/') child="${indent} " echo "$connectors_val" | awk -v key="${child}${key}:" ' { print } /^[[:space:]]*connectors:[[:space:]]*$/ && !done { print key; done=1 } ' } ensure_fanout_connector() { local connectors_val connectors_val=$(get_connectors_or_default "$1") connectors_val=$(ensure_connector_key "$connectors_val" "$GATEWAY_LB_CONNECTOR") if echo "$connectors_val" | grep -qF "${FANOUT_CONNECTOR}:"; then fanout_add_pipeline "$connectors_val" "$FANOUT_GATEWAY_TARGET" return fi local indent child grand great pipe_key pipe_item indent=$(echo "$connectors_val" | grep -m1 '[^ ]' | sed 's/^\( *\).*/\1/') child="${indent} " grand="${child} " great="${grand} " pipe_key="${great} " pipe_item="${pipe_key} " printf '%s\n%s%s:\n%serror_mode: ignore\n%stable:\n%s- condition: "true"\n%spipelines:\n%s- %s\n' \ "$connectors_val" \ "$child" "$FANOUT_CONNECTOR" \ "$grand" \ "$grand" \ "$great" \ "$pipe_key" \ "$pipe_item" "$FANOUT_GATEWAY_TARGET" } # Add a pipeline name to the condition:"true" fan-out table (idempotent). fanout_add_pipeline() { local connectors_val="$1" local pipeline_name="$2" echo "$connectors_val" | awk \ -v fanout="${FANOUT_CONNECTOR}:" \ -v pipeline="$pipeline_name" \ 'BEGIN { in_fanout=0 in_table=0 in_entry=0 in_pipelines=0 found=0 added=0 fanout_indent=0 pipe_indent=0 } { line = $0 trimmed = line sub(/^[[:space:]]+/, "", trimmed) spaces = length(line) - length(trimmed) if (trimmed == fanout) { in_fanout=1 fanout_indent=spaces in_table=0 in_entry=0 in_pipelines=0 } else if (in_fanout && spaces <= fanout_indent && trimmed != "" && trimmed !~ /^#/) { if (in_pipelines && !added && !found) { printf "%*s- %s\n", pipe_indent, "", pipeline added=1 } in_fanout=0 in_table=0 in_entry=0 in_pipelines=0 } if (in_fanout && trimmed == "table:") { in_table=1 } if (in_fanout && in_table && trimmed ~ /^- condition:/ && trimmed ~ /true/) { in_entry=1 } if (in_fanout && in_entry && trimmed ~ /^pipelines:/) { in_pipelines=1 pipe_indent=spaces+2 } if (in_pipelines && trimmed ~ /^- /) { item = trimmed sub(/^- /, "", item) if (item == pipeline) found=1 } if (in_pipelines && spaces <= fanout_indent+4 && trimmed !~ /^- / && trimmed !~ /^pipelines:/ && trimmed != "") { if (!added && !found) { printf "%*s- %s\n", pipe_indent, "", pipeline added=1 } in_pipelines=0 } print line } END { if (in_pipelines && !added && !found) { printf "%*s- %s\n", pipe_indent, "", pipeline } }' } # Remove a pipeline from the fan-out table; gateway load-balance target is retained. fanout_remove_pipeline() { local connectors_val="$1" local pipeline_name="$2" local force="${3:-}" # "force" allows removing the protected gateway-lb target echo "$connectors_val" | awk \ -v fanout="${FANOUT_CONNECTOR}:" \ -v pipeline="$pipeline_name" \ -v keep="$FANOUT_GATEWAY_TARGET" \ -v force="$force" \ 'BEGIN { in_fanout=0; fanout_indent=0 } { line = $0 trimmed = line sub(/^[[:space:]]+/, "", trimmed) spaces = length(line) - length(trimmed) if (trimmed == fanout) { in_fanout=1 fanout_indent=spaces print line next } if (in_fanout && spaces <= fanout_indent && trimmed != "" && trimmed !~ /^#/) { in_fanout=0 } if (in_fanout && trimmed ~ /^- /) { item = trimmed sub(/^- /, "", item) if (item == pipeline && (item != keep || force == "force")) next } print line }' } detect_service_indent() { local n n=$(printf '%s\n' "$1" | awk ' # a pipeline key line: indented, contains "/", ends in ":" (no value), not a comment /^[[:space:]]*[^[:space:]#].*\/.*:[[:space:]]*$/ { line=$0; c=0; while (substr(line, c+1, 1) == " ") c++; last=c; found=1 } END { if (found) print last } ') if [[ -z "$n" ]]; then printf ' '; else printf '%*s' "$n" ''; fi } append_service_pipeline() { local service_val="$1" local pipeline_name="$2" local receivers="$3" local exporters="$4" local indent indent=$(detect_service_indent "$service_val") printf '%s\n%s%s:\n%s receivers: %s\n%s exporters: %s' \ "$service_val" \ "$indent" "$pipeline_name" \ "$indent" "$receivers" \ "$indent" "$exporters" } ensure_routing_pipeline() { local service_val="$1" if pipeline_exists "$service_val" "$ROUTING_PIPELINE"; then echo "$service_val" return fi log "ℹ️ Created pipeline '$ROUTING_PIPELINE' feeding $FANOUT_CONNECTOR." append_service_pipeline "$service_val" "$ROUTING_PIPELINE" "[roundrobin]" "[$FANOUT_CONNECTOR]" } # Repoint SaaS gateway pipelines (metrics/gateway-) from [roundrobin] to the # dedicated [roundrobin/gateway-lb-] connector, making the fan-out path acyclic. repoint_gateway_pipelines_to_lb() { local service_val="$1" echo "$service_val" | awk \ -v newrecv="[${GATEWAY_LB_CONNECTOR}]" \ -v prefix="$SIG_PREFIX" ' BEGIN { in_gw=0; gw_re = "^" prefix "/gateway-.+:$" } { line = $0 trimmed = line sub(/^[[:space:]]+/, "", trimmed) if (trimmed ~ gw_re) { in_gw=1 print line next } else if (trimmed ~ /^[a-z][a-z0-9]*\/[^:]+:$/) { in_gw=0 } if (in_gw && trimmed ~ /^receivers:[[:space:]]*\[roundrobin\][[:space:]]*$/) { indent = line sub(/[^ ].*/, "", indent) print indent "receivers: " newrecv next } print line }' } # Ensure the intermediate SaaS load-balance pipeline exists. It receives the # duplicated stream from routing/metrics-fanout and load-balances across the # gateway exporters via the dedicated roundrobin/gateway-lb-metrics connector. ensure_gateway_lb_pipeline() { local service_val="$1" if pipeline_exists "$service_val" "$GATEWAY_LB_PIPELINE"; then echo "$service_val" return fi log "ℹ️ Created pipeline '$GATEWAY_LB_PIPELINE' for SaaS round-robin path." append_service_pipeline "$service_val" "$GATEWAY_LB_PIPELINE" "$CUSTOMER_PIPELINE_RECEIVER" "[${GATEWAY_LB_CONNECTOR}]" } # Wire fan-out routing infrastructure in the service config. ensure_fanout_service_wiring() { local service_val="$1" service_val=$(ensure_routing_pipeline "$service_val") service_val=$(repoint_gateway_pipelines_to_lb "$service_val") service_val=$(ensure_gateway_lb_pipeline "$service_val") echo "$service_val" } # Echo each pipeline name listed in the active fan-out connector's condition:"true" # table (one per line). Mirrors the table walk in fanout_remove_pipeline. Used to # decide when the last customer pipeline has been removed. fanout_table_pipelines() { local connectors_val="$1" echo "$connectors_val" | awk \ -v fanout="${FANOUT_CONNECTOR}:" ' BEGIN { in_fanout=0; in_pipes=0; fanout_indent=0 } { line = $0 trimmed = line sub(/^[[:space:]]+/, "", trimmed) spaces = length(line) - length(trimmed) if (trimmed == fanout) { in_fanout=1; fanout_indent=spaces; in_pipes=0; next } if (in_fanout && spaces <= fanout_indent && trimmed != "" && trimmed !~ /^#/) { in_fanout=0; in_pipes=0 } if (in_fanout && trimmed ~ /^pipelines:/) { in_pipes=1; next } if (in_pipes && trimmed ~ /^- /) { item=trimmed; sub(/^- /,"",item); print item } else if (in_pipes && trimmed != "" && trimmed !~ /^- /) { in_pipes=0 } }' } # Inverse of repoint_gateway_pipelines_to_lb: revert the SaaS gateway pipelines # (/gateway-) from the dedicated roundrobin/gateway-lb- connector # back to the primary [roundrobin], restoring the pristine ingest path. repoint_gateway_pipelines_to_primary() { local service_val="$1" echo "$service_val" | awk \ -v oldrecv="[${GATEWAY_LB_CONNECTOR}]" \ -v prefix="$SIG_PREFIX" ' BEGIN { in_gw=0; gw_re = "^" prefix "/gateway-.+:$" } { line = $0 trimmed = line sub(/^[[:space:]]+/, "", trimmed) if (trimmed ~ gw_re) { in_gw=1 print line next } else if (trimmed ~ /^[a-z][a-z0-9]*\/[^:]+:$/) { in_gw=0 } if (in_gw && trimmed == "receivers: " oldrecv) { indent = line sub(/[^ ].*/, "", indent) print indent "receivers: [roundrobin]" next } print line }' } # Tear down the active signal's fan-out connectors: remove routing/-fanout and # roundrobin/gateway-lb-, leaving the shared roundrobin/loadbalancing intact. teardown_fanout_connectors() { local connectors_val="$1" local ci ci=$(get_child_indent "$connectors_val" "connectors:") [[ -z "$ci" ]] && ci=2 connectors_val=$(remove_named_block "$connectors_val" "$FANOUT_CONNECTOR" "$ci") connectors_val=$(remove_named_block "$connectors_val" "$GATEWAY_LB_CONNECTOR" "$ci") printf '%s\n' "$connectors_val" } # Tear down the active signal's fan-out service wiring: revert the SaaS gateway # pipelines to the primary roundrobin, then drop the /routing and /gateway-lb # pipelines. Restores the pristine SaaS-only topology. teardown_fanout_service() { local service_val="$1" service_val=$(repoint_gateway_pipelines_to_primary "$service_val") service_val=$(remove_customer_pipeline "$service_val" "$ROUTING_PIPELINE") service_val=$(remove_customer_pipeline "$service_val" "$GATEWAY_LB_PIPELINE") printf '%s\n' "$service_val" } # Extract the receivers list from the first system pipeline in the service YAML. # Falls back to an empty list if nothing can be detected. get_system_receiver() { local service_val="$1" echo "$service_val" | awk ' /metrics\/gateway-/ { in_sys=1; next } in_sys && /receivers:/ { match($0, /\[.*\]/) if (RSTART > 0) { print substr($0, RSTART, RLENGTH); exit } } in_sys && /^[[:space:]]*[a-z]/ && !/receivers:/ && !/processors:/ && !/exporters:/ { in_sys=0 } ' } # Append a new customer pipeline block to the service YAML. # Customer pipelines receive duplicated metrics from routing/metrics-fanout. append_customer_pipeline() { local service_val="$1" local pipeline_name="$2" local exporter_name="$3" local indent indent=$(detect_service_indent "$service_val") printf '%s\n%s%s:\n%s receivers: %s\n%s exporters: [%s]' \ "$service_val" \ "$indent" "$pipeline_name" \ "$indent" "$CUSTOMER_PIPELINE_RECEIVER" \ "$indent" "$exporter_name" } # Remove an entire named pipeline block from the service YAML. The block boundary is # derived from the MATCHED pipeline key's own indent (not the value's first line), so it # is robust to non-uniformly-indented values: skipping stops at the first non-blank, # non-comment line indented at or below the pipeline key. remove_customer_pipeline() { local service_val="$1" local pipeline_name="$2" echo "$service_val" | awk \ -v pipeline="${pipeline_name}:" \ 'BEGIN { skip=0; bi=-1 } { # Count leading spaces line = $0; spaces = 0 while (substr(line, spaces+1, 1) == " ") spaces++ trimmed = substr(line, spaces+1) if (trimmed == pipeline) { skip=1; bi=spaces; next } if (skip) { # Stop skipping at a sibling/dedent: non-blank, non-comment, indent <= key if (trimmed != "" && trimmed !~ /^#/ && spaces <= bi) { skip=0 } else { next } } print }' } # ============================================================================== # PARSE COMMAND & OPTIONS # ============================================================================== # Show per-command help, falling back to main help for unknown commands show_command_help() { case "$1" in get-gateway-hosts) usage_get_gateway_hosts ;; list-hosts) usage_list_hosts ;; get-metric-configs) usage_get_metric_configs ;; list-exporters) usage_list_exporters ;; add-gateway-host) usage_add_gateway_host ;; remove-gateway-host) usage_remove_gateway_host ;; setup-prerequisites) usage_setup_prerequisites ;; refresh-config) usage_refresh_config ;; add-exporter) usage_add_exporter ;; update-exporter) usage_update_exporter ;; remove-exporter) usage_remove_exporter ;; list-processors) usage_list_processors ;; add-processor) usage_add_processor ;; update-processor) usage_update_processor ;; remove-processor) usage_remove_processor ;; list-extensions) usage_list_extensions ;; add-extension) usage_add_extension ;; update-extension) usage_update_extension ;; remove-extension) usage_remove_extension ;; *) echo "" echo "❌ Unknown command: '$1'" echo "" echo "Run '$(basename "$0") --help' for the list of available commands." echo "" exit 1 ;; esac } # ============================================================================== # EXECUTE COMMAND # ============================================================================== # Ensure YAML block params end with a trailing newline (CM concatenates snippets; # command substitution strips them). is_yaml_block_param() { case "$1" in "$PARAM_EXPORTER"|"$PARAM_PROCESSOR"|"$PARAM_SERVICE"|"$PARAM_EXTENSION"|"$PARAM_CONNECTORS"|"$PARAM_EXPORTER_LOGS"|"$PARAM_SERVICE_LOGS") return 0 ;; *) return 1 ;; esac } send_config_update() { local message="$1" shift local items_json="[]" while [[ $# -ge 2 ]]; do local _name="$1" _value="$2" if is_yaml_block_param "$_name" && [[ -n "$_value" && "$_value" != *$'\n' ]]; then _value="${_value}"$'\n' fi items_json=$(echo "$items_json" | jq --arg n "$_name" --arg v "$_value" '. + [{name:$n,value:$v}]') shift 2 done local body payload body=$(jq -n --argjson items "$items_json" '{items:$items}') payload=$(jq -n \ --arg url "/api/v31/cm/allHosts/config?message=${message}" \ --argjson body "$body" \ '{items:[{method:"PUT",url:$url,body:$body,contentType:"application/json"}]}') send_batch "$payload" } require_config_file() { local cmd_name="$1" if [[ -z "$CONFIG_FILE" ]]; then echo "❌ Error: --file is required." echo " Usage: $0 ${cmd_name} --file " exit 1 fi if [[ ! -f "$CONFIG_FILE" ]]; then echo "❌ Error: File '$CONFIG_FILE' not found." exit 1 fi } get_exporter_root_indent() { local exporter_val="$1" local indent indent=$(echo "$exporter_val" | grep -m1 '[^ ]' | sed 's/^\( *\).*/\1/' | wc -c) echo $((indent - 1)) } parse_names_from_normalized_config() { local root_indent="$1" parse_exporter_names "$(cat "$NORMALIZED_CONFIG")" "$root_indent" } handle_get_gateway_hosts() { [[ "$SHOW_HELP" == true ]] && usage_get_gateway_hosts fetch_global_config CURR_GATEWAY_HOSTS=$(get_config_value "otelcol_gateway_hosts") echo "" echo "otelcol_gateway_hosts:" if [[ -z "$CURR_GATEWAY_HOSTS" || "$CURR_GATEWAY_HOSTS" == "null" ]]; then echo " " else echo "$CURR_GATEWAY_HOSTS" | tr ',' '\n' | while IFS= read -r h; do echo " - $h" done fi echo "" } handle_list_hosts() { [[ "$SHOW_HELP" == true ]] && usage_list_hosts fetch_global_config CURR_GATEWAY_HOSTS=$(get_config_value "otelcol_gateway_hosts") ALL_HOSTS_RESPONSE=$(curl -k -s -u "$CM_USER:$CM_PASS" "$CM_BASE_URL/api/v40/hosts") if [[ -z "$ALL_HOSTS_RESPONSE" ]]; then echo "❌ Error: Could not fetch hosts list." exit 1 fi echo "" printf "%-50s %-18s %s\n" "HOSTNAME" "IP" "GATEWAY HOST" printf "%-50s %-18s %s\n" "--------" "--" "------------" while IFS= read -r entry; do hostname=$(echo "$entry" | jq -r '.hostname') ip=$(echo "$entry" | jq -r '.ipAddress') if echo "$CURR_GATEWAY_HOSTS" | grep -qF "$hostname"; then status="✅ yes" else status=" no" fi printf "%-50s %-18s %s\n" "$hostname" "$ip" "$status" done < <(echo "$ALL_HOSTS_RESPONSE" | jq -c '.items[]') echo "" } handle_list_exporters() { [[ "$SHOW_HELP" == true ]] && usage_list_exporters fetch_global_config # Exporters live in TWO separate params — metrics (otelcol_gateway_external_metrics_exporter) # and logs (otelcol_gateway_external_logs_exporter). An exporter added with --signal both is # defined in each, so a name can appear under both signals; list all of them, deduped, with # a SIGNAL column. (System exporters are signal-specific and use a per-signal name prefix.) local m_val l_val m_names="" l_names="" m_sys l_sys m_val=$(get_config_value "$PARAM_EXPORTER") l_val=$(get_config_value "$PARAM_EXPORTER_LOGS") if [[ ( -z "$m_val" || "$m_val" == "null" ) && ( -z "$l_val" || "$l_val" == "null" ) ]]; then log "⚠️ No exporter config found." exit 0 fi [[ -n "$m_val" && "$m_val" != "null" ]] && \ m_names=$(parse_exporter_names "$m_val" "$(get_exporter_root_indent "$m_val")") [[ -n "$l_val" && "$l_val" != "null" ]] && \ l_names=$(parse_exporter_names "$l_val" "$(get_exporter_root_indent "$l_val")") # Canonical per-signal system-exporter prefixes (single source of truth: select_signal). select_signal metrics >/dev/null 2>&1; m_sys="$SIG_SYSTEM_EXPORTER_PREFIX" select_signal logs >/dev/null 2>&1; l_sys="$SIG_SYSTEM_EXPORTER_PREFIX" # Union of names, metrics first then logs-only, deduped. local all_names all_names=$(printf '%s\n%s\n' "$m_names" "$l_names" | grep -v '^[[:space:]]*$' | awk '!seen[$0]++') echo "" printf "%-50s %-15s %s\n" "EXPORTER NAME" "SIGNAL" "TYPE" printf "%-50s %-15s %s\n" "-------------" "------" "----" local count=0 name sig in_m in_l is_sys while IFS= read -r name; do [[ -z "$name" ]] && continue in_m=0; in_l=0; is_sys=0 printf '%s\n' "$m_names" | grep -qxF "$name" && in_m=1 printf '%s\n' "$l_names" | grep -qxF "$name" && in_l=1 if [[ $in_m -eq 1 && $in_l -eq 1 ]]; then sig="metrics, logs" elif [[ $in_m -eq 1 ]]; then sig="metrics" else sig="logs"; fi { [[ $in_m -eq 1 ]] && echo "$name" | grep -q "$m_sys"; } && is_sys=1 { [[ $in_l -eq 1 ]] && echo "$name" | grep -q "$l_sys"; } && is_sys=1 if [[ $is_sys -eq 1 ]]; then printf "%-50s %-15s %s\n" "$name" "$sig" "system (read-only)" else printf "%-50s %-15s %s\n" "$name" "$sig" "customer" fi count=$((count + 1)) done <<< "$all_names" echo "" echo "Total: $count exporter(s)" echo "" } handle_get_metric_configs() { [[ "$SHOW_HELP" == true ]] && usage_get_metric_configs fetch_global_config CURR_VAL_EXPORTER=$(get_config_value "$PARAM_EXPORTER") CURR_VAL_SERVICE=$(get_config_value "$PARAM_SERVICE") CURR_VAL_CONNECTORS=$(get_config_value "$PARAM_CONNECTORS") echo "" echo "=== $PARAM_EXPORTER ===" echo "${CURR_VAL_EXPORTER:-}" echo "" echo "=== $PARAM_SERVICE ===" echo "${CURR_VAL_SERVICE:-}" echo "" echo "=== $PARAM_CONNECTORS ===" echo "$(get_connectors_or_default "$CURR_VAL_CONNECTORS")" echo "" } handle_add_gateway_host() { [[ "$SHOW_HELP" == true ]] && usage_add_gateway_host if [[ ${#HOSTS[@]} -eq 0 ]]; then echo "❌ Error: At least one --host is required." echo " Usage: $0 add-gateway-host --host [--host ...]" exit 1 fi # Gateway hosts must be the CM host's domain name (FQDN), not an IP. Validate # each --host against the CM host inventory; if an IP was given, point the user # at the matching hostname instead of silently accepting a bad value. local hosts_resp hosts_resp=$(curl -k -s -u "$CM_USER:$CM_PASS" "$CM_BASE_URL/api/v40/hosts") if [[ -n "$hosts_resp" ]] && echo "$hosts_resp" | jq -e '.items' >/dev/null 2>&1; then for host in "${HOSTS[@]}"; do if echo "$hosts_resp" | jq -e --arg h "$host" '.items[]? | select(.hostname==$h)' >/dev/null 2>&1; then continue # valid hostname fi local match_name match_name=$(echo "$hosts_resp" | jq -r --arg h "$host" '.items[]? | select(.ipAddress==$h) | .hostname' 2>/dev/null | head -1) if [[ -n "$match_name" ]]; then echo "❌ Error: '$host' is an IP address. Use the host's domain name instead: '$match_name'." else echo "❌ Error: '$host' is not a known Cloudera Manager host." echo " Run './obs-cli.sh list-hosts' to see valid host names (use the domain name, not an IP)." fi exit 1 done else log "⚠️ Could not fetch host inventory to validate --host; proceeding without validation." fi fetch_global_config CURR_GATEWAY_HOSTS=$(get_config_value "otelcol_gateway_hosts") ALREADY_PRESENT=() NEW_HOSTS=() for host in "${HOSTS[@]}"; do if echo "$CURR_GATEWAY_HOSTS" | tr ',' '\n' | grep -qxF "$host"; then ALREADY_PRESENT+=("$host") else NEW_HOSTS+=("$host") fi done [[ ${#ALREADY_PRESENT[@]} -gt 0 ]] && echo "ℹ️ Already configured (skipping): ${ALREADY_PRESENT[*]}" if [[ ${#NEW_HOSTS[@]} -eq 0 ]]; then echo "ℹ️ No changes needed — all specified gateway host(s) are already configured." exit 0 fi ADDED=$(IFS=','; echo "${NEW_HOSTS[*]}") if [[ -n "$CURR_GATEWAY_HOSTS" && "$CURR_GATEWAY_HOSTS" != "null" ]]; then NEW_GATEWAY_HOSTS="$CURR_GATEWAY_HOSTS,$ADDED" else NEW_GATEWAY_HOSTS="$ADDED" fi log "⬆️ Adding: $ADDED" log "ℹ️ New otelcol_gateway_hosts: $NEW_GATEWAY_HOSTS" send_config_update "Modified%20OpenTelemetry%20Collector%20Gateway%20Hosts" "otelcol_gateway_hosts" "$NEW_GATEWAY_HOSTS" echo "✅ Added gateway host(s): $ADDED" } handle_remove_gateway_host() { [[ "$SHOW_HELP" == true ]] && usage_remove_gateway_host if [[ ${#HOSTS[@]} -eq 0 ]]; then echo "❌ Error: At least one --host is required." echo " Usage: $0 remove-gateway-host --host [--host ...]" exit 1 fi fetch_global_config CURR_GATEWAY_HOSTS=$(get_config_value "otelcol_gateway_hosts") if [[ -z "$CURR_GATEWAY_HOSTS" || "$CURR_GATEWAY_HOSTS" == "null" ]]; then echo "❌ Error: No gateway hosts are configured; nothing to remove." exit 1 fi REMOVABLE=() NOT_FOUND=() NEW_GATEWAY_HOSTS="$CURR_GATEWAY_HOSTS" for host in "${HOSTS[@]}"; do if echo "$CURR_GATEWAY_HOSTS" | tr ',' '\n' | grep -qxF "$host"; then REMOVABLE+=("$host") NEW_GATEWAY_HOSTS=$(echo "$NEW_GATEWAY_HOSTS" | tr ',' '\n' | grep -vxF "$host" | paste -sd ',' -) else NOT_FOUND+=("$host") fi done [[ ${#NOT_FOUND[@]} -gt 0 ]] && echo "⚠️ Not configured as gateway host(s) (skipping): ${NOT_FOUND[*]}" if [[ ${#REMOVABLE[@]} -eq 0 ]]; then echo "❌ Error: None of the specified host(s) are configured as gateway hosts; nothing removed." exit 1 fi REMOVED=$(IFS=','; echo "${REMOVABLE[*]}") log "⬆️ Removing: $REMOVED" log "ℹ️ New otelcol_gateway_hosts: ${NEW_GATEWAY_HOSTS:-}" send_config_update "Modified%20OpenTelemetry%20Collector%20Gateway%20Hosts" "otelcol_gateway_hosts" "$NEW_GATEWAY_HOSTS" echo "✅ Removed gateway host(s): $REMOVED" } handle_add_exporter() { [[ "$SHOW_HELP" == true ]] && usage_add_exporter require_config_file "add-exporter" fetch_global_config local curr_gateway_hosts curr_gateway_hosts=$(get_config_value "otelcol_gateway_hosts") if [[ -z "$curr_gateway_hosts" || "$curr_gateway_hosts" == "null" ]]; then echo "❌ No OTel gateway host is configured (otelcol_gateway_hosts is empty)." echo " At least one gateway host is mandatory before adding an exporter. Add one first:" echo " $0 add-gateway-host --host " echo " (or seed it together with prerequisites: $0 setup-prerequisites --host )" echo " The exporter was NOT added." exit 1 fi if [[ "${SETUP_PREREQS:-false}" == true ]]; then require_auth configure_prerequisites fetch_global_config # re-read so RAW_RESPONSE reflects the just-seeded values fi local THANOS_DISABLE=1 if should_disable_thanos; then THANOS_DISABLE=0; fi if add_exporter_prereqs_missing "$THANOS_DISABLE"; then echo "❌ Cloudera SaaS Observability / DMP prerequisites are not configured (${THANOS_DISABLE_REASON})." echo " The exporter was NOT added. Choose one:" echo " 1) Configure the real prerequisites (altus credentials, DMP flags," echo " remote_write_url) per Cloudera documentation, then re-run this command." echo " 2) Self-managed / air-gapped: seed dummy placeholder values (disables the" echo " DMP export path) by re-running with --setup-prerequisites:" echo " $0 add-exporter --file $CONFIG_FILE --setup-prerequisites" exit 1 fi CURR_VAL_CONNECTORS=$(get_config_value "$PARAM_CONNECTORS") UPDATED_CONNECTORS_VAL=$(get_connectors_or_default "$CURR_VAL_CONNECTORS") # Parse the exporter file once — the YAML is signal-agnostic. validate_yaml_indentation "$CONFIG_FILE" 0 # user file always has root keys at indent 0 FILE_EXPORTER_NAMES=() while IFS= read -r name; do FILE_EXPORTER_NAMES+=("$name") done < <(parse_names_from_normalized_config 0) if [[ ${#FILE_EXPORTER_NAMES[@]} -eq 0 ]]; then echo "❌ Error: Could not parse any exporter names from the config file." exit 1 fi log "ℹ️ Found ${#FILE_EXPORTER_NAMES[@]} exporter(s): ${FILE_EXPORTER_NAMES[*]}" AUTH_REFS=() while IFS= read -r ref; do [[ -n "$ref" ]] && AUTH_REFS+=("$ref") done < <(parse_authenticator_refs "$CONFIG_FILE") if [[ ${#AUTH_REFS[@]} -gt 0 ]]; then local curr_ext missing_ext=() curr_ext=$(get_config_value "$PARAM_EXTENSION") for ref in "${AUTH_REFS[@]}"; do extension_defined "$ref" "$curr_ext" || missing_ext+=("$ref") done if [[ ${#missing_ext[@]} -gt 0 ]]; then echo "❌ The exporter authenticates via an extension that is not configured:" for e in "${missing_ext[@]}"; do echo " - $e"; done echo " The exporter was NOT added. Add the extension first, then retry:" echo " $0 add-extension --file " exit 1 fi fi SEND_ARGS=() # accumulates per-signal exporter+service param/value pairs DISABLE_NOTES=() # per-signal gateway-lb pipelines dropped by should_disable_thanos for sig in "${SIGNALS[@]}"; do select_signal "$sig" local curr_exporter curr_service root_indent curr_exporter=$(get_config_value "$SIG_EXPORTER_PARAM") curr_service=$(get_config_value "$SIG_SERVICE_PARAM") if [[ -z "$curr_exporter" || "$curr_exporter" == "null" ]]; then echo "❌ Error: '$SIG_EXPORTER_PARAM' not found." exit 1 fi root_indent=$(get_exporter_root_indent "$curr_exporter") local dup=() sys=() for name in "${FILE_EXPORTER_NAMES[@]}"; do if is_system_exporter "$name"; then sys+=("$name") elif echo "$curr_exporter" | grep -qF "$name"; then dup+=("$name") fi done if [[ ${#sys[@]} -gt 0 ]]; then echo "❌ Error: System exporters cannot be modified ($sig):" for n in "${sys[@]}"; do echo " - $n"; done exit 1 fi if [[ ${#dup[@]} -gt 0 ]]; then echo "❌ Error: Exporter(s) already exist in $sig (use update-exporter to modify):" for n in "${dup[@]}"; do echo " - $n"; done exit 1 fi local scaled_exporter updated_exporter updated_service scaled_exporter=$(indent_block "$root_indent" "$(cat "$NORMALIZED_CONFIG")") updated_exporter="$curr_exporter $scaled_exporter" updated_service="$curr_service" UPDATED_CONNECTORS_VAL=$(ensure_fanout_connector "$UPDATED_CONNECTORS_VAL") updated_service=$(ensure_fanout_service_wiring "$updated_service") for name in "${FILE_EXPORTER_NAMES[@]}"; do is_system_exporter "$name" && continue local pipeline_name pipeline_name=$(exporter_to_pipeline "$name") if ! pipeline_exists "$updated_service" "$pipeline_name"; then updated_service=$(append_customer_pipeline "$updated_service" "$pipeline_name" "$name") UPDATED_CONNECTORS_VAL=$(fanout_add_pipeline "$UPDATED_CONNECTORS_VAL" "$pipeline_name") log "ℹ️ [$sig] Created pipeline '$pipeline_name' for exporter '$name' (fan-out duplicate)." fi done if [[ "$THANOS_DISABLE" == 0 ]]; then UPDATED_CONNECTORS_VAL=$(fanout_remove_pipeline "$UPDATED_CONNECTORS_VAL" "$GATEWAY_LB_PIPELINE" force) DISABLE_NOTES+=("$GATEWAY_LB_PIPELINE") fi SEND_ARGS+=("$SIG_EXPORTER_PARAM" "$updated_exporter" "$SIG_SERVICE_PARAM" "$updated_service") done if [[ ${#DISABLE_NOTES[@]} -gt 0 ]]; then echo "✅ Exporter configured — your third-party destination now receives all gateway telemetry." for pl in "${DISABLE_NOTES[@]}"; do echo " Disabled the built-in Cloudera SaaS/DMP fan-out ($pl) so exports run without 4xx/5xx." done if prerequisites_are_placeholder; then echo " Self-managed / air-gapped mode: the DMP export path stays disabled by design." else echo " DMP is not fully configured (${THANOS_DISABLE_REASON}); the fan-out now routes only to your exporter." fi fi log "⬆️ Sending exporter/service/connectors update..." send_config_update "Modified%20OTel%20Telemetry" \ "${SEND_ARGS[@]}" \ "$PARAM_CONNECTORS" "$UPDATED_CONNECTORS_VAL" } handle_update_exporter() { [[ "$SHOW_HELP" == true ]] && usage_update_exporter require_config_file "update-exporter" fetch_global_config CURR_VAL_EXPORTER=$(get_config_value "$PARAM_EXPORTER") CURR_VAL_SERVICE=$(get_config_value "$PARAM_SERVICE") if [[ -z "$CURR_VAL_EXPORTER" || "$CURR_VAL_EXPORTER" == "null" ]]; then echo "❌ Error: '$PARAM_EXPORTER' not found." exit 1 fi EXISTING_ROOT_INDENT=$(get_exporter_root_indent "$CURR_VAL_EXPORTER") validate_yaml_indentation "$CONFIG_FILE" 0 # user file always has root keys at indent 0 FILE_EXPORTER_NAMES=() while IFS= read -r name; do FILE_EXPORTER_NAMES+=("$name") done < <(parse_names_from_normalized_config 0) if [[ ${#FILE_EXPORTER_NAMES[@]} -eq 0 ]]; then echo "❌ Error: Could not parse any exporter names from the config file." exit 1 fi log "ℹ️ Found ${#FILE_EXPORTER_NAMES[@]} exporter(s): ${FILE_EXPORTER_NAMES[*]}" SYSTEM_NAMES=() NOT_FOUND_NAMES=() for name in "${FILE_EXPORTER_NAMES[@]}"; do if is_system_exporter "$name"; then SYSTEM_NAMES+=("$name") elif ! echo "$CURR_VAL_EXPORTER" | grep -qF "$name"; then NOT_FOUND_NAMES+=("$name") fi done if [[ ${#SYSTEM_NAMES[@]} -gt 0 ]]; then echo "❌ Error: System exporters cannot be modified:" for n in "${SYSTEM_NAMES[@]}"; do echo " - $n"; done exit 1 fi if [[ ${#NOT_FOUND_NAMES[@]} -gt 0 ]]; then echo "❌ Error: Exporter(s) not found (use add-exporter to add new ones):" for n in "${NOT_FOUND_NAMES[@]}"; do echo " - $n"; done exit 1 fi UPDATED_EXPORTER_VAL="$CURR_VAL_EXPORTER" for name in "${FILE_EXPORTER_NAMES[@]}"; do UPDATED_EXPORTER_VAL=$(remove_named_block "$UPDATED_EXPORTER_VAL" "$name" "$EXISTING_ROOT_INDENT") log "ℹ️ Replaced block for '$name'." done local scaled_exporter scaled_exporter=$(indent_block "$EXISTING_ROOT_INDENT" "$(cat "$NORMALIZED_CONFIG")") UPDATED_EXPORTER_VAL="$UPDATED_EXPORTER_VAL $scaled_exporter" log "ℹ️ Service pipeline unchanged (exporter names are the same)." log "⬆️ Sending exporter update..." send_config_update "Modified%20OTel%20Metrics" "$PARAM_EXPORTER" "$UPDATED_EXPORTER_VAL" "$PARAM_SERVICE" "$CURR_VAL_SERVICE" } handle_remove_exporter() { [[ "$SHOW_HELP" == true ]] && usage_remove_exporter if [[ ${#EXPORTERS[@]} -eq 0 ]]; then echo "❌ Error: At least one --exporter is required." echo " Usage: $0 remove-exporter --exporter [--exporter ...]" exit 1 fi fetch_global_config # Shared across signals: connectors + processor definitions. CURR_VAL_CONNECTORS=$(get_config_value "$PARAM_CONNECTORS") UPDATED_CONNECTORS_VAL=$(get_connectors_or_default "$CURR_VAL_CONNECTORS") CURR_VAL_PROCESSOR=$(get_config_value "$PARAM_PROCESSOR") PROC_ROOT_INDENT=$(get_child_indent "$CURR_VAL_PROCESSOR" "processors:") UPDATED_PROCESSOR_VAL="$CURR_VAL_PROCESSOR" # System-exporter guard (a name that matches any requested signal's reserved prefix). SYSTEM_NAMES=() for name in "${EXPORTERS[@]}"; do for sig in "${SIGNALS[@]}"; do select_signal "$sig" if is_system_exporter "$name"; then SYSTEM_NAMES+=("$name"); break; fi done done if [[ ${#SYSTEM_NAMES[@]} -gt 0 ]]; then echo "❌ Error: System exporters cannot be removed:" for n in "${SYSTEM_NAMES[@]}"; do echo " - $n"; done exit 1 fi FINAL_METRICS_SVC=$(get_config_value "$PARAM_SERVICE") FINAL_LOGS_SVC=$(get_config_value "$PARAM_SERVICE_LOGS") SEND_ARGS=() CANDIDATE_PROCS=() TORN_DOWN_SIGNALS=() local any_found=false for sig in "${SIGNALS[@]}"; do select_signal "$sig" local curr_exporter curr_service root_indent curr_exporter=$(get_config_value "$SIG_EXPORTER_PARAM") curr_service=$(get_config_value "$SIG_SERVICE_PARAM") # Signal family absent (never onboarded) — nothing to remove here. [[ -z "$curr_exporter" || "$curr_exporter" == "null" ]] && continue root_indent=$(get_exporter_root_indent "$curr_exporter") local exporter_names exporter_names=$(parse_exporter_names "$curr_exporter" "$root_indent") local updated_exporter="$curr_exporter" local updated_service="$curr_service" local changed=false for name in "${EXPORTERS[@]}"; do printf '%s\n' "$exporter_names" | grep -qxF "$name" || continue any_found=true; changed=true updated_exporter=$(remove_named_block "$updated_exporter" "$name" "$root_indent") local pipeline_name pipeline_name=$(exporter_to_pipeline "$name") while IFS= read -r pproc; do [[ -z "$pproc" ]] && continue CANDIDATE_PROCS+=("$pproc") done < <(echo "$curr_service" | awk \ -v pl="${pipeline_name}:" \ 'BEGIN{in_p=0} index($0,pl){in_p=1;next} in_p && /processors:/{ match($0,/\[.*\]/); if(RSTART>0){print substr($0,RSTART,RLENGTH)}; in_p=0 } in_p && /exporters:/{in_p=0}' \ | tr -d '[]' | tr ',' '\n' | sed 's/^[[:space:]]*//' | sed '/^$/d') updated_service=$(remove_customer_pipeline "$updated_service" "$pipeline_name") if echo "$UPDATED_CONNECTORS_VAL" | grep -qF "${FANOUT_CONNECTOR}:"; then UPDATED_CONNECTORS_VAL=$(fanout_remove_pipeline "$UPDATED_CONNECTORS_VAL" "$pipeline_name") fi log "ℹ️ [$sig] Removed exporter '$name' and pipeline '$pipeline_name'." done if [[ "$changed" == true ]] && echo "$UPDATED_CONNECTORS_VAL" | grep -qF "${FANOUT_CONNECTOR}:"; then local remaining remaining=$(fanout_table_pipelines "$UPDATED_CONNECTORS_VAL" | grep -vxF "$GATEWAY_LB_PIPELINE" | grep -c . || true) if [[ "$remaining" -eq 0 ]]; then UPDATED_CONNECTORS_VAL=$(teardown_fanout_connectors "$UPDATED_CONNECTORS_VAL") updated_service=$(teardown_fanout_service "$updated_service") TORN_DOWN_SIGNALS+=("$sig") log "ℹ️ [$sig] Removed the last third-party exporter — tore down the $sig fan-out (restored SaaS-only topology)." fi fi if [[ "$changed" == true ]]; then if [[ "$sig" == logs ]]; then FINAL_LOGS_SVC="$updated_service"; else FINAL_METRICS_SVC="$updated_service"; fi SEND_ARGS+=("$SIG_EXPORTER_PARAM" "$updated_exporter" "$SIG_SERVICE_PARAM" "$updated_service") fi done if [[ "$any_found" != true ]]; then echo "❌ Error: Exporter(s) not found in current config:" for n in "${EXPORTERS[@]}"; do echo " - $n"; done exit 1 fi if [[ ${#CANDIDATE_PROCS[@]} -gt 0 ]]; then local all_services="$FINAL_METRICS_SVC"$'\n'"$FINAL_LOGS_SVC" local seen_procs=" " for pname in "${CANDIDATE_PROCS[@]}"; do [[ -z "$pname" ]] && continue [[ "$seen_procs" == *" $pname "* ]] && continue # dedupe seen_procs+="$pname " still_used=$(echo "$all_services" | grep -cF "$pname" || true) if [[ "$still_used" -eq 0 ]]; then UPDATED_PROCESSOR_VAL=$(remove_named_block "$UPDATED_PROCESSOR_VAL" "$pname" "$PROC_ROOT_INDENT") log "ℹ️ Removed processor '$pname' (no longer referenced by any pipeline)." else log "ℹ️ Kept processor '$pname' (still used in $still_used other pipeline(s))." fi done fi if [[ "$UPDATED_PROCESSOR_VAL" != "$CURR_VAL_PROCESSOR" ]]; then log "⬆️ Sending exporter/processor/service/connectors update..." send_config_update "Modified%20OTel%20Telemetry" \ "${SEND_ARGS[@]}" \ "$PARAM_PROCESSOR" "$UPDATED_PROCESSOR_VAL" \ "$PARAM_CONNECTORS" "$UPDATED_CONNECTORS_VAL" else log "⬆️ Sending exporter/service/connectors update (processors unchanged)..." send_config_update "Modified%20OTel%20Telemetry" \ "${SEND_ARGS[@]}" \ "$PARAM_CONNECTORS" "$UPDATED_CONNECTORS_VAL" fi if [[ ${#TORN_DOWN_SIGNALS[@]} -gt 0 ]]; then fetch_cm_config if dmp_prereqs_are_dummy; then local dmp_flag dmp_seen=" " DMP_ARGS=() for sig in "${TORN_DOWN_SIGNALS[@]}"; do [[ "$sig" == logs ]] && dmp_flag="$PARAM_DMP_LOGS" || dmp_flag="$PARAM_DMP_METRICS" [[ "$dmp_seen" == *" $dmp_flag "* ]] && continue dmp_seen+="$dmp_flag " [[ "$(get_cm_config_value "$dmp_flag")" == "true" ]] && DMP_ARGS+=("$dmp_flag" "false") done if [[ ${#DMP_ARGS[@]} -gt 0 ]]; then echo "ℹ️ Prerequisites are placeholder/dummy — disabling the DMP export path so the" echo " restored SaaS-only topology doesn't fan out to an unreachable destination." set_cm_config "Disabled%20DMP%20export%20(placeholder%20prerequisites)" "${DMP_ARGS[@]}" fi fi fi } handle_list_processors() { [[ "$SHOW_HELP" == true ]] && usage_list_processors fetch_global_config CURR_VAL_PROCESSOR=$(get_config_value "$PARAM_PROCESSOR") if [[ -z "$CURR_VAL_PROCESSOR" || "$CURR_VAL_PROCESSOR" == "null" ]]; then echo "No processors configured." exit 0 fi EXISTING_ROOT_INDENT=$(get_child_indent "$CURR_VAL_PROCESSOR" "processors:") # Processors are shared across signals, so a link can live in either service # param; index both before reporting. local links links=$(printf '%s\n%s\n' \ "$(service_processor_links "$(get_config_value "$PARAM_SERVICE")")" \ "$(service_processor_links "$(get_config_value "$PARAM_SERVICE_LOGS")")") echo "" printf "%-45s %-12s %s\n" "PROCESSOR NAME" "TYPE" "LINKED PIPELINES" printf "%-45s %-12s %s\n" "--------------" "----" "----------------" local pname linked ptype while IFS= read -r pname; do linked=$(awk -F'\t' -v n="$pname" '$2 == n { print $1 }' <<< "$links" \ | awk '!seen[$0]++' | paste -sd, - | sed 's/,/, /g') if echo "$pname" | grep -qE '^(filter|pidfilter|metricstransform)/'; then ptype="system" else ptype="customer" fi printf "%-45s %-12s %s\n" "$pname" "$ptype" "${linked:-(not linked)}" done < <(parse_exporter_names "$CURR_VAL_PROCESSOR" "$EXISTING_ROOT_INDENT") echo "" } handle_add_processor() { [[ "$SHOW_HELP" == true ]] && usage_add_processor if [[ -z "$CONFIG_FILE" && ${#PROCESSORS[@]} -eq 0 ]]; then echo "❌ Error: Provide --file to define a new processor, or" echo " --processor --exporter to link an existing one." echo " Usage: $0 add-processor --help" exit 1 fi if [[ -n "$CONFIG_FILE" && ${#PROCESSORS[@]} -gt 0 ]]; then echo "❌ Error: --file and --processor cannot be used together." echo " Use --file to define a new processor, or" echo " --processor to link an already-defined one." exit 1 fi if [[ ${#PROCESSORS[@]} -gt 0 && ${#EXPORTERS[@]} -eq 0 ]]; then echo "❌ Error: --exporter is required when using --processor." exit 1 fi if [[ -n "$CONFIG_FILE" && ! -f "$CONFIG_FILE" ]]; then echo "❌ Error: File '$CONFIG_FILE' not found." exit 1 fi fetch_global_config CURR_VAL_PROCESSOR=$(get_config_value "$PARAM_PROCESSOR") METRICS_SVC=$(get_config_value "$PARAM_SERVICE") LOGS_SVC=$(get_config_value "$PARAM_SERVICE_LOGS") if [[ -z "$CURR_VAL_PROCESSOR" || "$CURR_VAL_PROCESSOR" == "null" ]]; then echo "❌ Error: '$PARAM_PROCESSOR' not found in CM config." exit 1 fi EXISTING_ROOT_INDENT=$(get_child_indent "$CURR_VAL_PROCESSOR" "processors:") UPDATED_PROCESSOR_VAL="$CURR_VAL_PROCESSOR" LINK_NAMES=() if [[ -n "$CONFIG_FILE" ]]; then validate_yaml_indentation "$CONFIG_FILE" 0 FILE_PROCESSOR_NAMES=() while IFS= read -r pname; do FILE_PROCESSOR_NAMES+=("$pname") done < <(parse_names_from_normalized_config "$FILE_ROOT_INDENT") if [[ ${#FILE_PROCESSOR_NAMES[@]} -eq 0 ]]; then echo "❌ Error: Could not parse any processor names from the config file." exit 1 fi log "ℹ️ Found ${#FILE_PROCESSOR_NAMES[@]} processor(s): ${FILE_PROCESSOR_NAMES[*]}" DUPLICATE_PROC=() for pname in "${FILE_PROCESSOR_NAMES[@]}"; do if component_defined "$pname" "$CURR_VAL_PROCESSOR" "processors:"; then DUPLICATE_PROC+=("$pname") fi done if [[ ${#DUPLICATE_PROC[@]} -gt 0 ]]; then echo "❌ Error: Processor(s) already exist (use update-processor to modify):" for n in "${DUPLICATE_PROC[@]}"; do echo " - $n"; done exit 1 fi local indented_procs indented_procs=$(indent_block "$EXISTING_ROOT_INDENT" "$(cat "$NORMALIZED_CONFIG")") UPDATED_PROCESSOR_VAL="$CURR_VAL_PROCESSOR $indented_procs" LINK_NAMES=("${FILE_PROCESSOR_NAMES[@]}") fi if [[ ${#PROCESSORS[@]} -gt 0 ]]; then NOT_FOUND_PROC=() for pname in "${PROCESSORS[@]}"; do if ! component_defined "$pname" "$CURR_VAL_PROCESSOR" "processors:"; then NOT_FOUND_PROC+=("$pname") fi done if [[ ${#NOT_FOUND_PROC[@]} -gt 0 ]]; then echo "❌ Error: Processor(s) not found in otelcol_processors:" for n in "${NOT_FOUND_PROC[@]}"; do echo " - $n"; done echo " Use add-processor --file to define them first." exit 1 fi LINK_NAMES=("${PROCESSORS[@]}") fi CHANGED_METRICS=false CHANGED_LOGS=false if [[ ${#EXPORTERS[@]} -gt 0 ]]; then local matched="" # newline-delimited exporters that matched >=1 pipeline for sig in "${SIGNALS[@]}"; do select_signal "$sig" local svc sig_changed=false if [[ "$sig" == logs ]]; then svc="$LOGS_SVC"; else svc="$METRICS_SVC"; fi for exporter_name in "${EXPORTERS[@]}"; do pl=$(exporter_to_pipeline "$exporter_name") pipeline_exists "$svc" "$pl" || continue matched="$matched $exporter_name" for pname in "${LINK_NAMES[@]}"; do if pipeline_field_contains "$svc" "$pl" "processors" "$pname"; then echo "⚠️ Processor '$pname' is already linked to pipeline '$pl'. Skipping." continue fi svc=$(add_to_pipeline_field "$svc" "$pl" "processors" "$pname") sig_changed=true log "ℹ️ [$sig] Linked '$pname' to pipeline '$pl'." done done if [[ "$sig_changed" == true ]]; then if [[ "$sig" == logs ]]; then LOGS_SVC="$svc"; CHANGED_LOGS=true; else METRICS_SVC="$svc"; CHANGED_METRICS=true; fi fi done UNMATCHED=() for exporter_name in "${EXPORTERS[@]}"; do if ! printf '%s\n' "$matched" | grep -qxF "$exporter_name"; then UNMATCHED+=("$exporter_name") fi done if [[ ${#UNMATCHED[@]} -gt 0 ]]; then echo "❌ Error: No pipeline found for exporter(s) in the requested signal(s) (${SIGNALS[*]}). Run add-exporter first:" for n in "${UNMATCHED[@]}"; do echo " - $n"; done exit 1 fi else log "ℹ️ Processor definition added to otelcol_processors (not linked to any pipeline yet)." fi log "⬆️ Sending processor/service update..." SEND_ARGS=("$PARAM_PROCESSOR" "$UPDATED_PROCESSOR_VAL") [[ "$CHANGED_METRICS" == true ]] && SEND_ARGS+=("$PARAM_SERVICE" "$METRICS_SVC") [[ "$CHANGED_LOGS" == true ]] && SEND_ARGS+=("$PARAM_SERVICE_LOGS" "$LOGS_SVC") send_config_update "Modified%20OTel%20Processors" "${SEND_ARGS[@]}" } handle_update_processor() { [[ "$SHOW_HELP" == true ]] && usage_update_processor require_config_file "update-processor" fetch_global_config CURR_VAL_PROCESSOR=$(get_config_value "$PARAM_PROCESSOR") if [[ -z "$CURR_VAL_PROCESSOR" || "$CURR_VAL_PROCESSOR" == "null" ]]; then echo "❌ Error: '$PARAM_PROCESSOR' not found." exit 1 fi EXISTING_ROOT_INDENT=$(get_child_indent "$CURR_VAL_PROCESSOR" "processors:") validate_yaml_indentation "$CONFIG_FILE" 0 FILE_PROCESSOR_NAMES=() while IFS= read -r pname; do FILE_PROCESSOR_NAMES+=("$pname") done < <(parse_names_from_normalized_config "$FILE_ROOT_INDENT") if [[ ${#FILE_PROCESSOR_NAMES[@]} -eq 0 ]]; then echo "❌ Error: Could not parse any processor names from the config file." exit 1 fi NOT_FOUND_PROC=() for pname in "${FILE_PROCESSOR_NAMES[@]}"; do if ! echo "$CURR_VAL_PROCESSOR" | grep -qF "$pname"; then NOT_FOUND_PROC+=("$pname") fi done if [[ ${#NOT_FOUND_PROC[@]} -gt 0 ]]; then echo "❌ Error: Processor(s) not found (use add-processor to add new ones):" for n in "${NOT_FOUND_PROC[@]}"; do echo " - $n"; done exit 1 fi UPDATED_PROCESSOR_VAL="$CURR_VAL_PROCESSOR" for pname in "${FILE_PROCESSOR_NAMES[@]}"; do UPDATED_PROCESSOR_VAL=$(remove_named_block "$UPDATED_PROCESSOR_VAL" "$pname" "$EXISTING_ROOT_INDENT") log "ℹ️ Replaced block for processor '$pname'." done local indented_procs indented_procs=$(indent_block "$EXISTING_ROOT_INDENT" "$(cat "$NORMALIZED_CONFIG")") UPDATED_PROCESSOR_VAL="$UPDATED_PROCESSOR_VAL $indented_procs" log "ℹ️ Service pipeline unchanged (processor names are the same)." log "⬆️ Sending processor update..." send_config_update "Modified%20OTel%20Processors" "$PARAM_PROCESSOR" "$UPDATED_PROCESSOR_VAL" } handle_remove_processor() { [[ "$SHOW_HELP" == true ]] && usage_remove_processor if [[ ${#PROCESSORS[@]} -eq 0 ]]; then echo "❌ Error: At least one --processor is required." echo " Usage: $0 remove-processor --help" exit 1 fi fetch_global_config CURR_VAL_PROCESSOR=$(get_config_value "$PARAM_PROCESSOR") METRICS_SVC=$(get_config_value "$PARAM_SERVICE") LOGS_SVC=$(get_config_value "$PARAM_SERVICE_LOGS") if [[ -z "$CURR_VAL_PROCESSOR" || "$CURR_VAL_PROCESSOR" == "null" ]]; then echo "❌ Error: '$PARAM_PROCESSOR' not found." exit 1 fi EXISTING_ROOT_INDENT=$(get_child_indent "$CURR_VAL_PROCESSOR" "processors:") NOT_FOUND_PROC=() for pname in "${PROCESSORS[@]}"; do if ! component_defined "$pname" "$CURR_VAL_PROCESSOR" "processors:"; then NOT_FOUND_PROC+=("$pname") fi done if [[ ${#NOT_FOUND_PROC[@]} -gt 0 ]]; then echo "❌ Error: Processor(s) not found in current config:" for n in "${NOT_FOUND_PROC[@]}"; do echo " - $n"; done exit 1 fi UPDATED_PROCESSOR_VAL="$CURR_VAL_PROCESSOR" CHANGED_METRICS=false CHANGED_LOGS=false if [[ ${#EXPORTERS[@]} -gt 0 ]]; then local matched_pipeline=false for sig in "${SIGNALS[@]}"; do select_signal "$sig" local svc sig_changed=false if [[ "$sig" == logs ]]; then svc="$LOGS_SVC"; else svc="$METRICS_SVC"; fi for exporter_name in "${EXPORTERS[@]}"; do pl=$(exporter_to_pipeline "$exporter_name") pipeline_exists "$svc" "$pl" || continue matched_pipeline=true for pname in "${PROCESSORS[@]}"; do if pipeline_field_contains "$svc" "$pl" "processors" "$pname"; then svc=$(remove_from_pipeline_field "$svc" "$pl" "processors" "$pname") sig_changed=true log "ℹ️ [$sig] Unlinked '$pname' from pipeline '$pl' (definition kept)." fi done done if [[ "$sig_changed" == true ]]; then if [[ "$sig" == logs ]]; then LOGS_SVC="$svc"; CHANGED_LOGS=true; else METRICS_SVC="$svc"; CHANGED_METRICS=true; fi fi done # A wrong --exporter (e.g. a pipeline name) matches nothing; without this the # command would fall through and report success having changed nothing. if [[ "$matched_pipeline" != true ]]; then echo "❌ Error: No matching pipeline found for exporter(s): ${EXPORTERS[*]}" echo " Pass the exporter name (e.g. file/demo), not the pipeline name." echo " Run '$0 list-exporters' to see the configured exporter names." exit 1 fi if [[ "$CHANGED_METRICS" != true && "$CHANGED_LOGS" != true ]]; then echo "ℹ️ Nothing to unlink: processor(s) [${PROCESSORS[*]}] are not linked to the" echo " pipeline(s) for exporter(s) [${EXPORTERS[*]}] in signal(s) [${SIGNALS[*]}]. Nothing changed." exit 0 fi else # Delete the definition(s) and scrub every pipeline in every requested signal. for pname in "${PROCESSORS[@]}"; do UPDATED_PROCESSOR_VAL=$(remove_named_block "$UPDATED_PROCESSOR_VAL" "$pname" "$EXISTING_ROOT_INDENT") done for sig in "${SIGNALS[@]}"; do select_signal "$sig" local svc sig_changed=false if [[ "$sig" == logs ]]; then svc="$LOGS_SVC"; else svc="$METRICS_SVC"; fi while IFS= read -r line; do if echo "$line" | grep -qE "^[[:space:]]+${SIG_PREFIX}/[^[:space:]]+:"; then pl=$(echo "$line" | sed 's/:[[:space:]]*$//' | sed 's/^[[:space:]]*//') for pname in "${PROCESSORS[@]}"; do if pipeline_field_contains "$svc" "$pl" "processors" "$pname"; then svc=$(remove_from_pipeline_field "$svc" "$pl" "processors" "$pname") sig_changed=true fi done fi done <<< "$svc" if [[ "$sig_changed" == true ]]; then if [[ "$sig" == logs ]]; then LOGS_SVC="$svc"; CHANGED_LOGS=true; else METRICS_SVC="$svc"; CHANGED_METRICS=true; fi fi done log "ℹ️ Deleted processor definition(s) and removed from all pipelines." fi log "⬆️ Sending processor/service update..." SEND_ARGS=("$PARAM_PROCESSOR" "$UPDATED_PROCESSOR_VAL") [[ "$CHANGED_METRICS" == true ]] && SEND_ARGS+=("$PARAM_SERVICE" "$METRICS_SVC") [[ "$CHANGED_LOGS" == true ]] && SEND_ARGS+=("$PARAM_SERVICE_LOGS" "$LOGS_SVC") send_config_update "Modified%20OTel%20Processors" "${SEND_ARGS[@]}" } handle_list_extensions() { [[ "$SHOW_HELP" == true ]] && usage_list_extensions fetch_global_config CURR_VAL_EXTENSION=$(get_config_value "$PARAM_EXTENSION") if [[ -z "$CURR_VAL_EXTENSION" || "$CURR_VAL_EXTENSION" == "null" ]]; then echo "No extensions configured." exit 0 fi EXISTING_ROOT_INDENT=$(get_child_indent "$CURR_VAL_EXTENSION" "extensions:") echo "" printf "%s\n" "EXTENSION NAME" printf "%s\n" "--------------" while IFS= read -r pname; do printf "%s\n" "$pname" done < <(parse_exporter_names "$CURR_VAL_EXTENSION" "$EXISTING_ROOT_INDENT") echo "" } handle_add_extension() { [[ "$SHOW_HELP" == true ]] && usage_add_extension require_config_file "add-extension" fetch_global_config CURR_VAL_EXTENSION=$(get_config_value "$PARAM_EXTENSION") CURR_VAL_SERVICE=$(get_config_value "$PARAM_SERVICE") if [[ -z "$CURR_VAL_EXTENSION" || "$CURR_VAL_EXTENSION" == "null" ]]; then echo "❌ Error: '$PARAM_EXTENSION' not found." exit 1 fi EXISTING_ROOT_INDENT=$(get_child_indent "$CURR_VAL_EXTENSION" "extensions:") validate_yaml_indentation "$CONFIG_FILE" 0 FILE_EXTENSION_NAMES=() while IFS= read -r name; do FILE_EXTENSION_NAMES+=("$name") done < <(parse_names_from_normalized_config "$FILE_ROOT_INDENT") if [[ ${#FILE_EXTENSION_NAMES[@]} -eq 0 ]]; then echo "❌ Error: Could not parse any extension names from the config file." exit 1 fi log "ℹ️ Found ${#FILE_EXTENSION_NAMES[@]} extension(s): ${FILE_EXTENSION_NAMES[*]}" DUPLICATE_NAMES=() SYSTEM_NAMES=() for name in "${FILE_EXTENSION_NAMES[@]}"; do if is_system_extension "$name"; then SYSTEM_NAMES+=("$name") elif echo "$CURR_VAL_EXTENSION" | grep -qF "$name"; then DUPLICATE_NAMES+=("$name") fi done if [[ ${#SYSTEM_NAMES[@]} -gt 0 ]]; then echo "❌ Error: System extensions cannot be modified:" for n in "${SYSTEM_NAMES[@]}"; do echo " - $n"; done exit 1 fi if [[ ${#DUPLICATE_NAMES[@]} -gt 0 ]]; then echo "❌ Error: Extension(s) already exist (use update-extension to modify):" for n in "${DUPLICATE_NAMES[@]}"; do echo " - $n"; done exit 1 fi local indented_exts indented_exts=$(indent_block "$EXISTING_ROOT_INDENT" "$(cat "$NORMALIZED_CONFIG")") UPDATED_EXTENSION_VAL="$CURR_VAL_EXTENSION $indented_exts" send_config_update "Modified%20OTel%20Extensions" "$PARAM_EXTENSION" "$UPDATED_EXTENSION_VAL" } handle_update_extension() { [[ "$SHOW_HELP" == true ]] && usage_update_extension require_config_file "update-extension" fetch_global_config CURR_VAL_EXTENSION=$(get_config_value "$PARAM_EXTENSION") if [[ -z "$CURR_VAL_EXTENSION" || "$CURR_VAL_EXTENSION" == "null" ]]; then echo "❌ Error: '$PARAM_EXTENSION' not found." exit 1 fi EXISTING_ROOT_INDENT=$(get_child_indent "$CURR_VAL_EXTENSION" "extensions:") validate_yaml_indentation "$CONFIG_FILE" 0 FILE_EXTENSION_NAMES=() while IFS= read -r pname; do FILE_EXTENSION_NAMES+=("$pname") done < <(parse_names_from_normalized_config "$FILE_ROOT_INDENT") if [[ ${#FILE_EXTENSION_NAMES[@]} -eq 0 ]]; then echo "❌ Error: Could not parse any extension names from the config file." exit 1 fi NOT_FOUND_NAMES=() for pname in "${FILE_EXTENSION_NAMES[@]}"; do if ! echo "$CURR_VAL_EXTENSION" | grep -qF "$pname"; then NOT_FOUND_NAMES+=("$pname") fi done if [[ ${#NOT_FOUND_NAMES[@]} -gt 0 ]]; then echo "❌ Error: Extension(s) not found (use add-extension to add new ones):" for n in "${NOT_FOUND_NAMES[@]}"; do echo " - $n"; done exit 1 fi UPDATED_EXTENSION_VAL="$CURR_VAL_EXTENSION" for pname in "${FILE_EXTENSION_NAMES[@]}"; do UPDATED_EXTENSION_VAL=$(remove_named_block "$UPDATED_EXTENSION_VAL" "$pname" "$EXISTING_ROOT_INDENT") log "ℹ️ Replaced block for extension '$pname'." done local indented_exts indented_exts=$(indent_block "$EXISTING_ROOT_INDENT" "$(cat "$NORMALIZED_CONFIG")") UPDATED_EXTENSION_VAL="$UPDATED_EXTENSION_VAL $indented_exts" log "ℹ️ Service pipeline unchanged (extension names are the same)." log "⬆️ Sending extension update..." send_config_update "Modified%20OTel%20Extensions" "$PARAM_EXTENSION" "$UPDATED_EXTENSION_VAL" } handle_remove_extension() { [[ "$SHOW_HELP" == true ]] && usage_remove_extension if [[ ${#EXTENSIONS[@]} -eq 0 ]]; then echo "❌ Error: At least one --extension is required." echo " Usage: $0 remove-extension --help" exit 1 fi fetch_global_config CURR_VAL_EXTENSION=$(get_config_value "$PARAM_EXTENSION") CURR_VAL_SERVICE=$(get_config_value "$PARAM_SERVICE") if [[ -z "$CURR_VAL_EXTENSION" || "$CURR_VAL_EXTENSION" == "null" ]]; then echo "❌ Error: '$PARAM_EXTENSION' not found." exit 1 fi EXISTING_ROOT_INDENT=$(get_child_indent "$CURR_VAL_EXTENSION" "extensions:") NOT_FOUND_PROC=() for pname in "${EXTENSIONS[@]}"; do if ! echo "$CURR_VAL_EXTENSION" | grep -qF "$pname"; then NOT_FOUND_PROC+=("$pname") fi done if [[ ${#NOT_FOUND_PROC[@]} -gt 0 ]]; then echo "❌ Error: Extension(s) not found in current config:" for n in "${NOT_FOUND_PROC[@]}"; do echo " - $n"; done exit 1 fi if [[ ${#EXPORTERS[@]} -gt 0 ]]; then for exporter_name in "${EXPORTERS[@]}"; do pl=$(exporter_to_pipeline "$exporter_name") if ! pipeline_exists "$CURR_VAL_SERVICE" "$pl"; then echo "❌ Error: No pipeline found for exporter '$exporter_name'." exit 1 fi done fi UPDATED_EXTENSION_VAL="$CURR_VAL_EXTENSION" UPDATED_SERVICE_VAL="$CURR_VAL_SERVICE" for pname in "${EXTENSIONS[@]}"; do if [[ ${#EXPORTERS[@]} -gt 0 ]]; then for exporter_name in "${EXPORTERS[@]}"; do pl=$(exporter_to_pipeline "$exporter_name") UPDATED_SERVICE_VAL=$(remove_from_pipeline_field "$UPDATED_SERVICE_VAL" "$pl" "extensions" "$pname") log "ℹ️ Unlinked '$pname' from pipeline '$pl' (definition kept)." done else UPDATED_EXTENSION_VAL=$(remove_named_block "$UPDATED_EXTENSION_VAL" "$pname" "$EXISTING_ROOT_INDENT") while IFS= read -r line; do if echo "$line" | grep -qE '^[[:space:]]+metrics/[^[:space:]]+:'; then pl=$(echo "$line" | sed 's/:[[:space:]]*$//' | sed 's/^[[:space:]]*//') UPDATED_SERVICE_VAL=$(remove_from_pipeline_field "$UPDATED_SERVICE_VAL" "$pl" "extensions" "$pname") fi done <<< "$CURR_VAL_SERVICE" log "ℹ️ Deleted '$pname' definition and removed from all pipelines." fi done log "⬆️ Sending extension/service update..." send_config_update "Modified%20OTel%20Extensions" "$PARAM_EXTENSION" "$UPDATED_EXTENSION_VAL" "$PARAM_SERVICE" "$UPDATED_SERVICE_VAL" } RAW_CM_CONFIG="" fetch_cm_config() { RAW_CM_CONFIG=$(curl -k -s -u "$CM_USER:$CM_PASS" \ "$CM_BASE_URL/api/v31/cm/config?view=full") } get_cm_config_value() { local name="$1" echo "$RAW_CM_CONFIG" | jq -r --arg T "$name" \ '.items[]? | select((.name|ascii_downcase) == ($T|ascii_downcase)) | if .value != null then .value else .default end' 2>/dev/null } altus_creds_present() { local acct="$1" resp ak [[ -z "$acct" || "$acct" == "null" || "$acct" == "dummy" ]] && return 1 resp=$(curl -k -s -u "$CM_USER:$CM_PASS" \ "$CM_BASE_URL/api/v16/externalAccounts/type/ALTUS_ACCESS_KEY_AUTH?view=EXPORT_REDACTED") ak=$(echo "$resp" | jq -r --arg n "$acct" \ '((.items // [])[] | select(.name==$n) | .accountConfigs.items[]? | select(.name=="access_key_id") | .value) // ""' 2>/dev/null) # Present only if we positively read a real, non-dummy access key. [[ -n "$ak" && "$ak" != "dummy" && "$ak" != "null" ]] } should_disable_thanos() { fetch_cm_config local dmp_m dmp_l altus rw dmp_m=$(get_cm_config_value "$PARAM_DMP_METRICS") dmp_l=$(get_cm_config_value "$PARAM_DMP_LOGS") altus=$(get_cm_config_value "$PARAM_ALTUS_ACCOUNT") rw=$(get_config_value "$PARAM_REMOTE_WRITE_URL") THANOS_DISABLE_REASON="" [[ "$dmp_m" != "true" ]] && { THANOS_DISABLE_REASON="$PARAM_DMP_METRICS!=true"; return 0; } [[ "$dmp_l" != "true" ]] && { THANOS_DISABLE_REASON="$PARAM_DMP_LOGS!=true"; return 0; } [[ -z "$rw" || "$rw" == "null" ]] && { THANOS_DISABLE_REASON="remote_write_url missing"; return 0; } [[ "$rw" == "$DUMMY_REMOTE_WRITE_URL" ]] && { THANOS_DISABLE_REASON="remote_write_url is dummy"; return 0; } if ! altus_creds_present "$altus"; then THANOS_DISABLE_REASON="altus credentials missing/dummy"; return 0; fi return 1 } prerequisites_are_placeholder() { [[ "$(get_cm_config_value "$PARAM_ALTUS_ACCOUNT")" == "$ALTUS_ACCOUNT_NAME" ]] \ && [[ "$(get_cm_config_value "$PARAM_DMP_METRICS")" == "true" ]] \ && [[ "$(get_cm_config_value "$PARAM_DMP_LOGS")" == "true" ]] \ && [[ "$(get_config_value "$PARAM_REMOTE_WRITE_URL")" == "$DUMMY_REMOTE_WRITE_URL" ]] } dmp_prereqs_are_dummy() { local rw altus rw=$(get_config_value "$PARAM_REMOTE_WRITE_URL") altus=$(get_cm_config_value "$PARAM_ALTUS_ACCOUNT") [[ -z "$rw" || "$rw" == "null" || "$rw" == "$DUMMY_REMOTE_WRITE_URL" ]] && return 0 altus_creds_present "$altus" || return 0 # creds_present rc!=0 => dummy/missing return 1 } add_exporter_prereqs_missing() { local thanos_disable if [[ -n "${1:-}" ]]; then thanos_disable="$1" else if should_disable_thanos; then thanos_disable=0; else thanos_disable=1; fi fi [[ "$thanos_disable" == 0 ]] \ && ! prerequisites_are_placeholder \ && [[ "${SETUP_PREREQS:-false}" != true ]] } set_cm_config() { local message="$1"; shift local items_json="[]" while [[ $# -ge 2 ]]; do items_json=$(echo "$items_json" | jq --arg n "$1" --arg v "$2" '. + [{name:$n,value:$v}]') shift 2 done local body payload body=$(jq -n --argjson items "$items_json" '{items:$items}') payload=$(jq -n \ --arg url "/api/v31/cm/config?message=${message}" \ --argjson body "$body" \ '{items:[{method:"PUT",url:$url,body:$body,contentType:"application/json"}]}') send_batch "$payload" } # True if an ALTUS_ACCESS_KEY_AUTH external account with the given name exists. altus_account_exists() { local name="${1:-$ALTUS_ACCOUNT_NAME}" resp resp=$(curl -k -s -u "$CM_USER:$CM_PASS" \ "$CM_BASE_URL/api/v16/externalAccounts/type/ALTUS_ACCESS_KEY_AUTH?view=EXPORT_REDACTED") echo "$resp" | jq -e --arg n "$name" '(.items // [])[] | select(.name == $n)' >/dev/null 2>&1 } # Build the externalAccounts/create JSON payload (pure; testable). altus_create_payload() { local name="${1:-$ALTUS_ACCOUNT_NAME}" access="${2:-dummy}" priv="${3:-dummy}" ts="${4:-}" [[ -z "$ts" ]] && ts=$(date -u +%Y-%m-%dT%H:%M:%S.000Z) jq -n --arg n "$name" --arg ts "$ts" --arg a "$access" --arg p "$priv" \ '{name:$n, displayName:$n, createdTime:$ts, lastModifiedTime:$ts, accountConfigs:{items:[{name:"access_key_id",value:$a}, {name:"private_key",value:( $p | if test("BEGIN") then gsub("\\\\n"; "\n") | split("\n") | map(select(length>0)) | .[0] + "\n" + (.[1:-1]|join("")) + "\n" + .[-1] else . end | gsub("\n"; "\\n") )}]}, typeName:"ALTUS_ACCESS_KEY_AUTH"}' } # Create a dummy ALTUS_ACCESS_KEY_AUTH external account. create_altus_account() { local name="${1:-$ALTUS_ACCOUNT_NAME}" access="${2:-dummy}" priv="${3:-dummy}" local payload tmp code payload=$(altus_create_payload "$name" "$access" "$priv") tmp=$(mktemp) code=$(curl -k -s -o "$tmp" -w "%{http_code}" -u "$CM_USER:$CM_PASS" \ -X POST "$CM_BASE_URL/api/v16/externalAccounts/create" \ -H "Content-Type: application/json" -d "$payload") if [[ "$code" -ge 200 && "$code" -lt 300 ]]; then log "✅ Created altus external account '$name'." else echo "❌ Failed to create altus account '$name' (HTTP $code)." cat "$tmp"; rm -f "$tmp"; exit 1 fi rm -f "$tmp" } fetch_mgmt_roles() { curl -k -s -u "$CM_USER:$CM_PASS" "$CM_BASE_URL/api/v31/cm/service/roles" } tp_role_exists() { local resp="${1:-$(fetch_mgmt_roles)}" echo "$resp" | jq -e --arg t "$TP_ROLE_TYPE" \ '(.items // [])[] | select(.type == $t)' >/dev/null 2>&1 } mgmt_host_id() { local resp="${1:-$(fetch_mgmt_roles)}" echo "$resp" | jq -r '(.items // [])[].hostRef.hostId // empty' 2>/dev/null | head -1 } create_tp_role() { local host_id="$1" payload tmp code payload=$(jq -n --arg t "$TP_ROLE_TYPE" --arg h "$host_id" \ '{items:[{type:$t, hostRef:{hostId:$h}}]}') tmp=$(mktemp) code=$(curl -k -s -o "$tmp" -w "%{http_code}" -u "$CM_USER:$CM_PASS" \ -X POST "$CM_BASE_URL/api/v31/cm/service/roles" \ -H "Content-Type: application/json" -d "$payload") if [[ "$code" -ge 200 && "$code" -lt 300 ]]; then log "✅ Created TelemetryPublisher role on hostId '$host_id'." else echo "❌ Failed to create TelemetryPublisher role (HTTP $code)." cat "$tmp"; rm -f "$tmp"; exit 1 fi rm -f "$tmp" } # Echo the current telemetrypublisher_safety_valve override ("" if unset). tp_safety_valve_current() { local resp resp=$(curl -k -s -u "$CM_USER:$CM_PASS" \ "$CM_BASE_URL/api/v31/cm/service/roleConfigGroups/$TP_ROLE_CONFIG_GROUP/config") echo "$resp" | jq -r --arg n "$PARAM_TP_SAFETY_VALVE" \ '(.items // [])[] | select(.name == $n) | (.value // "")' 2>/dev/null } # Set the telemetrypublisher_safety_valve on the TP base role config group. set_tp_safety_valve() { local value="$1" payload tmp code payload=$(jq -n --arg n "$PARAM_TP_SAFETY_VALVE" --arg v "$value" \ '{items:[{name:$n, value:$v}]}') tmp=$(mktemp) code=$(curl -k -s -o "$tmp" -w "%{http_code}" -u "$CM_USER:$CM_PASS" \ -X PUT "$CM_BASE_URL/api/v31/cm/service/roleConfigGroups/$TP_ROLE_CONFIG_GROUP/config" \ -H "Content-Type: application/json" -d "$payload") if [[ "$code" -ge 200 && "$code" -lt 300 ]]; then log "✅ Set $PARAM_TP_SAFETY_VALVE (dummy) on $TP_ROLE_CONFIG_GROUP." else echo "❌ Failed to set $PARAM_TP_SAFETY_VALVE (HTTP $code)." cat "$tmp"; rm -f "$tmp"; exit 1 fi rm -f "$tmp" } configure_telemetrypublisher() { local tp_changed=1 roles roles=$(fetch_mgmt_roles) # single GET /cm/service/roles, reused by both checks below if tp_role_exists "$roles"; then log "ℹ️ TelemetryPublisher role already present — leaving as-is." else local host_id host_id=$(mgmt_host_id "$roles") [[ -z "$host_id" ]] && { echo "❌ Could not determine an mgmt host for the TelemetryPublisher role."; exit 1; } create_tp_role "$host_id" echo " • cm/service/roles: created $TP_ROLE_TYPE on hostId=$host_id (role is stopped — start it or run --refresh)" tp_changed=0 fi local current current=$(tp_safety_valve_current) if [[ -z "$current" ]]; then set_tp_safety_valve "$DUMMY_TP_SAFETY_VALVE" echo " • $TP_ROLE_CONFIG_GROUP: set dummy $PARAM_TP_SAFETY_VALVE" tp_changed=0 else log "ℹ️ $PARAM_TP_SAFETY_VALVE already set — leaving as-is." fi return $tp_changed } configure_prerequisites() { echo "⚙️ Checking prerequisites (non-destructive; only missing values are added)..." fetch_cm_config local altus dmp_m dmp_l rw_override altus=$(get_cm_config_value "$PARAM_ALTUS_ACCOUNT") dmp_m=$(get_cm_config_value "$PARAM_DMP_METRICS") dmp_l=$(get_cm_config_value "$PARAM_DMP_LOGS") fetch_global_config rw_override=$(echo "$RAW_RESPONSE" | jq -r --arg T "$PARAM_REMOTE_WRITE_URL" '.items[]? | select(.name==$T) | (.value // "")' 2>/dev/null) local cm_args=() changed=0 if [[ -z "$altus" || "$altus" == "null" ]]; then altus_account_exists "$ALTUS_ACCOUNT_NAME" || create_altus_account "$ALTUS_ACCOUNT_NAME" "dummy" "$DUMMY_CDP_PRIVATE_KEY" cm_args+=("$PARAM_ALTUS_ACCOUNT" "$ALTUS_ACCOUNT_NAME") elif [[ "$altus" == "$ALTUS_ACCOUNT_NAME" ]]; then if altus_account_exists "$altus"; then log "ℹ️ $PARAM_ALTUS_ACCOUNT already set ('$altus') and account exists — leaving as-is." else echo " • $PARAM_ALTUS_ACCOUNT points to '$altus' but that external account is missing — (re)creating it." create_altus_account "$altus" "dummy" "$DUMMY_CDP_PRIVATE_KEY" fi else if altus_account_exists "$altus"; then log "ℹ️ $PARAM_ALTUS_ACCOUNT already set ('$altus') — leaving as-is." else echo " ⚠️ $PARAM_ALTUS_ACCOUNT points to '$altus' but no such ALTUS_ACCESS_KEY_AUTH account exists." echo " Create that account in CM, or clear $PARAM_ALTUS_ACCOUNT so setup can seed the dummy placeholder." fi fi [[ "$dmp_m" != "true" ]] && cm_args+=("$PARAM_DMP_METRICS" "true") [[ "$dmp_l" != "true" ]] && cm_args+=("$PARAM_DMP_LOGS" "true") if [[ ${#cm_args[@]} -gt 0 ]]; then set_cm_config "Modified%20Observability%20Prerequisites" "${cm_args[@]}" echo " • /cm/config: added/enabled ${cm_args[*]}" changed=1 fi if [[ -z "$rw_override" || "$rw_override" == "null" ]]; then send_config_update "Modified%20OTel%20Remote%20Write%20URL" "$PARAM_REMOTE_WRITE_URL" "$DUMMY_REMOTE_WRITE_URL" echo " • /cm/allHosts/config: set $PARAM_REMOTE_WRITE_URL=$DUMMY_REMOTE_WRITE_URL" changed=1 else log "ℹ️ $PARAM_REMOTE_WRITE_URL override already set ('$rw_override') — leaving as-is." fi # TelemetryPublisher role + safety valve (the logs pipeline depends on it). if configure_telemetrypublisher; then changed=1; fi [[ "$changed" -eq 0 ]] && echo "ℹ️ All prerequisites already present — nothing to add." } usage_setup_prerequisites() { cat < ...] --user --pass

Seeds prerequisites for a self-managed / airgapped OBS deployment (Customer 3/4), NON-DESTRUCTIVELY — only values that are missing or disabled are added; existing real values are never overwritten: • creates + references a dummy ALTUS_ACCESS_KEY_AUTH account ('$ALTUS_ACCOUNT_NAME') only if $PARAM_ALTUS_ACCOUNT is unset (/cm/config) • enables DMP flags $PARAM_DMP_METRICS / $PARAM_DMP_LOGS only if not already true (/cm/config) • sets a dummy $PARAM_REMOTE_WRITE_URL=$DUMMY_REMOTE_WRITE_URL only if unset (/cm/allHosts/config) • creates the mgmt $TP_ROLE_TYPE role (on the mgmt host) only if absent, and seeds a dummy $PARAM_TP_SAFETY_VALVE only if unset — the logs pipeline depends on TP. The role is created stopped; start it (or run --refresh) to bring it up. • optionally sets gateway hosts (--host) Does NOT add an exporter. Run 'add-exporter --file ' afterwards — if the DMP path is not fully real it auto-disables the thanos/DMP fan-out (avoids export 4xx/5xx). EOF exit 0 } handle_setup_prerequisites() { [[ "$SHOW_HELP" == true ]] && usage_setup_prerequisites require_auth configure_prerequisites if [[ ${#HOSTS[@]} -gt 0 ]]; then echo "⚙️ Configuring gateway hosts: ${HOSTS[*]}" # Subshell so a no-op 'exit 0' inside the handler doesn't end this command. ( handle_add_gateway_host ) || { echo "❌ Gateway host configuration failed."; exit 1; } fi echo "" echo "✅ setup-prerequisites complete." echo " Next: add-exporter --file — it auto-disables the thanos/DMP" echo " fan-out while remote_write_url is the dummy placeholder ($DUMMY_REMOTE_WRITE_URL)." } usage_refresh_config() { { cat < a single, no-downtime "Deploy Client Configuration and Refresh" (clusters/{cluster}/commands/deployClientConfigsAndRefresh). • The Management Service, if stale -> a service restart (cm/service/commands/restart). • Anything already FRESH is skipped. Each submitted command is polled to completion. This is the same apply step performed by the --refresh flag on other commands; run it on demand when you did not pass --refresh, or when config went stale by other means. It writes no OTel config of its own. SYNOPSIS refresh-config EOF _print_common_opts cat </dev/null) [[ "$active" != "true" ]] && break if [[ "$waited" -ge "$CM_POLL_TIMEOUT" ]]; then echo " ⏳ Command $id still running after ${CM_POLL_TIMEOUT}s; not waiting further." return 0 fi sleep "$CM_POLL_INTERVAL"; waited=$((waited + CM_POLL_INTERVAL)) done success=$(echo "$resp" | jq -r '.success // "unknown"' 2>/dev/null) msg=$(echo "$resp" | jq -r '.resultMessage // ""' 2>/dev/null) if [[ "$success" == "true" ]]; then echo " ✅ Command $id succeeded${msg:+ — $msg}" else echo " ❌ Command $id did not succeed (success=$success)${msg:+ — $msg}" fi } # True (0) if mgmt OR any cluster service reports non-FRESH staleness. # Uses only the two staleness-check GET APIs. _any_stale() { local mgmt clusters cl enc mgmt=$(cm_v41_get "/cm/service" | jq -r '.configStalenessStatus // "FRESH"' 2>/dev/null) [[ -n "$mgmt" && "$mgmt" != "FRESH" ]] && return 0 clusters=$(cm_v41_get "/clusters" | jq -r '.items[]?.name' 2>/dev/null) while IFS= read -r cl; do [[ -z "$cl" ]] && continue enc=$(urlencode "$cl") if cm_v41_get "/clusters/$enc/services" \ | jq -e '[.items[]? | select(.configStalenessStatus != "FRESH" or .clientConfigStalenessStatus != "FRESH")] | length > 0' \ >/dev/null 2>&1; then return 0 fi done <<< "$clusters" return 1 } # Detect stale config and apply it. With arg "wait", poll briefly first because CM # computes staleness asynchronously right after a config write (the flag path); the # standalone refresh-config command checks once (staleness is already computed). apply_stale_config() { local wait_compute="${1:-}" require_auth echo "" echo "🔎 refresh: checking Cloudera Manager for stale configuration..." if [[ "$wait_compute" == "wait" ]]; then local waited=0 until _any_stale; do if [[ "$waited" -ge 60 ]]; then echo " ℹ️ Nothing stale — configuration already applied (nothing to do)." return 0 fi sleep 5; waited=$((waited + 5)) done else if ! _any_stale; then echo " ℹ️ Nothing stale — nothing to do." return 0 fi fi # --- Cluster: GET services staleness, POST deployClientConfigsAndRefresh --- local clusters cl enc stale_svcs resp id clusters=$(cm_v41_get "/clusters" | jq -r '.items[]?.name' 2>/dev/null) while IFS= read -r cl; do [[ -z "$cl" ]] && continue enc=$(urlencode "$cl") stale_svcs=$(cm_v41_get "/clusters/$enc/services" \ | jq -r '.items[]? | select(.configStalenessStatus != "FRESH" or .clientConfigStalenessStatus != "FRESH") | .name' 2>/dev/null) [[ -z "${stale_svcs// }" ]] && continue echo "" echo "Apply plan (cluster '$cl'): DEPLOY CLIENT CONFIGS + REFRESH" printf ' • %s\n' $stale_svcs resp=$(cm_v41_post "/clusters/$enc/commands/deployClientConfigsAndRefresh") id=$(echo "$resp" | jq -r '.id // empty' 2>/dev/null) if [[ -z "$id" ]]; then echo " ❌ deployClientConfigsAndRefresh failed: $(echo "$resp" | jq -r '.message // .' 2>/dev/null)" else echo " 🔄 Submitted deployClientConfigsAndRefresh (command $id)..." cm_wait_command "$id" fi done <<< "$clusters" # --- Mgmt: GET /cm/service staleness, POST /cm/service/commands/restart --- # Mgmt config changes are not refreshable, so the only apply is a restart. local mgmt_status mgmt_status=$(cm_v41_get "/cm/service" | jq -r '.configStalenessStatus // "FRESH"' 2>/dev/null) if [[ -n "$mgmt_status" && "$mgmt_status" != "FRESH" ]]; then echo "" echo "Apply plan (Management Service): RESTART (mgmt is $mgmt_status)" resp=$(cm_v41_post "/cm/service/commands/restart") id=$(echo "$resp" | jq -r '.id // empty' 2>/dev/null) if [[ -z "$id" ]]; then echo " ❌ mgmt restart failed: $(echo "$resp" | jq -r '.message // .' 2>/dev/null)" else echo " ♻️ Restarting Management Service (command $id)..." cm_wait_command "$id" fi else echo " ℹ️ Management Service is FRESH — no mgmt action needed." fi echo "" echo "✅ refresh: apply step complete." } handle_refresh_config() { [[ "$SHOW_HELP" == true ]] && usage_refresh_config apply_stale_config } # When sourced as a library (e.g. by tests), stop here: all functions are # defined above, but no argument parsing or command dispatch should run. if [[ -n "${OBS_CLI_LIBRARY_MODE:-}" ]]; then return 0 2>/dev/null || exit 0 fi if [[ $# -eq 0 ]]; then usage; fi COMMAND="" CM_BASE_URL="$DEFAULT_CM_BASE_URL" CM_USER="$DEFAULT_CM_USER" CM_PASS="$DEFAULT_CM_PASS" HOSTS=() EXPORTERS=() PROCESSORS=() EXTENSIONS=() CONFIG_FILE="" SHOW_HELP=false SETUP_PREREQS=false SIGNAL="" # --signal metrics|logs|both (empty => both) SIGNALS=() REFRESH=false # --refresh: apply stale config after a mutating command while [[ $# -gt 0 ]]; do case "$1" in --url) CM_BASE_URL="$2"; shift 2 ;; --user) CM_USER="$2"; shift 2 ;; --pass) CM_PASS="$2"; shift 2 ;; --host) HOSTS+=("$2"); shift 2 ;; --exporter) EXPORTERS+=("$2"); shift 2 ;; --processor) PROCESSORS+=("$2"); shift 2 ;; --extension) EXTENSIONS+=("$2"); shift 2 ;; --file) CONFIG_FILE="$2"; shift 2 ;; --signal) SIGNAL="$2"; shift 2 ;; --setup-prerequisites) SETUP_PREREQS=true; shift ;; --refresh) REFRESH=true; shift ;; --debug) DEBUG=true; shift ;; --help|-h) SHOW_HELP=true; shift ;; -*) echo "" echo "❌ Unknown option: '$1'" echo "" if [[ -n "$COMMAND" ]]; then show_command_help "$COMMAND" else usage fi ;; *) if [[ -z "$COMMAND" ]]; then COMMAND="$1"; shift else echo "" echo "❌ Unexpected argument: '$1'" echo "" show_command_help "$COMMAND" fi ;; esac done # Expand --signal into SIGNALS (validates the value). build_signals if [[ -z "$COMMAND" ]]; then usage; fi case "$COMMAND" in get-gateway-hosts) handle_get_gateway_hosts ;; list-hosts) handle_list_hosts ;; list-exporters) handle_list_exporters ;; get-metric-configs) handle_get_metric_configs ;; add-gateway-host) handle_add_gateway_host ;; remove-gateway-host) handle_remove_gateway_host ;; setup-prerequisites) handle_setup_prerequisites ;; refresh-config) handle_refresh_config ;; add-exporter) handle_add_exporter ;; update-exporter) handle_update_exporter ;; remove-exporter) handle_remove_exporter ;; list-processors) handle_list_processors ;; add-processor) handle_add_processor ;; update-processor) handle_update_processor ;; remove-processor) handle_remove_processor ;; list-extensions) handle_list_extensions ;; add-extension) handle_add_extension ;; update-extension) handle_update_extension ;; remove-extension) handle_remove_extension ;; *) show_command_help "$COMMAND" ;; esac # --refresh: after a mutating command wrote config, apply any resulting stale config. # Read-only commands (list-*, get-*) and refresh-config itself are intentionally excluded # (refresh-config already applies via its handler; this would double-fire). if [[ "$REFRESH" == true ]]; then case "$COMMAND" in add-exporter|update-exporter|remove-exporter|add-processor|update-processor|remove-processor|\ add-extension|update-extension|remove-extension|add-gateway-host|remove-gateway-host|setup-prerequisites) apply_stale_config wait ;; esac fi