{
  "uid": "ghost-overview",
  "title": "Ghost",
  "tags": [
    "ghost"
  ],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "30s",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "templating": {
    "list": [
      {
        "name": "service",
        "label": "Service",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "prometheus"
        },
        "query": {
          "query": "label_values(traces_spanmetrics_calls_total, service)",
          "refId": "service"
        },
        "includeAll": true,
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "refresh": 2
      }
    ]
  },
  "panels": [
    {
      "type": "timeseries",
      "title": "Requests per second",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "fillOpacity": 10
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (service) (rate(traces_spanmetrics_calls_total{service=~\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))",
          "legendFormat": "{{service}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Error rate",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "min": 0,
          "custom": {
            "fillOpacity": 10
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (service) (rate(traces_spanmetrics_calls_total{service=~\"$service\", span_kind=\"SPAN_KIND_SERVER\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / sum by (service) (rate(traces_spanmetrics_calls_total{service=~\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))",
          "legendFormat": "{{service}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Request latency",
      "description": "Server spans across every instrumented service.",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "fillOpacity": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "calcs": [
            "lastNotNull",
            "max"
          ],
          "placement": "right"
        }
      },
      "targets": [
        {
          "refId": "p50",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.50, sum by (service, le) (rate(traces_spanmetrics_latency_bucket{service=~\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
          "legendFormat": "{{service}} p50"
        },
        {
          "refId": "p90",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.90, sum by (service, le) (rate(traces_spanmetrics_latency_bucket{service=~\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
          "legendFormat": "{{service}} p90"
        },
        {
          "refId": "p99",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.99, sum by (service, le) (rate(traces_spanmetrics_latency_bucket{service=~\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))",
          "legendFormat": "{{service}} p99"
        }
      ]
    },
    {
      "type": "table",
      "title": "Slowest endpoints (p99)",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 17
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {
        "sortBy": [
          {
            "displayName": "Value",
            "desc": true
          }
        ]
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "topk(20, histogram_quantile(0.99, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))))",
          "instant": true,
          "format": "table"
        }
      ],
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true
            }
          }
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Outbound calls, p99 (database, S3, HTTP)",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 17
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "fillOpacity": 0
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "topk(10, histogram_quantile(0.99, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"$service\", span_kind=\"SPAN_KIND_CLIENT\"}[$__rate_interval]))))",
          "legendFormat": "{{service}} {{span_name}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "API route latency (OTel HTTP metrics)",
      "description": "Emitted by the API's own SDK, so it carries the matched route rather than the span name.",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 26
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "fillOpacity": 0
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "p50",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=~\"$service\"}[$__rate_interval])))",
          "legendFormat": "p50"
        },
        {
          "refId": "p99",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=~\"$service\"}[$__rate_interval])))",
          "legendFormat": "p99"
        },
        {
          "refId": "route",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "topk(10, histogram_quantile(0.99, sum by (http_route, le) (rate(http_server_request_duration_seconds_bucket{job=~\"$service\"}[$__rate_interval]))))",
          "legendFormat": "p99 {{http_route}}"
        }
      ]
    },
    {
      "type": "traces",
      "title": "Slow traces (over 500ms)",
      "datasource": {
        "type": "tempo",
        "uid": "tempo"
      },
      "gridPos": {
        "h": 10,
        "w": 24,
        "x": 0,
        "y": 35
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "tempo",
            "uid": "tempo"
          },
          "queryType": "traceql",
          "query": "{ duration > 500ms }",
          "limit": 20,
          "tableType": "traces"
        }
      ]
    },
    {
      "type": "flamegraph",
      "title": "CPU profile (API)",
      "description": "CPU time only. A frame that is wide here is burning the processor.",
      "datasource": {
        "type": "grafana-pyroscope-datasource",
        "uid": "pyroscope"
      },
      "gridPos": {
        "h": 12,
        "w": 12,
        "x": 0,
        "y": 45
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "grafana-pyroscope-datasource",
            "uid": "pyroscope"
          },
          "queryType": "profile",
          "profileTypeId": "wall:cpu:nanoseconds:wall:nanoseconds",
          "labelSelector": "{service_name=\"ghost-api\"}",
          "groupBy": []
        }
      ]
    },
    {
      "type": "flamegraph",
      "title": "Wall profile (API)",
      "description": "Wall clock, so a frame wide here but narrow in the CPU profile is waiting on postgres, S3 or git.",
      "datasource": {
        "type": "grafana-pyroscope-datasource",
        "uid": "pyroscope"
      },
      "gridPos": {
        "h": 12,
        "w": 12,
        "x": 12,
        "y": 45
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "grafana-pyroscope-datasource",
            "uid": "pyroscope"
          },
          "queryType": "profile",
          "profileTypeId": "wall:wall:nanoseconds:wall:nanoseconds",
          "labelSelector": "{service_name=\"ghost-api\"}",
          "groupBy": []
        }
      ]
    },
    {
      "type": "row",
      "title": "Application",
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 57
      },
      "collapsed": false,
      "panels": []
    },
    {
      "type": "stat",
      "title": "Commits",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 0,
        "y": 58
      },
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {
        "graphMode": "area",
        "colorMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(ghost_commits)"
        }
      ],
      "description": "Commits indexed on default branches. A force push that drops commits lowers it."
    },
    {
      "type": "stat",
      "title": "Repositories",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 4,
        "y": 58
      },
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {
        "graphMode": "area",
        "colorMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(ghost_repositories)"
        }
      ]
    },
    {
      "type": "stat",
      "title": "Users",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 8,
        "y": 58
      },
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {
        "graphMode": "area",
        "colorMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(ghost_users)"
        }
      ]
    },
    {
      "type": "stat",
      "title": "Stars",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 12,
        "y": 58
      },
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {
        "graphMode": "area",
        "colorMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(ghost_stars)"
        }
      ]
    },
    {
      "type": "stat",
      "title": "Open issues",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 16,
        "y": 58
      },
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {
        "graphMode": "area",
        "colorMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(ghost_issues{state=\"open\"})"
        }
      ]
    },
    {
      "type": "stat",
      "title": "Open pull requests",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 20,
        "y": 58
      },
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "options": {
        "graphMode": "area",
        "colorMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(ghost_pull_requests{state=\"open\"})"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Commits over time",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 63
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(ghost_commits)",
          "legendFormat": "commits"
        }
      ],
      "description": "The instance total, sampled every collection."
    },
    {
      "type": "timeseries",
      "title": "Repositories by visibility",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 63
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum by (visibility) (ghost_repositories)",
          "legendFormat": "{{visibility}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Pushed bytes",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 71
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(increase(ghost_push_bytes_total[$__rate_interval]))",
          "legendFormat": "pushed"
        }
      ],
      "description": "Packfile bytes accepted into the write-ahead log. Counters restart with the process, which `increase` accounts for."
    },
    {
      "type": "timeseries",
      "title": "Pushes and ref updates",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 71
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10
          }
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(increase(ghost_pushes_total[$__rate_interval]))",
          "legendFormat": "pushes"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "sum(increase(ghost_ref_updates_total[$__rate_interval]))",
          "legendFormat": "ref updates"
        }
      ]
    },
    {
      "type": "row",
      "title": "Delivery",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 79
      },
      "panels": []
    },
    {
      "type": "timeseries",
      "title": "Delivery lag",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 80
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10
          },
          "unit": "s"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "max(ghost_delivery_lag_seconds)",
          "legendFormat": "lag"
        }
      ],
      "description": "How overdue the most overdue pending job is. This is the delivery SLO: it stays near zero while apps/delivery keeps up, and climbs when it is down or saturated. Every API instance reports the same number, hence max."
    },
    {
      "type": "timeseries",
      "title": "Pending deliveries",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 80
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10
          },
          "unit": "short"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "max by (kind) (ghost_delivery_jobs{status=\"pending\"})",
          "legendFormat": "{{kind}}"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "max by (kind) (ghost_delivery_jobs{status=\"dead\"})",
          "legendFormat": "{{kind}} gave up"
        }
      ],
      "description": "Jobs waiting for a first try or a retry, and jobs Ghost gave up on (kept 30 days)."
    },
    {
      "type": "timeseries",
      "title": "Webhook success rate",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 88
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "max(ghost_delivery_attempts{outcome=\"succeeded\"}) / clamp_min(max(ghost_delivery_attempts{outcome=\"succeeded\"}) + max(ghost_delivery_attempts{outcome=\"failed\"}), 1)",
          "legendFormat": "success"
        }
      ],
      "description": "Share of webhook attempts in the last five minutes that got a 2xx. One broken receiver pulls this down; the delivery log shows which."
    },
    {
      "type": "timeseries",
      "title": "Webhook attempt p99",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 88
      },
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10
          },
          "unit": "ms"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "expr": "max(ghost_delivery_attempt_p99_milliseconds)",
          "legendFormat": "p99"
        }
      ],
      "description": "p99 time to send one webhook and read its response, over the last five minutes. The send deadline is 10 s."
    }
  ]
}