delivery: circuit breaker, per-host limit, pruning and metrics 04047d512 days ago 944 lines · 22.0 KB 1 { 2 " uid " : " ghost-overview " , 3 " title " : 4 " tags " : 5 " ghost " 6 ] , 7 " timezone " : 8 " schemaVersion " : 9 " refresh " : 10 " time " : 11 " from " : 12 " to " : 13 } , 14 " templating " : 15 " list " : 16 { 17 " name " : 18 " label " : 19 " type " : 20 " datasource " : 21 " type " : 22 " uid " : 23 } , 24 " query " : 25 " query " : 26 " refId " : 27 } , 28 " includeAll " : 29 " multi " : 30 " current " : 31 " text " : 32 " value " : 33 } , 34 " refresh " : 35 } 36 ] 37 } , 38 " panels " : 39 { 40 " type " : 41 " title " : 42 " datasource " : 43 " type " : 44 " uid " : 45 } , 46 " gridPos " : 47 " h " : 48 " w " : 49 " x " : 50 " y " : 51 } , 52 " fieldConfig " : 53 " defaults " : 54 " unit " : 55 " custom " : 56 " fillOpacity " : 57 } 58 } , 59 " overrides " : 60 } , 61 " targets " : 62 { 63 " refId " : 64 " datasource " : 65 " type " : 66 " uid " : 67 } , 68 " expr " : 69 " legendFormat " : 70 } 71 ] 72 } , 73 { 74 " type " : 75 " title " : 76 " datasource " : 77 " type " : 78 " uid " : 79 } , 80 " gridPos " : 81 " h " : 82 " w " : 83 " x " : 84 " y " : 85 } , 86 " fieldConfig " : 87 " defaults " : 88 " unit " : 89 " min " : 90 " custom " : 91 " fillOpacity " : 92 } 93 } , 94 " overrides " : 95 } , 96 " targets " : 97 { 98 " refId " : 99 " datasource " : 100 " type " : 101 " uid " : 102 } , 103 " expr " : 104 " legendFormat " : 105 } 106 ] 107 } , 108 { 109 " type " : 110 " title " : 111 " description " : 112 " datasource " : 113 " type " : 114 " uid " : 115 } , 116 " gridPos " : 117 " h " : 118 " w " : 119 " x " : 120 " y " : 121 } , 122 " fieldConfig " : 123 " defaults " : 124 " unit " : 125 " custom " : 126 " fillOpacity " : 127 } 128 } , 129 " overrides " : 130 } , 131 " options " : 132 " legend " : 133 " displayMode " : 134 " calcs " : 135 " lastNotNull " , 136 " max " 137 ] , 138 " placement " : 139 } 140 } , 141 " targets " : 142 { 143 " refId " : 144 " datasource " : 145 " type " : 146 " uid " : 147 } , 148 " expr " : 149 " legendFormat " : 150 } , 151 { 152 " refId " : 153 " datasource " : 154 " type " : 155 " uid " : 156 } , 157 " expr " : 158 " legendFormat " : 159 } , 160 { 161 " refId " : 162 " datasource " : 163 " type " : 164 " uid " : 165 } , 166 " expr " : 167 " legendFormat " : 168 } 169 ] 170 } , 171 { 172 " type " : 173 " title " : 174 " datasource " : 175 " type " : 176 " uid " : 177 } , 178 " gridPos " : 179 " h " : 180 " w " : 181 " x " : 182 " y " : 183 } , 184 " fieldConfig " : 185 " defaults " : 186 " unit " : 187 } , 188 " overrides " : 189 } , 190 " options " : 191 " sortBy " : 192 { 193 " displayName " : 194 " desc " : 195 } 196 ] 197 } , 198 " targets " : 199 { 200 " refId " : 201 " datasource " : 202 " type " : 203 " uid " : 204 } , 205 " expr " : 206 " instant " : 207 " format " : 208 } 209 ] , 210 " transformations " : 211 { 212 " id " : 213 " options " : 214 " excludeByName " : 215 " Time " : 216 } 217 } 218 } 219 ] 220 } , 221 { 222 " type " : 223 " title " : 224 " datasource " : 225 " type " : 226 " uid " : 227 } , 228 " gridPos " : 229 " h " : 230 " w " : 231 " x " : 232 " y " : 233 } , 234 " fieldConfig " : 235 " defaults " : 236 " unit " : 237 " custom " : 238 " fillOpacity " : 239 } 240 } , 241 " overrides " : 242 } , 243 " targets " : 244 { 245 " refId " : 246 " datasource " : 247 " type " : 248 " uid " : 249 } , 250 " expr " : 251 " legendFormat " : 252 } 253 ] 254 } , 255 { 256 " type " : 257 " title " : 258 " description " : 259 " datasource " : 260 " type " : 261 " uid " : 262 } , 263 " gridPos " : 264 " h " : 265 " w " : 266 " x " : 267 " y " : 268 } , 269 " fieldConfig " : 270 " defaults " : 271 " unit " : 272 " custom " : 273 " fillOpacity " : 274 } 275 } , 276 " overrides " : 277 } , 278 " targets " : 279 { 280 " refId " : 281 " datasource " : 282 " type " : 283 " uid " : 284 } , 285 " expr " : 286 " legendFormat " : 287 } , 288 { 289 " refId " : 290 " datasource " : 291 " type " : 292 " uid " : 293 } , 294 " expr " : 295 " legendFormat " : 296 } , 297 { 298 " refId " : 299 " datasource " : 300 " type " : 301 " uid " : 302 } , 303 " expr " : 304 " legendFormat " : 305 } 306 ] 307 } , 308 { 309 " type " : 310 " title " : 311 " datasource " : 312 " type " : 313 " uid " : 314 } , 315 " gridPos " : 316 " h " : 317 " w " : 318 " x " : 319 " y " : 320 } , 321 " targets " : 322 { 323 " refId " : 324 " datasource " : 325 " type " : 326 " uid " : 327 } , 328 " queryType " : 329 " query " : 330 " limit " : 331 " tableType " : 332 } 333 ] 334 } , 335 { 336 " type " : 337 " title " : 338 " description " : 339 " datasource " : 340 " type " : 341 " uid " : 342 } , 343 " gridPos " : 344 " h " : 345 " w " : 346 " x " : 347 " y " : 348 } , 349 " targets " : 350 { 351 " refId " : 352 " datasource " : 353 " type " : 354 " uid " : 355 } , 356 " queryType " : 357 " profileTypeId " : 358 " labelSelector " : 359 " groupBy " : 360 } 361 ] 362 } , 363 { 364 " type " : 365 " title " : 366 " description " : 367 " datasource " : 368 " type " : 369 " uid " : 370 } , 371 " gridPos " : 372 " h " : 373 " w " : 374 " x " : 375 " y " : 376 } , 377 " targets " : 378 { 379 " refId " : 380 " datasource " : 381 " type " : 382 " uid " : 383 } , 384 " queryType " : 385 " profileTypeId " : 386 " labelSelector " : 387 " groupBy " : 388 } 389 ] 390 } , 391 { 392 " type " : 393 " title " : 394 " gridPos " : 395 " h " : 396 " w " : 397 " x " : 398 " y " : 399 } , 400 " collapsed " : 401 " panels " : 402 } , 403 { 404 " type " : 405 " title " : 406 " datasource " : 407 " type " : 408 " uid " : 409 } , 410 " gridPos " : 411 " h " : 412 " w " : 413 " x " : 414 " y " : 415 } , 416 " fieldConfig " : 417 " defaults " : 418 " overrides " : 419 } , 420 " options " : 421 " graphMode " : 422 " colorMode " : 423 " reduceOptions " : 424 " calcs " : 425 " lastNotNull " 426 ] , 427 " fields " : 428 " values " : 429 } 430 } , 431 " targets " : 432 { 433 " refId " : 434 " datasource " : 435 " type " : 436 " uid " : 437 } , 438 " expr " : 439 } 440 ] , 441 " description " : 442 } , 443 { 444 " type " : 445 " title " : 446 " datasource " : 447 " type " : 448 " uid " : 449 } , 450 " gridPos " : 451 " h " : 452 " w " : 453 " x " : 454 " y " : 455 } , 456 " fieldConfig " : 457 " defaults " : 458 " overrides " : 459 } , 460 " options " : 461 " graphMode " : 462 " colorMode " : 463 " reduceOptions " : 464 " calcs " : 465 " lastNotNull " 466 ] , 467 " fields " : 468 " values " : 469 } 470 } , 471 " targets " : 472 { 473 " refId " : 474 " datasource " : 475 " type " : 476 " uid " : 477 } , 478 " expr " : 479 } 480 ] 481 } , 482 { 483 " type " : 484 " title " : 485 " datasource " : 486 " type " : 487 " uid " : 488 } , 489 " gridPos " : 490 " h " : 491 " w " : 492 " x " : 493 " y " : 494 } , 495 " fieldConfig " : 496 " defaults " : 497 " overrides " : 498 } , 499 " options " : 500 " graphMode " : 501 " colorMode " : 502 " reduceOptions " : 503 " calcs " : 504 " lastNotNull " 505 ] , 506 " fields " : 507 " values " : 508 } 509 } , 510 " targets " : 511 { 512 " refId " : 513 " datasource " : 514 " type " : 515 " uid " : 516 } , 517 " expr " : 518 } 519 ] 520 } , 521 { 522 " type " : 523 " title " : 524 " datasource " : 525 " type " : 526 " uid " : 527 } , 528 " gridPos " : 529 " h " : 530 " w " : 531 " x " : 532 " y " : 533 } , 534 " fieldConfig " : 535 " defaults " : 536 " overrides " : 537 } , 538 " options " : 539 " graphMode " : 540 " colorMode " : 541 " reduceOptions " : 542 " calcs " : 543 " lastNotNull " 544 ] , 545 " fields " : 546 " values " : 547 } 548 } , 549 " targets " : 550 { 551 " refId " : 552 " datasource " : 553 " type " : 554 " uid " : 555 } , 556 " expr " : 557 } 558 ] 559 } , 560 { 561 " type " : 562 " title " : 563 " datasource " : 564 " type " : 565 " uid " : 566 } , 567 " gridPos " : 568 " h " : 569 " w " : 570 " x " : 571 " y " : 572 } , 573 " fieldConfig " : 574 " defaults " : 575 " overrides " : 576 } , 577 " options " : 578 " graphMode " : 579 " colorMode " : 580 " reduceOptions " : 581 " calcs " : 582 " lastNotNull " 583 ] , 584 " fields " : 585 " values " : 586 } 587 } , 588 " targets " : 589 { 590 " refId " : 591 " datasource " : 592 " type " : 593 " uid " : 594 } , 595 " expr " : 596 } 597 ] 598 } , 599 { 600 " type " : 601 " title " : 602 " datasource " : 603 " type " : 604 " uid " : 605 } , 606 " gridPos " : 607 " h " : 608 " w " : 609 " x " : 610 " y " : 611 } , 612 " fieldConfig " : 613 " defaults " : 614 " overrides " : 615 } , 616 " options " : 617 " graphMode " : 618 " colorMode " : 619 " reduceOptions " : 620 " calcs " : 621 " lastNotNull " 622 ] , 623 " fields " : 624 " values " : 625 } 626 } , 627 " targets " : 628 { 629 " refId " : 630 " datasource " : 631 " type " : 632 " uid " : 633 } , 634 " expr " : 635 } 636 ] 637 } , 638 { 639 " type " : 640 " title " : 641 " datasource " : 642 " type " : 643 " uid " : 644 } , 645 " gridPos " : 646 " h " : 647 " w " : 648 " x " : 649 " y " : 650 } , 651 " fieldConfig " : 652 " defaults " : 653 " custom " : 654 " fillOpacity " : 655 } 656 } , 657 " overrides " : 658 } , 659 " targets " : 660 { 661 " refId " : 662 " datasource " : 663 " type " : 664 " uid " : 665 } , 666 " expr " : 667 " legendFormat " : 668 } 669 ] , 670 " description " : 671 } , 672 { 673 " type " : 674 " title " : 675 " datasource " : 676 " type " : 677 " uid " : 678 } , 679 " gridPos " : 680 " h " : 681 " w " : 682 " x " : 683 " y " : 684 } , 685 " fieldConfig " : 686 " defaults " : 687 " custom " : 688 " fillOpacity " : 689 } 690 } , 691 " overrides " : 692 } , 693 " targets " : 694 { 695 " refId " : 696 " datasource " : 697 " type " : 698 " uid " : 699 } , 700 " expr " : 701 " legendFormat " : 702 } 703 ] 704 } , 705 { 706 " type " : 707 " title " : 708 " datasource " : 709 " type " : 710 " uid " : 711 } , 712 " gridPos " : 713 " h " : 714 " w " : 715 " x " : 716 " y " : 717 } , 718 " fieldConfig " : 719 " defaults " : 720 " custom " : 721 " fillOpacity " : 722 } , 723 " unit " : 724 } , 725 " overrides " : 726 } , 727 " targets " : 728 { 729 " refId " : 730 " datasource " : 731 " type " : 732 " uid " : 733 } , 734 " expr " : 735 " legendFormat " : 736 } 737 ] , 738 " description " : 739 } , 740 { 741 " type " : 742 " title " : 743 " datasource " : 744 " type " : 745 " uid " : 746 } , 747 " gridPos " : 748 " h " : 749 " w " : 750 " x " : 751 " y " : 752 } , 753 " fieldConfig " : 754 " defaults " : 755 " custom " : 756 " fillOpacity " : 757 } 758 } , 759 " overrides " : 760 } , 761 " targets " : 762 { 763 " refId " : 764 " datasource " : 765 " type " : 766 " uid " : 767 } , 768 " expr " : 769 " legendFormat " : 770 } , 771 { 772 " refId " : 773 " datasource " : 774 " type " : 775 " uid " : 776 } , 777 " expr " : 778 " legendFormat " : 779 } 780 ] 781 } , 782 { 783 " type " : 784 " title " : 785 " collapsed " : 786 " gridPos " : 787 " h " : 788 " w " : 789 " x " : 790 " y " : 791 } , 792 " panels " : 793 } , 794 { 795 " type " : 796 " title " : 797 " datasource " : 798 " type " : 799 " uid " : 800 } , 801 " gridPos " : 802 " h " : 803 " w " : 804 " x " : 805 " y " : 806 } , 807 " fieldConfig " : 808 " defaults " : 809 " custom " : 810 " fillOpacity " : 811 } , 812 " unit " : 813 } , 814 " overrides " : 815 } , 816 " targets " : 817 { 818 " refId " : 819 " datasource " : 820 " type " : 821 " uid " : 822 } , 823 " expr " : 824 " legendFormat " : 825 } 826 ] , 827 " description " : 828 } , 829 { 830 " type " : 831 " title " : 832 " datasource " : 833 " type " : 834 " uid " : 835 } , 836 " gridPos " : 837 " h " : 838 " w " : 839 " x " : 840 " y " : 841 } , 842 " fieldConfig " : 843 " defaults " : 844 " custom " : 845 " fillOpacity " : 846 } , 847 " unit " : 848 } , 849 " overrides " : 850 } , 851 " targets " : 852 { 853 " refId " : 854 " datasource " : 855 " type " : 856 " uid " : 857 } , 858 " expr " : 859 " legendFormat " : 860 } , 861 { 862 " refId " : 863 " datasource " : 864 " type " : 865 " uid " : 866 } , 867 " expr " : 868 " legendFormat " : 869 } 870 ] , 871 " description " : 872 } , 873 { 874 " type " : 875 " title " : 876 " datasource " : 877 " type " : 878 " uid " : 879 } , 880 " gridPos " : 881 " h " : 882 " w " : 883 " x " : 884 " y " : 885 } , 886 " fieldConfig " : 887 " defaults " : 888 " custom " : 889 " fillOpacity " : 890 } , 891 " unit " : 892 } , 893 " overrides " : 894 } , 895 " targets " : 896 { 897 " refId " : 898 " datasource " : 899 " type " : 900 " uid " : 901 } , 902 " expr " : 903 " legendFormat " : 904 } 905 ] , 906 " description " : 907 } , 908 { 909 " type " : 910 " title " : 911 " datasource " : 912 " type " : 913 " uid " : 914 } , 915 " gridPos " : 916 " h " : 917 " w " : 918 " x " : 919 " y " : 920 } , 921 " fieldConfig " : 922 " defaults " : 923 " custom " : 924 " fillOpacity " : 925 } , 926 " unit " : 927 } , 928 " overrides " : 929 } , 930 " targets " : 931 { 932 " refId " : 933 " datasource " : 934 " type " : 935 " uid " : 936 } , 937 " expr " : 938 " legendFormat " : 939 } 940 ] , 941 " description " : 942 } 943 ] 944 }
"
Ghost
"
,
[
"
browser
"
,
39
,
"
30s
"
,
{
"
now-6h
"
,
"
now
"
{
[
"
service
"
,
"
Service
"
,
"
query
"
,
{
"
prometheus
"
,
"
prometheus
"
{
"
label_values(traces_spanmetrics_calls_total, service)
"
,
"
service
"
true
,
true
,
{
"
All
"
,
"
$__all
"
2
[
"
timeseries
"
,
"
Requests per second
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
0
,
0
{
{
"
reqps
"
,
{
10
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum by (service) (rate(traces_spanmetrics_calls_total{service=~
\"
$service
\"
, span_kind=
\"
SPAN_KIND_SERVER
\"
}[$__rate_interval]))
"
,
"
{{service}}
"
"
timeseries
"
,
"
Error rate
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
12
,
0
{
{
"
percentunit
"
,
0
,
{
10
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum by (service) (rate(traces_spanmetrics_calls_total{service=~
\"
$service
\"
, span_kind=
\"
SPAN_KIND_SERVER
\"
, status_code=
\"
STATUS_CODE_ERROR
\"
}[$__rate_interval])) / sum by (service) (rate(traces_spanmetrics_calls_total{service=~
\"
$service
\"
, span_kind=
\"
SPAN_KIND_SERVER
\"
}[$__rate_interval]))
"
,
"
{{service}}
"
"
timeseries
"
,
"
Request latency
"
,
"
Server spans across every instrumented service.
"
,
{
"
prometheus
"
,
"
prometheus
"
{
9
,
24
,
0
,
8
{
{
"
s
"
,
{
0
[]
{
{
"
table
"
,
[
"
right
"
[
"
p50
"
,
{
"
prometheus
"
,
"
prometheus
"
"
histogram_quantile(0.50, sum by (service, le) (rate(traces_spanmetrics_latency_bucket{service=~
\"
$service
\"
, span_kind=
\"
SPAN_KIND_SERVER
\"
}[$__rate_interval])))
"
,
"
{{service}} p50
"
"
p90
"
,
{
"
prometheus
"
,
"
prometheus
"
"
histogram_quantile(0.90, sum by (service, le) (rate(traces_spanmetrics_latency_bucket{service=~
\"
$service
\"
, span_kind=
\"
SPAN_KIND_SERVER
\"
}[$__rate_interval])))
"
,
"
{{service}} p90
"
"
p99
"
,
{
"
prometheus
"
,
"
prometheus
"
"
histogram_quantile(0.99, sum by (service, le) (rate(traces_spanmetrics_latency_bucket{service=~
\"
$service
\"
, span_kind=
\"
SPAN_KIND_SERVER
\"
}[$__rate_interval])))
"
,
"
{{service}} p99
"
"
table
"
,
"
Slowest endpoints (p99)
"
,
{
"
prometheus
"
,
"
prometheus
"
{
9
,
12
,
0
,
17
{
{
"
s
"
[]
{
[
"
Value
"
,
true
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
topk(20, histogram_quantile(0.99, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~
\"
$service
\"
, span_kind=
\"
SPAN_KIND_SERVER
\"
}[$__rate_interval]))))
"
,
true
,
"
table
"
[
"
organize
"
,
{
{
true
"
timeseries
"
,
"
Outbound calls, p99 (database, S3, HTTP)
"
,
{
"
prometheus
"
,
"
prometheus
"
{
9
,
12
,
12
,
17
{
{
"
s
"
,
{
0
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
topk(10, histogram_quantile(0.99, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~
\"
$service
\"
, span_kind=
\"
SPAN_KIND_CLIENT
\"
}[$__rate_interval]))))
"
,
"
{{service}} {{span_name}}
"
"
timeseries
"
,
"
API route latency (OTel HTTP metrics)
"
,
"
Emitted by the API's own SDK, so it carries the matched route rather than the span name.
"
,
{
"
prometheus
"
,
"
prometheus
"
{
9
,
24
,
0
,
26
{
{
"
s
"
,
{
0
[]
[
"
p50
"
,
{
"
prometheus
"
,
"
prometheus
"
"
histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=~
\"
$service
\"
}[$__rate_interval])))
"
,
"
p50
"
"
p99
"
,
{
"
prometheus
"
,
"
prometheus
"
"
histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=~
\"
$service
\"
}[$__rate_interval])))
"
,
"
p99
"
"
route
"
,
{
"
prometheus
"
,
"
prometheus
"
"
topk(10, histogram_quantile(0.99, sum by (http_route, le) (rate(http_server_request_duration_seconds_bucket{job=~
\"
$service
\"
}[$__rate_interval]))))
"
,
"
p99 {{http_route}}
"
"
traces
"
,
"
Slow traces (over 500ms)
"
,
{
"
tempo
"
,
"
tempo
"
{
10
,
24
,
0
,
35
[
"
A
"
,
{
"
tempo
"
,
"
tempo
"
"
traceql
"
,
"
{ duration > 500ms }
"
,
20
,
"
traces
"
"
flamegraph
"
,
"
CPU profile (API)
"
,
"
CPU time only. A frame that is wide here is burning the processor.
"
,
{
"
grafana-pyroscope-datasource
"
,
"
pyroscope
"
{
12
,
12
,
0
,
45
[
"
A
"
,
{
"
grafana-pyroscope-datasource
"
,
"
pyroscope
"
"
profile
"
,
"
wall:cpu:nanoseconds:wall:nanoseconds
"
,
"
{service_name=
\"
ghost-api
\"
}
"
,
[]
"
flamegraph
"
,
"
Wall profile (API)
"
,
"
Wall clock, so a frame wide here but narrow in the CPU profile is waiting on postgres, S3 or git.
"
,
{
"
grafana-pyroscope-datasource
"
,
"
pyroscope
"
{
12
,
12
,
12
,
45
[
"
A
"
,
{
"
grafana-pyroscope-datasource
"
,
"
pyroscope
"
"
profile
"
,
"
wall:wall:nanoseconds:wall:nanoseconds
"
,
"
{service_name=
\"
ghost-api
\"
}
"
,
[]
"
row
"
,
"
Application
"
,
{
1
,
24
,
0
,
57
false
,
[]
"
stat
"
,
"
Commits
"
,
{
"
prometheus
"
,
"
prometheus
"
{
5
,
4
,
0
,
58
{
{}
,
[]
{
"
area
"
,
"
value
"
,
{
[
""
,
false
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(ghost_commits)
"
"
Commits indexed on default branches. A force push that drops commits lowers it.
"
"
stat
"
,
"
Repositories
"
,
{
"
prometheus
"
,
"
prometheus
"
{
5
,
4
,
4
,
58
{
{}
,
[]
{
"
area
"
,
"
value
"
,
{
[
""
,
false
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(ghost_repositories)
"
"
stat
"
,
"
Users
"
,
{
"
prometheus
"
,
"
prometheus
"
{
5
,
4
,
8
,
58
{
{}
,
[]
{
"
area
"
,
"
value
"
,
{
[
""
,
false
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(ghost_users)
"
"
stat
"
,
"
Stars
"
,
{
"
prometheus
"
,
"
prometheus
"
{
5
,
4
,
12
,
58
{
{}
,
[]
{
"
area
"
,
"
value
"
,
{
[
""
,
false
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(ghost_stars)
"
"
stat
"
,
"
Open issues
"
,
{
"
prometheus
"
,
"
prometheus
"
{
5
,
4
,
16
,
58
{
{}
,
[]
{
"
area
"
,
"
value
"
,
{
[
""
,
false
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(ghost_issues{state=
\"
open
\"
})
"
"
stat
"
,
"
Open pull requests
"
,
{
"
prometheus
"
,
"
prometheus
"
{
5
,
4
,
20
,
58
{
{}
,
[]
{
"
area
"
,
"
value
"
,
{
[
""
,
false
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(ghost_pull_requests{state=
\"
open
\"
})
"
"
timeseries
"
,
"
Commits over time
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
0
,
63
{
{
{
10
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(ghost_commits)
"
,
"
commits
"
"
The instance total, sampled every collection.
"
"
timeseries
"
,
"
Repositories by visibility
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
12
,
63
{
{
{
10
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum by (visibility) (ghost_repositories)
"
,
"
{{visibility}}
"
"
timeseries
"
,
"
Pushed bytes
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
0
,
71
{
{
{
10
"
bytes
"
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(increase(ghost_push_bytes_total[$__rate_interval]))
"
,
"
pushed
"
"
Packfile bytes accepted into the write-ahead log. Counters restart with the process, which `increase` accounts for.
"
"
timeseries
"
,
"
Pushes and ref updates
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
12
,
71
{
{
{
10
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(increase(ghost_pushes_total[$__rate_interval]))
"
,
"
pushes
"
"
B
"
,
{
"
prometheus
"
,
"
prometheus
"
"
sum(increase(ghost_ref_updates_total[$__rate_interval]))
"
,
"
ref updates
"
"
row
"
,
"
Delivery
"
,
false
,
{
1
,
24
,
0
,
79
[]
"
timeseries
"
,
"
Delivery lag
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
0
,
80
{
{
{
10
"
s
"
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
max(ghost_delivery_lag_seconds)
"
,
"
lag
"
"
How overdue the most overdue pending job is. This is the delivery SLO: it stays near zero while apps/delivery keeps up, and climbs when it is down or saturated. Every API instance reports the same number, hence max.
"
"
timeseries
"
,
"
Pending deliveries
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
12
,
80
{
{
{
10
"
short
"
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
max by (kind) (ghost_delivery_jobs{status=
\"
pending
\"
})
"
,
"
{{kind}}
"
"
B
"
,
{
"
prometheus
"
,
"
prometheus
"
"
max by (kind) (ghost_delivery_jobs{status=
\"
dead
\"
})
"
,
"
{{kind}} gave up
"
"
Jobs waiting for a first try or a retry, and jobs Ghost gave up on (kept 30 days).
"
"
timeseries
"
,
"
Webhook success rate
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
0
,
88
{
{
{
10
"
percentunit
"
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
max(ghost_delivery_attempts{outcome=
\"
succeeded
\"
}) / clamp_min(max(ghost_delivery_attempts{outcome=
\"
succeeded
\"
}) + max(ghost_delivery_attempts{outcome=
\"
failed
\"
}), 1)
"
,
"
success
"
"
Share of webhook attempts in the last five minutes that got a 2xx. One broken receiver pulls this down; the delivery log shows which.
"
"
timeseries
"
,
"
Webhook attempt p99
"
,
{
"
prometheus
"
,
"
prometheus
"
{
8
,
12
,
12
,
88
{
{
{
10
"
ms
"
[]
[
"
A
"
,
{
"
prometheus
"
,
"
prometheus
"
"
max(ghost_delivery_attempt_p99_milliseconds)
"
,
"
p99
"
"
p99 time to send one webhook and read its response, over the last five minutes. The send deadline is 10 s.
"