{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "id": null,
  "uid": "duckdb-cdf-operations",
  "title": "DuckDB CDF operations",
  "schemaVersion": 39,
  "version": 1,
  "tags": [
    "arcgis",
    "duckdb"
  ],
  "timezone": "browser",
  "editable": true,
  "graphTooltip": 1,
  "refresh": "30s",
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "annotations": {
    "list": []
  },
  "templating": {
    "list": [
      {
        "name": "job",
        "label": "Job",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "query": {
          "query": "label_values(arcgis_requests_total, job)",
          "refId": "VariableQuery"
        },
        "refresh": 1,
        "multi": true,
        "includeAll": true,
        "allValue": ".*",
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "options": []
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "timeseries",
      "title": "Requests per second",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (instance) (rate(arcgis_requests_total{job=~\"$job\"}[5m]))",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "Request p95",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 0,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, sum by (le, instance) (rate(arcgis_request_duration_seconds_bucket{job=~\"$job\"}[5m])))",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "Queue wait p95",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 8,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, sum by (le, instance) (rate(arcgis_queue_wait_duration_seconds_bucket{job=~\"$job\"}[5m])))",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 4,
      "type": "timeseries",
      "title": "Failed request fraction",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 8,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (instance) (rate(arcgis_requests_failed{job=~\"$job\"}[5m])) / clamp_min(sum by (instance) (rate(arcgis_requests_total{job=~\"$job\"}[5m])), 0.01)",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Pending operations",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 16,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "arcgis_queue_pending{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Deadline and admission failures",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 16,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum by (instance, reason) (rate(arcgis_queue_failures_total{job=~\"$job\"}[5m]))",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "ops"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "Process RSS",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 24,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "arcgis_process_rss_bytes{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 8,
      "type": "timeseries",
      "title": "Serialized cache bytes",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 24,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "arcgis_cache_bytes{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 9,
      "type": "timeseries",
      "title": "Cache hit fraction",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 32,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "rate(arcgis_cache_hits_total{job=~\"$job\"}[5m]) / clamp_min(rate(arcgis_cache_hits_total{job=~\"$job\"}[5m]) + rate(arcgis_cache_misses_total{job=~\"$job\"}[5m]), 0.01)",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 10,
      "type": "timeseries",
      "title": "Native buffer memory",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 32,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "arcgis_duckdb_buffer_bytes{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 11,
      "type": "timeseries",
      "title": "Observed native spill",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 40,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "arcgis_duckdb_spill_bytes{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 12,
      "type": "timeseries",
      "title": "Observed temporary files",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 40,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "arcgis_duckdb_temporary_file_bytes{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 13,
      "type": "timeseries",
      "title": "Source modification age (when known)",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 48,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "arcgis_source_age_seconds{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 14,
      "type": "timeseries",
      "title": "Source check age",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 48,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "time() - arcgis_source_checked_timestamp_seconds{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 15,
      "type": "timeseries",
      "title": "Resource check age per worker",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 0,
        "y": 56,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "time() - arcgis_duckdb_resource_checked_timestamp_seconds{job=~\"$job\"}",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 16,
      "type": "timeseries",
      "title": "Resource sample failures",
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "gridPos": {
        "x": 12,
        "y": 56,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(arcgis_resource_sample_failures_total{job=~\"$job\"}[5m])",
          "legendFormat": "{{instance}} {{worker}} {{datasource}} {{reason}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    }
  ]
}