Пробую .NET Core + Kubernetes + appmetrics + prometheus + grafana + jobs + health checks

kind: Deployment
apiVersion: apps/v1
metadata:
  name: metricsdemo
  labels:
    app: web
spec:
  replicas: 2 # сколько подов поднять (инстансов запущенных приложений)
 
  # селектор решает, на какие шаблоны распространяется деплой
  selector:
    matchLabels:
      app: metricsdemo
 
  template:
    metadata:
      labels:
        app: metricsdemo # по этой метке ищет selector в kind: Service
    spec:
      containers:
      - name: metricsdemo # имя деплоя
        image: localhost:5000/sansys/metricsdemo3 # образ в докере
        ports:
        - containerPort: 80 # какой порт слушает приложение внутри докера
 
# ВАЖНО: три дефиса делят файл, как бы на два отдельных ямла
---
 
kind: Service
apiVersion: v1
metadata:
  name: metricsdemo # имя для прометеуса __meta_kubernetes_service_name="metricsdemo", см https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config
  labels:
    apptype: business # имя для прометеуса __meta_kubernetes_service_label_apptype="business" - запомни
    instancetype: web # имя для прометеуса __meta_kubernetes_service_label_instancetype="web"
spec:
  selector:
    app: metricsdemo # селектор приложений по labels:app
  type: LoadBalancer # реверспрокси из вне до подов
  ports:
  - protocol: TCP # имя для прометеуса _meta_kubernetes_service_port_protocol="TCP"
    port: 9376
    targetPort: 80
    name: portapi # имя для прометеуса __meta_kubernetes_service_port_name="portapi"

private static void AutoDiscoverRoutes(HttpContext context)
{
    if (context.Request.Path.Value == "/favicon.ico")
        return;
 
    List<string> keys = new List<string>();
    List<string> vals = new List<string>();
 
    var routeData = context.GetRouteData();
    if (routeData != null)
    {
        keys.AddRange(routeData.Values.Keys);
        vals.AddRange(routeData.Values.Values.Select(p => p.ToString()));
    }
 
    keys.Add("method");     vals.Add(context.Request.Method);
    keys.Add("response");   vals.Add(context.Response.StatusCode.ToString());
    keys.Add("url");        vals.Add(context.Request.Path.Value);
 
    Program.Metrics.Measure.Counter.Increment(new CounterOptions
    {
        Name = "api",
        //ResetOnReporting = true, // обнулять, если коллетор собрал данные
        MeasurementUnit = Unit.Calls,
        Tags = new MetricTags(keys.ToArray(), vals.ToArray())
    });
}

apiVersion: v1
kind: ConfigMap # тип сущности, обрати внимание
metadata:
  name: prometheus-config # имя конфиг-маппы
  namespace: default
  labels:
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: EnsureExists
data:
  # имя файла в конфиге
  prometheus.yml: |
    global:
      scrape_interval:     5s # Default is every 1 minute.
      evaluation_interval: 5s # The default is every 1 minute.
    scrape_configs:
    - job_name: prometheus
      static_configs:
      - targets:
        - localhost:9090
 
    - job_name: business-metrics # просто придумал имя джоба
      metrics_path: /metrics
      kubernetes_sd_configs:
        - role: endpoints # какую сущность мониторить. ещё есть service,pod,ingress
      static_configs:
      - targets:
        - localhost:9090
      relabel_configs: # собираю метрики сервисов только из пространства default и приложений c меткой apptype = business
      - action: keep
        regex: default;business
        source_labels:
        - __meta_kubernetes_namespace
        - __meta_kubernetes_service_label_apptype

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: prometheus
  namespace: default
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: prometheus-server
    spec:
      containers:
        - name: prometheus
          image: prom/prometheus
          args:
            - "--config.file=/etc/config/prometheus.yml"
            - "--web.enable-lifecycle"
          ports:
            - containerPort: 9090
          volumeMounts:
            - name: prometheus-config-volume # какой вольюм монтировать
              mountPath: /etc/config/ # в качестве какой директории
      volumes:
        - name: prometheus-config-volume # объявление вольюма в деплое
          configMap:
            defaultMode: 420
            name: prometheus-config # имя конфиг-маппы
 
---
 
kind: Service
apiVersion: v1
metadata:
  name: prometheus
spec:
  selector:
    app: prometheus-server # селектор приложений по labels:app
  type: LoadBalancer # реверспрокси из вне до подов
  ports:
  - protocol: TCP
    port: 9090
    targetPort: 9090

apiVersion: v1
kind: ConfigMap
metadata:
  creationTimestamp: null
  name: grafana-provisioning-datasources
  namespace: default
data:
  all.yml: |
    datasources:
    - name: 'Prometheus'
      type: 'prometheus'
      access: 'proxy'
      org_id: 1
      url: 'http://prometheus:9090'
      is_default: true
      version: 1
      editable: true
 
---
 
apiVersion: v1
kind: ConfigMap
metadata:
  creationTimestamp: null
  name: grafana-provisioning-dashboards
  namespace: default
data:
  all.yml: |
    apiVersion: 1
     
    providers:
    - name: 'default'
      orgId: 1
      folder: ''
      type: file
      disableDeletion: false
      updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards
      options:
        path: /var/lib/grafana/dashboards
 
---
 
apiVersion: v1
kind: ConfigMap
metadata:
  creationTimestamp: null
  name: grafana-dashboards
  namespace: default
data:
  service-http-requests.json: |
    {
      "annotations": {
        "list": [
          {
            "builtIn": 1,
            "datasource": "-- Grafana --",
            "enable": true,
            "hide": true,
            "iconColor": "rgba(0, 211, 255, 1)",
            "name": "Annotations & Alerts",
            "type": "dashboard"
          }
        ]
      },
      "editable": true,
      "gnetId": null,
      "graphTooltip": 0,
      "links": [],
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "fill": 1,
          "gridPos": {
            "h": 9,
            "w": 12,
            "x": 0,
            "y": 0
          },
          "id": 2,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": true,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum by (response, action, url, app) (delta(application_api[15s]))",
              "format": "time_series",
              "interval": "15s",
              "intervalFactor": 1,
              "legendFormat": "{{app}} {{response}} - {{url}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Http requests",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "refresh": "5s",
      "schemaVersion": 16,
      "style": "dark",
      "tags": [],
      "templating": {
        "list": []
      },
      "time": {
        "from": "now-30m",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "",
      "title": "Business metrics",
      "uid": "Dm0tD0Qik",
      "version": 1
    }

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: grafana
  namespace: default
  labels:
    app: grafana
    component: core
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: grafana
        component: core
    spec:
      containers:
      - image: grafana/grafana
        name: grafana
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 100m
            memory: 100Mi
          requests:
            cpu: 100m
            memory: 100Mi
        env:
          - name: GF_AUTH_BASIC_ENABLED
            value: "true"
          - name: GF_AUTH_ANONYMOUS_ENABLED
            value: "true"
          - name: GF_AUTH_ANONYMOUS_ORG_ROLE
            value: Admin
        readinessProbe:
          httpGet:
            path: /login
            port: 3000
          # initialDelaySeconds: 30
          # timeoutSeconds: 1
        volumeMounts:
        - name: grafana-provisioning-datasources
          mountPath: /etc/grafana/provisioning/datasources/
        - name: grafana-provisioning-dashboards
          mountPath: /etc/grafana/provisioning/dashboards/
        - name: grafana-dashboards
          mountPath: /var/lib/grafana/dashboards/
      volumes:
        - name: grafana-provisioning-datasources
          configMap:
            defaultMode: 420
            name: grafana-provisioning-datasources
        - name: grafana-provisioning-dashboards
          configMap:
            defaultMode: 420
            name: grafana-provisioning-dashboards
        - name: grafana-dashboards
          configMap:
            defaultMode: 420
            name: grafana-dashboards
      nodeSelector:
        beta.kubernetes.io/os: linux
 
---
 
apiVersion: v1
kind: Service
metadata:
  name: grafana
  namespace: default
  labels:
    app: grafana
    component: core
spec:
  type: LoadBalancer
  ports:
  - protocol: TCP
    port: 3000
    targetPort: 3000
  selector:
    app: grafana
    component: core