┌─────────────────────────────────────────────────────────────────────────┐
│                            Hanzo O11y                                   │
├─────────────────────────────────────────────────────────────────────────┤
│                                                                         │
│  Data Sources                                                           │
│  ────────────                                                           │
│                                                                         │
│  ┌──────────┐  ┌──────────┐  ┌──────────┐  ┌───────────────────────┐  │
│  │ Services │  │   Nodes  │  │  Proxies │  │  Application SDKs     │  │
│  │ (pods)   │  │ (hosts)  │  │  (mesh)  │  │  (OTLP / Prometheus)  │  │
│  └────┬─────┘  └────┬─────┘  └────┬─────┘  └──────────┬────────────┘  │
│       │              │              │                    │               │
│       ▼              ▼              ▼                    ▼               │
│  ┌─────────────────────────────────────────────────────────────────┐   │
│  │                   OpenTelemetry Collector                        │   │
│  │   ────────────────────────────────────────                      │   │
│  │   Receives: OTLP (gRPC/HTTP), Prometheus scrape, syslog        │   │
│  │   Processes: batch, filter, transform, tail-sample              │   │
│  │   Exports: to Prometheus, Loki, Tempo                           │   │
│  └──────────┬──────────────────┬──────────────────┬───────────────┘   │
│             │                  │                   │                    │
│     Metrics │          Logs    │          Traces   │                    │
│             ▼                  ▼                   ▼                    │
│  ┌──────────────┐  ┌──────────────┐  ┌──────────────────┐            │
│  │  Prometheus  │  │    Loki      │  │     Tempo         │            │
│  │  :9090       │  │  :3100       │  │   :4317 (OTLP)    │            │
│  │  TSDB 30d    │  │  Index+Chunk │  │   :3200 (query)   │            │
│  │  PromQL      │  │  LogQL       │  │   TraceQL          │            │
│  └──────┬───────┘  └──────┬───────┘  └────────┬─────────┘            │
│         │                  │                    │                       │
│         └──────────────────┼────────────────────┘                      │
│                            ▼                                            │
│              ┌──────────────────────────┐                              │
│              │        Grafana           │                              │
│              │   o11y.hanzo.ai          │                              │
│              │   ──────────────         │                              │
│              │   Dashboards             │                              │
│              │   Explore (logs/traces)  │                              │
│              │   Alert rules + routing  │                              │
│              │   SLO tracking           │                              │
│              └──────────┬───────────────┘                              │
│                         │                                               │
│            ┌────────────┼────────────────┐                             │
│            ▼            ▼                ▼                              │
│  ┌──────────────┐ ┌──────────┐ ┌─────────────────┐                   │
│  │  PagerDuty   │ │  Slack   │ │  Webhooks       │                   │
│  │  (critical)  │ │  (warn)  │ │  (custom)       │                   │
│  └──────────────┘ └──────────┘ └─────────────────┘                   │
│                                                                         │
└─────────────────────────────────────────────────────────────────────────┘

# Push metrics using the OpenTelemetry HTTP endpoint
curl -X POST https://o11y.hanzo.ai/v1/metrics \
  -H "Authorization: Bearer $HANZO_TOKEN" \
  -H "Content-Type: application/x-protobuf" \
  --data-binary @metrics.pb

# Or query Prometheus directly
curl "https://o11y.hanzo.ai:9090/api/v1/query?query=up" \
  -H "Authorization: Bearer $HANZO_TOKEN"

curl "https://o11y.hanzo.ai/loki/api/v1/query_range" \
  -H "Authorization: Bearer $HANZO_TOKEN" \
  --data-urlencode 'query={service="gateway"} |= "error"' \
  --data-urlencode 'start=1708560000' \
  --data-urlencode 'end=1708646400' \
  --data-urlencode 'limit=100'

# Set environment variables for any OTLP-compatible application
export OTEL_EXPORTER_OTLP_ENDPOINT=https://o11y.hanzo.ai:4317
export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $HANZO_TOKEN"
export OTEL_SERVICE_NAME=my-service
export OTEL_RESOURCE_ATTRIBUTES="deployment.environment=production"

# PromQL: Request rate by service (last 5 minutes)
rate(http_requests_total[5m])

# PromQL: 99th percentile latency
histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

# PromQL: Error rate percentage
100 * rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m])

from prometheus_client import Counter, Histogram, start_http_server

# Define custom metrics
inference_requests = Counter(
    'inference_requests_total',
    'Total inference requests',
    ['model', 'status']
)

inference_latency = Histogram(
    'inference_duration_seconds',
    'Inference latency',
    ['model'],
    buckets=[0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
)

# Expose on :8000/metrics
start_http_server(8000)

# Record metrics
with inference_latency.labels(model="qwen3-4b").time():
    result = run_inference(prompt)
inference_requests.labels(model="qwen3-4b", status="ok").inc()

# Push logs directly via the Loki API
curl -X POST https://o11y.hanzo.ai/loki/api/v1/push \
  -H "Authorization: Bearer $HANZO_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "streams": [{
      "stream": { "service": "my-app", "level": "error" },
      "values": [
        ["1708646400000000000", "{\"msg\":\"connection timeout\",\"host\":\"db-01\"}"]
      ]
    }]
  }'

# All error logs from the gateway service
{service="gateway"} |= "error"

# JSON-parsed logs with latency > 1s
{service="engine"} | json | latency > 1s

# Log volume rate by service
sum by (service) (rate({namespace="hanzo"} [5m]))

# Errors with stack traces
{service="console", level="error"} |= "panic" | line_format "{{.msg}}\n{{.stacktrace}}"

# Query traces by service
curl "https://o11y.hanzo.ai/tempo/api/search?service.name=gateway&limit=20" \
  -H "Authorization: Bearer $HANZO_TOKEN"

# Get a specific trace by ID
curl "https://o11y.hanzo.ai/tempo/api/traces/abc123def456" \
  -H "Authorization: Bearer $HANZO_TOKEN"

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter

# Configure exporter
exporter = OTLPSpanExporter(
    endpoint="https://o11y.hanzo.ai:4317",
    headers={"Authorization": f"Bearer {HANZO_TOKEN}"}
)

provider = TracerProvider()
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)

tracer = trace.get_tracer("my-service")

# Create spans
with tracer.start_as_current_span("process_request") as span:
    span.set_attribute("model", "qwen3-4b")
    span.set_attribute("tokens.input", 1024)
    result = process(request)
    span.set_attribute("tokens.output", len(result.tokens))

# Example: High error rate alert
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: service-errors
  namespace: hanzo
spec:
  groups:
    - name: service.rules
      rules:
        - alert: HighErrorRate
          expr: |
            100 * rate(http_requests_total{status=~"5.."}[5m])
            / rate(http_requests_total[5m]) > 1
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Error rate above 1% for {{ $labels.service }}"
            runbook: "https://docs.hanzo.ai/runbooks/high-error-rate"

# Create an alert rule via the API
curl -X POST https://api.hanzo.ai/v1/o11y/alerts \
  -H "Authorization: Bearer $HANZO_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "High latency on gateway",
    "expr": "histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{service=\"gateway\"}[5m])) > 2",
    "for": "5m",
    "severity": "warning",
    "channels": [
      {"type": "slack", "webhook": "https://hooks.slack.com/services/..."},
      {"type": "pagerduty", "routing_key": "your-routing-key"},
      {"type": "webhook", "url": "https://your-app.com/alerts"}
    ]
  }'

# Import a dashboard from JSON
curl -X POST https://o11y.hanzo.ai/api/dashboards/db \
  -H "Authorization: Bearer $GRAFANA_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "dashboard": {
      "title": "My Service Dashboard",
      "panels": [
        {
          "title": "Request Rate",
          "type": "timeseries",
          "targets": [
            {"expr": "rate(http_requests_total{service=\"my-service\"}[5m])"}
          ]
        }
      ]
    },
    "overwrite": false
  }'

# Create an SLO
curl -X POST https://api.hanzo.ai/v1/o11y/slos \
  -H "Authorization: Bearer $HANZO_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "Gateway Availability",
    "description": "99.9% of requests return non-5xx in a 30-day window",
    "sli": {
      "type": "availability",
      "good": "http_requests_total{service=\"gateway\",status!~\"5..\"}",
      "total": "http_requests_total{service=\"gateway\"}"
    },
    "target": 0.999,
    "window": "30d",
    "alerts": {
      "burn_rate_1h": 14.4,
      "burn_rate_6h": 6.0
    }
  }'

# Cluster CPU utilization
1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))

# Memory pressure
node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.1

# Pod restart rate
rate(kube_pod_container_status_restarts_total[1h]) > 0

# PVC usage
kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes > 0.85

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: my-service
  namespace: hanzo
spec:
  selector:
    matchLabels:
      app: my-service
  endpoints:
    - port: metrics
      interval: 15s
      path: /metrics