Files
notifier/internal/metrics/metrics.go
T
igodwin 72f154ab07 feat(observability): slog-backed logging, Prometheus metrics, grpc health
- internal/logging now wraps log/slog; logging.format json/text finally
  works (json is the documented default). Same exported API.
- New internal/metrics: /metrics on the configured metrics port with
  notification gauges by status/type, queue depth, and HTTP request
  count/duration labeled by mux route pattern; sampled from service
  stats so the service layer stays metrics-agnostic.
- Standard grpc.health.v1 health service registered (k8s gRPC probes);
  gRPC MaxRecvMsgSize bounded to match the REST 1 MB body limit.
- Dedicated health listener on health_check.port serving /health and
  /readyz (probes now work in grpc-only mode); metrics, health, and
  REST servers all shut down gracefully.
- main wires retry backoff, CORS, readiness checks, and TLS from config.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-18 09:16:12 -07:00

160 lines
4.7 KiB
Go

// Package metrics exposes Prometheus metrics for the notifier service.
//
// Notification totals are sampled from the service's own stats rather than
// instrumented inline, so the service layer stays metrics-agnostic; the
// sampling interval bounds staleness at a few seconds, which is fine for
// counters scraped every 15-60s.
package metrics
import (
"context"
"net/http"
"strconv"
"time"
"github.com/igodwin/notifier/internal/domain"
"github.com/igodwin/notifier/internal/logging"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// Collector owns the notifier metrics and the sampling loop.
type Collector struct {
registry *prometheus.Registry
notificationsByStatus *prometheus.GaugeVec
notificationsByType *prometheus.GaugeVec
queueDepth prometheus.Gauge
httpRequests *prometheus.CounterVec
httpDuration *prometheus.HistogramVec
service domain.NotificationService
queue domain.Queue
logger *logging.Logger
}
// NewCollector creates and registers the notifier metrics.
func NewCollector(service domain.NotificationService, queue domain.Queue, logger *logging.Logger) *Collector {
registry := prometheus.NewRegistry()
c := &Collector{
registry: registry,
service: service,
queue: queue,
logger: logger,
notificationsByStatus: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "notifier_notifications",
Help: "Number of tracked notifications by status.",
}, []string{"status"}),
notificationsByType: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "notifier_notifications_by_type",
Help: "Number of tracked notifications by notification type.",
}, []string{"type"}),
queueDepth: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "notifier_queue_depth",
Help: "Number of messages currently waiting in the queue.",
}),
httpRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "notifier_http_requests_total",
Help: "REST API requests by method, path pattern, and status code.",
}, []string{"method", "path", "code"}),
httpDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "notifier_http_request_duration_seconds",
Help: "REST API request duration.",
Buckets: prometheus.DefBuckets,
}, []string{"method", "path"}),
}
registry.MustRegister(
c.notificationsByStatus,
c.notificationsByType,
c.queueDepth,
c.httpRequests,
c.httpDuration,
prometheus.NewGoCollector(),
prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
)
return c
}
// Handler returns the /metrics HTTP handler.
func (c *Collector) Handler() http.Handler {
return promhttp.HandlerFor(c.registry, promhttp.HandlerOpts{})
}
// Run samples service stats until ctx is cancelled.
func (c *Collector) Run(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = 10 * time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
c.sample(ctx)
}
}
}
func (c *Collector) sample(ctx context.Context) {
sampleCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if stats, err := c.service.GetStats(sampleCtx); err == nil {
c.notificationsByStatus.Reset()
for status, count := range stats.ByStatus {
c.notificationsByStatus.WithLabelValues(status).Set(float64(count))
}
c.notificationsByType.Reset()
for typ, count := range stats.ByType {
c.notificationsByType.WithLabelValues(typ).Set(float64(count))
}
} else if c.logger != nil {
c.logger.Debugf("metrics: failed to sample service stats: %v", err)
}
if size, err := c.queue.Size(sampleCtx); err == nil {
c.queueDepth.Set(float64(size))
}
}
// InstrumentHTTP wraps an HTTP handler with request count and duration
// metrics. The path label uses the route pattern when available (via
// gorilla/mux CurrentRoute) to keep cardinality bounded.
func (c *Collector) InstrumentHTTP(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(sw, r)
path := routePattern(r)
c.httpRequests.WithLabelValues(r.Method, path, strconv.Itoa(sw.status)).Inc()
c.httpDuration.WithLabelValues(r.Method, path).Observe(time.Since(start).Seconds())
})
}
type statusWriter struct {
http.ResponseWriter
status int
}
func (w *statusWriter) WriteHeader(status int) {
w.status = status
w.ResponseWriter.WriteHeader(status)
}
// routePattern extracts the mux route template (e.g. /api/v1/notifications/{id})
// so metrics don't explode into one series per notification ID.
func routePattern(r *http.Request) string {
if route := currentRoute(r); route != "" {
return route
}
return "unmatched"
}