feat(observability): slog-backed logging, Prometheus metrics, grpc health

- internal/logging now wraps log/slog; logging.format json/text finally
  works (json is the documented default). Same exported API.
- New internal/metrics: /metrics on the configured metrics port with
  notification gauges by status/type, queue depth, and HTTP request
  count/duration labeled by mux route pattern; sampled from service
  stats so the service layer stays metrics-agnostic.
- Standard grpc.health.v1 health service registered (k8s gRPC probes);
  gRPC MaxRecvMsgSize bounded to match the REST 1 MB body limit.
- Dedicated health listener on health_check.port serving /health and
  /readyz (probes now work in grpc-only mode); metrics, health, and
  REST servers all shut down gracefully.
- main wires retry backoff, CORS, readiness checks, and TLS from config.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

internal/metrics/metrics.go

@@ -0,0 +1,159 @@
+// Package metrics exposes Prometheus metrics for the notifier service.
+//
+// Notification totals are sampled from the service's own stats rather than
+// instrumented inline, so the service layer stays metrics-agnostic; the
+// sampling interval bounds staleness at a few seconds, which is fine for
+// counters scraped every 15-60s.
+package metrics
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/igodwin/notifier/internal/domain"
+	"github.com/igodwin/notifier/internal/logging"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// Collector owns the notifier metrics and the sampling loop.
+type Collector struct {
+	registry *prometheus.Registry
+
+	notificationsByStatus *prometheus.GaugeVec
+	notificationsByType   *prometheus.GaugeVec
+	queueDepth            prometheus.Gauge
+
+	httpRequests *prometheus.CounterVec
+	httpDuration *prometheus.HistogramVec
+
+	service domain.NotificationService
+	queue   domain.Queue
+	logger  *logging.Logger
+}
+
+// NewCollector creates and registers the notifier metrics.
+func NewCollector(service domain.NotificationService, queue domain.Queue, logger *logging.Logger) *Collector {
+	registry := prometheus.NewRegistry()
+
+	c := &Collector{
+		registry: registry,
+		service:  service,
+		queue:    queue,
+		logger:   logger,
+		notificationsByStatus: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "notifier_notifications",
+			Help: "Number of tracked notifications by status.",
+		}, []string{"status"}),
+		notificationsByType: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "notifier_notifications_by_type",
+			Help: "Number of tracked notifications by notification type.",
+		}, []string{"type"}),
+		queueDepth: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "notifier_queue_depth",
+			Help: "Number of messages currently waiting in the queue.",
+		}),
+		httpRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "notifier_http_requests_total",
+			Help: "REST API requests by method, path pattern, and status code.",
+		}, []string{"method", "path", "code"}),
+		httpDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "notifier_http_request_duration_seconds",
+			Help:    "REST API request duration.",
+			Buckets: prometheus.DefBuckets,
+		}, []string{"method", "path"}),
+	}
+
+	registry.MustRegister(
+		c.notificationsByStatus,
+		c.notificationsByType,
+		c.queueDepth,
+		c.httpRequests,
+		c.httpDuration,
+		prometheus.NewGoCollector(),
+		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
+	)
+
+	return c
+}
+
+// Handler returns the /metrics HTTP handler.
+func (c *Collector) Handler() http.Handler {
+	return promhttp.HandlerFor(c.registry, promhttp.HandlerOpts{})
+}
+
+// Run samples service stats until ctx is cancelled.
+func (c *Collector) Run(ctx context.Context, interval time.Duration) {
+	if interval <= 0 {
+		interval = 10 * time.Second
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			c.sample(ctx)
+		}
+	}
+}
+
+func (c *Collector) sample(ctx context.Context) {
+	sampleCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	if stats, err := c.service.GetStats(sampleCtx); err == nil {
+		c.notificationsByStatus.Reset()
+		for status, count := range stats.ByStatus {
+			c.notificationsByStatus.WithLabelValues(status).Set(float64(count))
+		}
+		c.notificationsByType.Reset()
+		for typ, count := range stats.ByType {
+			c.notificationsByType.WithLabelValues(typ).Set(float64(count))
+		}
+	} else if c.logger != nil {
+		c.logger.Debugf("metrics: failed to sample service stats: %v", err)
+	}
+
+	if size, err := c.queue.Size(sampleCtx); err == nil {
+		c.queueDepth.Set(float64(size))
+	}
+}
+
+// InstrumentHTTP wraps an HTTP handler with request count and duration
+// metrics. The path label uses the route pattern when available (via
+// gorilla/mux CurrentRoute) to keep cardinality bounded.
+func (c *Collector) InstrumentHTTP(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
+		next.ServeHTTP(sw, r)
+
+		path := routePattern(r)
+		c.httpRequests.WithLabelValues(r.Method, path, strconv.Itoa(sw.status)).Inc()
+		c.httpDuration.WithLabelValues(r.Method, path).Observe(time.Since(start).Seconds())
+	})
+}
+
+type statusWriter struct {
+	http.ResponseWriter
+	status int
+}
+
+func (w *statusWriter) WriteHeader(status int) {
+	w.status = status
+	w.ResponseWriter.WriteHeader(status)
+}
+
+// routePattern extracts the mux route template (e.g. /api/v1/notifications/{id})
+// so metrics don't explode into one series per notification ID.
+func routePattern(r *http.Request) string {
+	if route := currentRoute(r); route != "" {
+		return route
+	}
+	return "unmatched"
+}

internal/metrics/mux.go

@@ -0,0 +1,20 @@
+package metrics
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+)
+
+// currentRoute returns the gorilla/mux path template for the request, if the
+// request was matched by a mux router.
+func currentRoute(r *http.Request) string {
+	route := mux.CurrentRoute(r)
+	if route == nil {
+		return ""
+	}
+	if tmpl, err := route.GetPathTemplate(); err == nil {
+		return tmpl
+	}
+	return ""
+}