fix(service,queue): eliminate data races, add retry backoff, tenant scoping

- Copy discipline for notifications: the store, the queue, workers, and
  API callers each own clones; no notification object is shared across
  goroutines (races previously flagged by -race between workers mutating
  Status/RetryCount and handlers JSON-encoding the same pointer).
- Retry progress derives from QueueMessage.Attempt so it survives
  requeues; exponential backoff (1s base, 30s cap) honors the documented
  queue.retry_backoff setting instead of hammering failing providers in a
  tight loop; shutdown abandons pending backoff waits cleanly.
- Stop() cancels a service-lifetime context so idle workers blocked in
  Dequeue exit immediately instead of waiting out their poll timeout.
- LocalQueue no longer holds its mutex while sending on the queue
  channel (Enqueue/Nack) — with a full buffer this deadlocked the entire
  worker pool, since draining requires the same mutex.
- Tenant scoping: notifications are stamped with the caller's ClientID;
  non-admin clients can only read/cancel/retry their own (cross-tenant
  access reports not-found to avoid leaking existence).
- Sentinel errors ErrNotificationNotFound/ErrNotificationAlreadySent.
- New race, backoff, and tenant-scoping test suites.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-18 09:15:57 -07:00
parent 90287d5da0
commit 21990f2533
6 changed files with 924 additions and 91 deletions
+51
View File
@@ -1,9 +1,17 @@
package domain
import (
"errors"
"time"
)
// Sentinel errors for notification lookup and state transitions.
// Match with errors.Is.
var (
ErrNotificationNotFound = errors.New("notification not found")
ErrNotificationAlreadySent = errors.New("notification already sent")
)
// Priority defines the urgency level of a notification
type Priority int
@@ -107,6 +115,49 @@ type Notification struct {
// LastError stores the most recent error message if failed
LastError string `json:"last_error,omitempty"`
// ClientID identifies the API client that submitted the notification, used
// to scope visibility of notifications to the tenant that created them.
// Empty when auth is disabled.
ClientID string `json:"client_id,omitempty"`
}
// Clone returns a copy of the notification that shares no mutable state with
// the original. The service layer stores and hands out clones so that
// concurrent readers (REST/gRPC handlers) and writers (queue workers) never
// observe or race on the same underlying slices, maps, or pointer fields.
func (n *Notification) Clone() *Notification {
if n == nil {
return nil
}
clone := *n
if n.Recipients != nil {
clone.Recipients = append([]string(nil), n.Recipients...)
}
if n.CC != nil {
clone.CC = append([]string(nil), n.CC...)
}
if n.BCC != nil {
clone.BCC = append([]string(nil), n.BCC...)
}
if n.Metadata != nil {
clone.Metadata = make(map[string]interface{}, len(n.Metadata))
for k, v := range n.Metadata {
clone.Metadata[k] = v
}
}
if n.ScheduledFor != nil {
scheduledFor := *n.ScheduledFor
clone.ScheduledFor = &scheduledFor
}
if n.SentAt != nil {
sentAt := *n.SentAt
clone.SentAt = &sentAt
}
return &clone
}
// NotificationResult represents the outcome of sending a notification