fix(service,queue): eliminate data races, add retry backoff, tenant scoping

- Copy discipline for notifications: the store, the queue, workers, and
  API callers each own clones; no notification object is shared across
  goroutines (races previously flagged by -race between workers mutating
  Status/RetryCount and handlers JSON-encoding the same pointer).
- Retry progress derives from QueueMessage.Attempt so it survives
  requeues; exponential backoff (1s base, 30s cap) honors the documented
  queue.retry_backoff setting instead of hammering failing providers in a
  tight loop; shutdown abandons pending backoff waits cleanly.
- Stop() cancels a service-lifetime context so idle workers blocked in
  Dequeue exit immediately instead of waiting out their poll timeout.
- LocalQueue no longer holds its mutex while sending on the queue
  channel (Enqueue/Nack) — with a full buffer this deadlocked the entire
  worker pool, since draining requires the same mutex.
- Tenant scoping: notifications are stamped with the caller's ClientID;
  non-admin clients can only read/cancel/retry their own (cross-tenant
  access reports not-found to avoid leaking existence).
- Sentinel errors ErrNotificationNotFound/ErrNotificationAlreadySent.
- New race, backoff, and tenant-scoping test suites.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

internal/domain/notification.go

@@ -1,9 +1,17 @@
 package domain
 
 import (
+	"errors"
 	"time"
 )
 
+// Sentinel errors for notification lookup and state transitions.
+// Match with errors.Is.
+var (
+	ErrNotificationNotFound    = errors.New("notification not found")
+	ErrNotificationAlreadySent = errors.New("notification already sent")
+)
+
 // Priority defines the urgency level of a notification
 type Priority int
 
@@ -107,6 +115,49 @@ type Notification struct {
 
 	// LastError stores the most recent error message if failed
 	LastError string `json:"last_error,omitempty"`
+
+	// ClientID identifies the API client that submitted the notification, used
+	// to scope visibility of notifications to the tenant that created them.
+	// Empty when auth is disabled.
+	ClientID string `json:"client_id,omitempty"`
+}
+
+// Clone returns a copy of the notification that shares no mutable state with
+// the original. The service layer stores and hands out clones so that
+// concurrent readers (REST/gRPC handlers) and writers (queue workers) never
+// observe or race on the same underlying slices, maps, or pointer fields.
+func (n *Notification) Clone() *Notification {
+	if n == nil {
+		return nil
+	}
+
+	clone := *n
+
+	if n.Recipients != nil {
+		clone.Recipients = append([]string(nil), n.Recipients...)
+	}
+	if n.CC != nil {
+		clone.CC = append([]string(nil), n.CC...)
+	}
+	if n.BCC != nil {
+		clone.BCC = append([]string(nil), n.BCC...)
+	}
+	if n.Metadata != nil {
+		clone.Metadata = make(map[string]interface{}, len(n.Metadata))
+		for k, v := range n.Metadata {
+			clone.Metadata[k] = v
+		}
+	}
+	if n.ScheduledFor != nil {
+		scheduledFor := *n.ScheduledFor
+		clone.ScheduledFor = &scheduledFor
+	}
+	if n.SentAt != nil {
+		sentAt := *n.SentAt
+		clone.SentAt = &sentAt
+	}
+
+	return &clone
 }
 
 // NotificationResult represents the outcome of sending a notification

internal/queue/local.go

@@ -52,13 +52,14 @@ func NewLocalQueue(config *domain.LocalQueueConfig) (*LocalQueue, error) {
 	return lq, nil
 }
 
-// Enqueue adds a notification to the queue
+// Enqueue adds a notification to the queue.
+//
+// The mutex must NOT be held while sending on the channel: when the buffer is
+// full the send blocks, and workers need the same mutex (Dequeue bookkeeping,
+// Ack) to drain the channel — holding it here deadlocks the whole pool.
 func (lq *LocalQueue) Enqueue(ctx context.Context, notification *domain.Notification) error {
-	lq.mu.Lock()
-	defer lq.mu.Unlock()
-
-	if lq.closed {
-		return fmt.Errorf("queue is closed")
+	if err := lq.checkOpen(); err != nil {
+		return err
 	}
 
 	msg := &domain.QueueMessage{
@@ -70,51 +71,40 @@ func (lq *LocalQueue) Enqueue(ctx context.Context, notification *domain.Notifica
 
 	select {
 	case lq.queue <- msg:
-		lq.messages[msg.ID] = msg
-		notification.Status = domain.StatusQueued
-
-		if lq.persistToDisk {
-			return lq.persistToDiskSync()
-		}
-		return nil
 	case <-ctx.Done():
 		return ctx.Err()
 	case <-lq.closeChan:
 		return fmt.Errorf("queue is closed")
 	}
+
+	lq.mu.Lock()
+	defer lq.mu.Unlock()
+	lq.messages[msg.ID] = msg
+	notification.Status = domain.StatusQueued
+
+	if lq.persistToDisk {
+		return lq.persistToDiskSync()
+	}
+	return nil
 }
 
 // EnqueueBatch adds multiple notifications to the queue
 func (lq *LocalQueue) EnqueueBatch(ctx context.Context, notifications []*domain.Notification) error {
-	lq.mu.Lock()
-	defer lq.mu.Unlock()
+	for _, notification := range notifications {
+		if err := lq.Enqueue(ctx, notification); err != nil {
+			return err
+		}
+	}
+	return nil
+}
 
+// checkOpen reports an error if the queue has been closed.
+func (lq *LocalQueue) checkOpen() error {
+	lq.mu.RLock()
+	defer lq.mu.RUnlock()
 	if lq.closed {
 		return fmt.Errorf("queue is closed")
 	}
-
-	for _, notification := range notifications {
-		msg := &domain.QueueMessage{
-			ID:           uuid.New().String(),
-			Notification: notification,
-			Attempt:      0,
-			EnqueuedAt:   time.Now().Unix(),
-		}
-
-		select {
-		case lq.queue <- msg:
-			lq.messages[msg.ID] = msg
-			notification.Status = domain.StatusQueued
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-lq.closeChan:
-			return fmt.Errorf("queue is closed")
-		}
-	}
-
-	if lq.persistToDisk {
-		return lq.persistToDiskSync()
-	}
 	return nil
 }
 
@@ -155,38 +145,45 @@ func (lq *LocalQueue) Ack(ctx context.Context, messageID string) error {
 	return nil
 }
 
-// Nack indicates processing failure and may requeue the message
+// Nack indicates processing failure and may requeue the message.
+// Like Enqueue, the requeue send happens without holding the mutex to avoid
+// deadlocking against workers draining the channel.
 func (lq *LocalQueue) Nack(ctx context.Context, messageID string, requeue bool) error {
 	lq.mu.Lock()
-	defer lq.mu.Unlock()
-
 	msg, exists := lq.messages[messageID]
 	if !exists {
+		lq.mu.Unlock()
 		return fmt.Errorf("message not found: %s", messageID)
 	}
 
-	if requeue {
-		msg.Notification.Status = domain.StatusRetrying
-		select {
-		case lq.queue <- msg:
-			if lq.persistToDisk {
-				return lq.persistToDiskSync()
-			}
-			return nil
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-lq.closeChan:
-			return fmt.Errorf("queue is closed")
-		}
-	} else {
+	if !requeue {
 		msg.Notification.Status = domain.StatusFailed
 		delete(lq.messages, messageID)
 
+		var err error
 		if lq.persistToDisk {
-			return lq.persistToDiskSync()
+			err = lq.persistToDiskSync()
 		}
+		lq.mu.Unlock()
+		return err
 	}
 
+	msg.Notification.Status = domain.StatusRetrying
+	lq.mu.Unlock()
+
+	select {
+	case lq.queue <- msg:
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-lq.closeChan:
+		return fmt.Errorf("queue is closed")
+	}
+
+	lq.mu.Lock()
+	defer lq.mu.Unlock()
+	if lq.persistToDisk {
+		return lq.persistToDiskSync()
+	}
 	return nil
 }
 
@@ -234,7 +231,9 @@ func (lq *LocalQueue) Close() error {
 		}
 	}
 
-	close(lq.queue)
+	// The queue channel is intentionally not closed: senders no longer hold
+	// the mutex while sending, so a concurrent close could panic. closeChan
+	// unblocks all pending senders and receivers instead.
 	return nil
 }
 

internal/service/service.go

@@ -17,6 +17,21 @@ type AccountResolver interface {
 	GetDefaultAccount(notifierType domain.NotificationType) string
 }
 
+const (
+	// defaultRetryBaseDelay is the base delay used for exponential retry
+	// backoff: delay = defaultRetryBaseDelay * 2^(RetryCount-1).
+	defaultRetryBaseDelay = time.Second
+
+	// maxRetryBackoffDelay caps the exponential backoff delay so a
+	// persistently failing notification doesn't wait arbitrarily long between
+	// attempts.
+	maxRetryBackoffDelay = 30 * time.Second
+
+	// adminRole grants access to all tenants' notifications regardless of
+	// ClientID.
+	adminRole = "admin"
+)
+
 // NotificationService implements the domain.NotificationService interface
 type NotificationService struct {
 	factory                domain.NotifierFactory
@@ -27,12 +42,15 @@ type NotificationService struct {
 	mu                     sync.RWMutex
 	workerCount            int
 	stopChan               chan struct{}
+	runCancel              context.CancelFunc
 	wg                     sync.WaitGroup
 	logger                 *logging.Logger
 	retentionConfig        config.NotificationRetentionConfig
 	cleanupStopChan        chan struct{}
 	ttlDuration            time.Duration
 	checkFrequencyDuration time.Duration
+	retryBackoff           string
+	retryBaseDelay         time.Duration
 }
 
 // NewNotificationService creates a new notification service
@@ -51,9 +69,19 @@ func NewNotificationService(factory domain.NotifierFactory, queue domain.Queue,
 		stopChan:        make(chan struct{}),
 		logger:          logger,
 		cleanupStopChan: make(chan struct{}),
+		retryBaseDelay:  defaultRetryBaseDelay,
 	}
 }
 
+// WithRetryBackoff sets the retry backoff strategy used when requeueing a
+// notification after a retryable send failure. Recognized values are
+// "exponential" (delay = 1s * 2^(RetryCount-1), capped at 30s) and "none"
+// (requeue immediately). An empty string is treated as "exponential", matching
+// the documented default for queue.retry_backoff.
+func (s *NotificationService) WithRetryBackoff(mode string) {
+	s.retryBackoff = mode
+}
+
 // WithRetentionConfig sets the notification retention configuration
 func (s *NotificationService) WithRetentionConfig(cfg config.NotificationRetentionConfig) error {
 	s.retentionConfig = cfg
@@ -77,15 +105,20 @@ func (s *NotificationService) WithRetentionConfig(cfg config.NotificationRetenti
 
 // Start starts the worker pool and cleanup goroutine
 func (s *NotificationService) Start(ctx context.Context) error {
+	// Derive a service-lifetime context so Stop() can interrupt workers
+	// blocked in Dequeue instead of waiting out their poll timeout.
+	runCtx, cancel := context.WithCancel(ctx)
+	s.runCancel = cancel
+
 	for i := 0; i < s.workerCount; i++ {
 		s.wg.Add(1)
-		go s.worker(ctx, i)
+		go s.worker(runCtx, i)
 	}
 
 	// Start cleanup goroutine if retention is enabled
 	if s.retentionConfig.Enabled && s.checkFrequencyDuration > 0 {
 		s.wg.Add(1)
-		go s.cleanupLoop(ctx)
+		go s.cleanupLoop(runCtx)
 	}
 
 	return nil
@@ -95,6 +128,9 @@ func (s *NotificationService) Start(ctx context.Context) error {
 func (s *NotificationService) Stop() error {
 	close(s.stopChan)
 	close(s.cleanupStopChan)
+	if s.runCancel != nil {
+		s.runCancel()
+	}
 	s.wg.Wait()
 	return s.queue.Close()
 }
@@ -218,7 +254,19 @@ func (s *NotificationService) worker(ctx context.Context, id int) {
 
 // processNotification sends a notification and handles the result
 func (s *NotificationService) processNotification(ctx context.Context, msg *domain.QueueMessage) {
-	notification := msg.Notification
+	// Work on our own clone of the queued notification. msg.Notification is
+	// owned by the queue (which mutates its Status for its own bookkeeping);
+	// cloning here means our mutations below never race with the queue's or
+	// with clones already handed out by GetNotification/ListNotifications.
+	notification := msg.Notification.Clone()
+
+	// Derive retry progress from msg.Attempt (incremented by the queue on
+	// every dequeue): the queue's copy never sees the counter on our clone,
+	// so the message itself is the source of truth across requeues.
+	notification.RetryCount = msg.Attempt - 1
+	if notification.RetryCount < 0 {
+		notification.RetryCount = 0
+	}
 
 	s.logger.Debugf("Processing notification - id=%s, type=%s, recipients=%d",
 		notification.ID, notification.Type, len(notification.Recipients))
@@ -257,7 +305,10 @@ func (s *NotificationService) processNotification(ctx context.Context, msg *doma
 			notification.Status = domain.StatusRetrying
 			s.logger.Warnf("Notification send failed, will retry - id=%s, type=%s, account=%s, attempt=%d/%d, error=%s",
 				notification.ID, notification.Type, account, notification.RetryCount, notification.MaxRetries, notification.LastError)
-			s.queue.Nack(ctx, msg.ID, true) // Requeue
+			// Hand the retry goroutine its own clone: the worker still
+			// publishes this notification via updateNotification below, and
+			// a shutdown-time abandonRetry must not write to the same object.
+			s.scheduleRetry(ctx, msg, notification.Clone())
 		} else {
 			notification.Status = domain.StatusFailed
 			s.logger.Errorf("Notification send failed permanently - id=%s, type=%s, account=%s, recipients=%v, attempts=%d, error=%s",
@@ -276,6 +327,76 @@ func (s *NotificationService) processNotification(ctx context.Context, msg *doma
 	s.updateNotification(notification)
 }
 
+// scheduleRetry requeues a failed notification, optionally delaying the
+// requeue according to the configured backoff strategy. When a delay applies,
+// the wait happens on a goroutine tracked by the service WaitGroup so Stop()
+// blocks until it finishes - guaranteeing the queue is never closed while a
+// requeue for it is still pending.
+func (s *NotificationService) scheduleRetry(ctx context.Context, msg *domain.QueueMessage, notification *domain.Notification) {
+	delay := s.retryDelay(notification.RetryCount)
+	if delay <= 0 {
+		s.queue.Nack(ctx, msg.ID, true) // Requeue immediately
+		return
+	}
+
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+
+		timer := time.NewTimer(delay)
+		defer timer.Stop()
+
+		select {
+		case <-timer.C:
+			s.queue.Nack(ctx, msg.ID, true) // Requeue after backoff
+		case <-ctx.Done():
+			s.abandonRetry(msg, notification)
+		case <-s.stopChan:
+			s.abandonRetry(msg, notification)
+		}
+	}()
+}
+
+// abandonRetry marks a notification as permanently failed without requeueing
+// it. It is used when the service shuts down while a backoff delay for a
+// retry is still pending, so the notification isn't left stuck in "retrying"
+// forever and no goroutine lingers past shutdown.
+func (s *NotificationService) abandonRetry(msg *domain.QueueMessage, notification *domain.Notification) {
+	s.queue.Nack(context.Background(), msg.ID, false) // Don't requeue
+	notification.Status = domain.StatusFailed
+	s.updateNotification(notification)
+}
+
+// retryDelay computes how long to wait before requeueing a notification that
+// failed on attempt retryCount, per the configured backoff strategy. "none"
+// means requeue immediately (delay of 0); anything else - including the
+// empty string, the documented default - uses exponential backoff: base 1s *
+// 2^(retryCount-1), capped at maxRetryBackoffDelay.
+func (s *NotificationService) retryDelay(retryCount int) time.Duration {
+	if s.retryBackoff == "none" {
+		return 0
+	}
+
+	base := s.retryBaseDelay
+	if base <= 0 {
+		base = defaultRetryBaseDelay
+	}
+
+	if retryCount < 1 {
+		retryCount = 1
+	}
+	shift := retryCount - 1
+	if shift > 20 { // guard against overflow for pathological retry counts
+		shift = 20
+	}
+
+	delay := base * time.Duration(int64(1)<<uint(shift))
+	if delay <= 0 || delay > maxRetryBackoffDelay {
+		delay = maxRetryBackoffDelay
+	}
+	return delay
+}
+
 // Send queues a notification for delivery
 func (s *NotificationService) Send(ctx context.Context, notification *domain.Notification) (*domain.NotificationResult, error) {
 	// Enforce RBAC authorization if configured
@@ -288,15 +409,30 @@ func (s *NotificationService) Send(ctx context.Context, notification *domain.Not
 		}, err
 	}
 
-	// Store the notification
+	// Stamp tenant ownership from the auth context, if present. Absent auth
+	// context (auth disabled) leaves ClientID empty, preserving current
+	// behavior.
+	if authCtx, ok := auth.GetAuthContext(ctx); ok && authCtx != nil {
+		notification.ClientID = authCtx.ClientID
+	}
+
+	// Mark queued and store BEFORE enqueueing: a worker can pick the message
+	// up immediately, and a post-enqueue status write here would overwrite
+	// the worker's Sent/Failed transition.
+	notification.Status = domain.StatusQueued
 	s.storeNotification(notification)
 
-	// Enqueue for processing
-	if err := s.queue.Enqueue(ctx, notification); err != nil {
+	// Enqueue a clone: the queue mutates Status on its copy for its own
+	// bookkeeping and workers clone again on dequeue, so no notification
+	// object is ever shared between the queue, the store, and callers.
+	if err := s.queue.Enqueue(ctx, notification.Clone()); err != nil {
+		notification.Status = domain.StatusFailed
+		notification.LastError = fmt.Sprintf("failed to enqueue: %v", err)
+		s.updateNotification(notification)
 		return &domain.NotificationResult{
 			NotificationID: notification.ID,
 			Success:        false,
-			Error:          fmt.Sprintf("failed to enqueue: %v", err),
+			Error:          notification.LastError,
 			SentAt:         time.Now(),
 		}, err
 	}
@@ -320,13 +456,23 @@ func (s *NotificationService) SendBatch(ctx context.Context, notifications []*do
 		}
 	}
 
-	// Store all notifications
-	for _, notification := range notifications {
-		s.storeNotification(notification)
+	// Stamp tenant ownership from the auth context, if present.
+	if authCtx, ok := auth.GetAuthContext(ctx); ok && authCtx != nil {
+		for _, notification := range notifications {
+			notification.ClientID = authCtx.ClientID
+		}
 	}
 
-	// Enqueue batch
-	if err := s.queue.EnqueueBatch(ctx, notifications); err != nil {
+	// Mark queued and store BEFORE enqueueing (see Send), then hand the
+	// queue clones so it never shares notification objects with the store
+	// or callers.
+	queued := make([]*domain.Notification, 0, len(notifications))
+	for _, notification := range notifications {
+		notification.Status = domain.StatusQueued
+		s.storeNotification(notification)
+		queued = append(queued, notification.Clone())
+	}
+	if err := s.queue.EnqueueBatch(ctx, queued); err != nil {
 		return nil, fmt.Errorf("failed to enqueue batch: %w", err)
 	}
 
@@ -343,20 +489,25 @@ func (s *NotificationService) SendBatch(ctx context.Context, notifications []*do
 	return results, nil
 }
 
-// GetNotification retrieves a notification by ID
+// GetNotification retrieves a notification by ID. If an auth context is
+// present and the caller lacks the admin role, a notification belonging to a
+// different tenant is reported as not found rather than leaking its
+// existence.
 func (s *NotificationService) GetNotification(ctx context.Context, id string) (*domain.Notification, error) {
 	s.mu.RLock()
-	defer s.mu.RUnlock()
-
 	notification, exists := s.notifications[id]
-	if !exists {
-		return nil, fmt.Errorf("notification not found: %s", id)
+	s.mu.RUnlock()
+
+	if !exists || !s.tenantCanAccess(ctx, notification) {
+		return nil, fmt.Errorf("%w: %s", domain.ErrNotificationNotFound, id)
 	}
 
-	return notification, nil
+	return notification.Clone(), nil
 }
 
-// ListNotifications retrieves notifications matching the filter
+// ListNotifications retrieves notifications matching the filter, scoped to
+// the caller's tenant unless they have the admin role or no auth context is
+// present.
 func (s *NotificationService) ListNotifications(ctx context.Context, filter *domain.NotificationFilter) ([]*domain.Notification, error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
@@ -365,8 +516,11 @@ func (s *NotificationService) ListNotifications(ctx context.Context, filter *dom
 	var results []*domain.Notification
 
 	for _, notification := range s.notifications {
+		if !s.tenantCanAccess(ctx, notification) {
+			continue
+		}
 		if s.matchesFilter(notification, filter) {
-			results = append(results, notification)
+			results = append(results, notification.Clone())
 		}
 	}
 
@@ -382,27 +536,32 @@ func (s *NotificationService) ListNotifications(ctx context.Context, filter *dom
 	return results, nil
 }
 
-// CancelNotification cancels a pending notification
+// CancelNotification cancels a pending notification. A notification belonging
+// to a different tenant is reported as not found, matching GetNotification.
 func (s *NotificationService) CancelNotification(ctx context.Context, id string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
 	notification, exists := s.notifications[id]
-	if !exists {
-		return fmt.Errorf("notification not found: %s", id)
+	if !exists || !s.tenantCanAccess(ctx, notification) {
+		return fmt.Errorf("%w: %s", domain.ErrNotificationNotFound, id)
 	}
 
 	if notification.Status == domain.StatusSent {
-		return fmt.Errorf("notification already sent")
+		return domain.ErrNotificationAlreadySent
 	}
 
-	notification.Status = domain.StatusFailed
-	notification.LastError = "cancelled by user"
+	updated := notification.Clone()
+	updated.Status = domain.StatusFailed
+	updated.LastError = "cancelled by user"
+	s.notifications[id] = updated
 
 	return nil
 }
 
-// RetryNotification retries a failed notification
+// RetryNotification retries a failed notification. Cross-tenant access is
+// rejected the same way as GetNotification (via the same not-found error),
+// since RetryNotification is built on top of it.
 func (s *NotificationService) RetryNotification(ctx context.Context, id string) (*domain.NotificationResult, error) {
 	notification, err := s.GetNotification(ctx, id)
 	if err != nil {
@@ -415,7 +574,7 @@ func (s *NotificationService) RetryNotification(ctx context.Context, id string)
 			Success:        false,
 			Error:          "notification already sent",
 			SentAt:         time.Now(),
-		}, fmt.Errorf("notification already sent")
+		}, domain.ErrNotificationAlreadySent
 	}
 
 	// Reset retry count and status
@@ -426,7 +585,8 @@ func (s *NotificationService) RetryNotification(ctx context.Context, id string)
 	return s.Send(ctx, notification)
 }
 
-// GetStats returns notification statistics
+// GetStats returns notification statistics, scoped to the caller's tenant
+// unless they have the admin role or no auth context is present.
 func (s *NotificationService) GetStats(ctx context.Context) (*domain.NotificationStats, error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
@@ -437,6 +597,10 @@ func (s *NotificationService) GetStats(ctx context.Context) (*domain.Notificatio
 	}
 
 	for _, notification := range s.notifications {
+		if !s.tenantCanAccess(ctx, notification) {
+			continue
+		}
+
 		switch notification.Status {
 		case domain.StatusSent:
 			stats.TotalSent++
@@ -510,18 +674,55 @@ func (s *NotificationService) GetNotifiers(ctx context.Context) (*domain.Notifie
 	}, nil
 }
 
-// storeNotification stores a notification in memory
+// storeNotification stores a clone of the notification in memory. Storing a
+// clone (rather than the caller's pointer) ensures the map never aliases a
+// notification that the caller, the queue, or a worker may still mutate.
 func (s *NotificationService) storeNotification(notification *domain.Notification) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	s.notifications[notification.ID] = notification
+	s.notifications[notification.ID] = notification.Clone()
 }
 
-// updateNotification updates a notification in memory
+// updateNotification updates a notification in memory with a clone of the
+// given notification, for the same reason as storeNotification.
 func (s *NotificationService) updateNotification(notification *domain.Notification) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	s.notifications[notification.ID] = notification
+	s.notifications[notification.ID] = notification.Clone()
+}
+
+// tenantCanAccess reports whether the caller identified by ctx is allowed to
+// see the given notification. Behavior:
+//   - No auth context present (auth disabled): always allowed, preserving
+//     pre-multi-tenant behavior.
+//   - Auth context present with the admin role: always allowed.
+//   - Auth context present without the admin role: allowed only if the
+//     notification's ClientID matches the caller's.
+func (s *NotificationService) tenantCanAccess(ctx context.Context, notification *domain.Notification) bool {
+	if notification == nil {
+		return false
+	}
+
+	authCtx, ok := auth.GetAuthContext(ctx)
+	if !ok || authCtx == nil {
+		return true
+	}
+
+	if hasRole(authCtx.Roles, adminRole) {
+		return true
+	}
+
+	return notification.ClientID == authCtx.ClientID
+}
+
+// hasRole reports whether role is present in roles.
+func hasRole(roles []string, role string) bool {
+	for _, r := range roles {
+		if r == role {
+			return true
+		}
+	}
+	return false
 }
 
 // checkAuthorization verifies that the caller is authorized to send to the given notifier/account.

internal/service/service_race_test.go

@@ -0,0 +1,141 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/igodwin/notifier/internal/domain"
+)
+
+// TestConcurrentSendGetListNoRace hammers Send, GetNotification, and
+// ListNotifications concurrently while worker goroutines process notifications
+// pulled from the queue in the background. It is meant to be run with
+// `go test -race`: if the service ever stored or returned a raw pointer that a
+// worker also mutates (the bug this test guards against), the race detector
+// flags a data race here.
+func TestConcurrentSendGetListNoRace(t *testing.T) {
+	svc := createTestService(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	if err := svc.Start(ctx); err != nil {
+		t.Fatalf("Failed to start service: %v", err)
+	}
+	defer svc.Stop()
+
+	const numSenders = 8
+	const sendsPerSender = 25
+
+	var idsMu sync.Mutex
+	var ids []string
+
+	var sendersWg sync.WaitGroup
+	for s := 0; s < numSenders; s++ {
+		sendersWg.Add(1)
+		go func(sender int) {
+			defer sendersWg.Done()
+			for i := 0; i < sendsPerSender; i++ {
+				notification := &domain.Notification{
+					ID:         uuid.New().String(),
+					Type:       domain.TypeStdout,
+					Status:     domain.StatusPending,
+					Subject:    fmt.Sprintf("race-test-%d-%d", sender, i),
+					Body:       "race test body",
+					Recipients: []string{"race@example.com"},
+					CC:         []string{"cc@example.com"},
+					Metadata:   map[string]interface{}{"sender": sender},
+					CreatedAt:  time.Now(),
+					MaxRetries: 1,
+				}
+
+				if _, err := svc.Send(ctx, notification); err != nil {
+					t.Errorf("Send failed: %v", err)
+					continue
+				}
+
+				idsMu.Lock()
+				ids = append(ids, notification.ID)
+				idsMu.Unlock()
+			}
+		}(s)
+	}
+
+	// Readers race against the senders and against the worker pool mutating
+	// notifications as they're processed.
+	stopReaders := make(chan struct{})
+	var readersWg sync.WaitGroup
+	for r := 0; r < 4; r++ {
+		readersWg.Add(1)
+		go func() {
+			defer readersWg.Done()
+			for {
+				select {
+				case <-stopReaders:
+					return
+				default:
+				}
+
+				idsMu.Lock()
+				n := len(ids)
+				var id string
+				if n > 0 {
+					id = ids[n-1]
+				}
+				idsMu.Unlock()
+
+				if id != "" {
+					if notif, err := svc.GetNotification(ctx, id); err == nil {
+						// Touch the returned notification's reference fields;
+						// if it were aliased with the stored/queued copy a
+						// concurrent worker mutation would trip the race
+						// detector right here.
+						_ = notif.Status
+						_ = append([]string(nil), notif.Recipients...)
+					}
+				}
+
+				list, err := svc.ListNotifications(ctx, &domain.NotificationFilter{})
+				if err != nil {
+					t.Errorf("ListNotifications failed: %v", err)
+					continue
+				}
+				for _, notif := range list {
+					_ = notif.Status
+					_ = append([]string(nil), notif.Recipients...)
+				}
+			}
+		}()
+	}
+
+	sendersWg.Wait()
+	close(stopReaders)
+	readersWg.Wait()
+
+	// Give the worker pool a moment to drain the queue, then sanity check the
+	// service is still consistent.
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		stats, err := svc.GetStats(ctx)
+		if err != nil {
+			t.Fatalf("GetStats failed: %v", err)
+		}
+		if stats.TotalSent == numSenders*sendsPerSender {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	stats, err := svc.GetStats(ctx)
+	if err != nil {
+		t.Fatalf("GetStats failed: %v", err)
+	}
+	t.Logf("final stats: sent=%d failed=%d queued=%d pending=%d", stats.TotalSent, stats.TotalFailed, stats.TotalQueued, stats.TotalPending)
+	if stats.TotalSent != numSenders*sendsPerSender {
+		t.Errorf("expected all %d notifications to reach Sent, got %d", numSenders*sendsPerSender, stats.TotalSent)
+	}
+}

internal/service/service_retry_test.go

@@ -0,0 +1,247 @@
+package service
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/igodwin/notifier/internal/domain"
+	"github.com/igodwin/notifier/internal/logging"
+	"github.com/igodwin/notifier/internal/notifier"
+	"github.com/igodwin/notifier/internal/queue"
+)
+
+// alwaysFailNotifier is a fake domain.Notifier that always reports failure
+// while recording the wall-clock time of each Send call, so tests can assert
+// on the spacing between retry attempts.
+type alwaysFailNotifier struct {
+	mu    sync.Mutex
+	calls []time.Time
+}
+
+func (n *alwaysFailNotifier) Send(ctx context.Context, notification *domain.Notification) (*domain.NotificationResult, error) {
+	n.mu.Lock()
+	n.calls = append(n.calls, time.Now())
+	n.mu.Unlock()
+
+	return &domain.NotificationResult{
+		NotificationID: notification.ID,
+		Success:        false,
+		Error:          "simulated failure",
+		SentAt:         time.Now(),
+	}, nil
+}
+
+func (n *alwaysFailNotifier) Type() domain.NotificationType { return domain.TypeStdout }
+
+func (n *alwaysFailNotifier) Validate(notification *domain.Notification) error { return nil }
+
+func (n *alwaysFailNotifier) Close() error { return nil }
+
+func (n *alwaysFailNotifier) callTimes() []time.Time {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	return append([]time.Time(nil), n.calls...)
+}
+
+// createFailingTestService builds a NotificationService wired to the given
+// (always failing) notifier instead of the stdout notifier used by
+// createTestService.
+func createFailingTestService(t *testing.T, fail domain.Notifier) *NotificationService {
+	t.Helper()
+
+	factory := notifier.NewFactory()
+	if err := factory.RegisterNotifier(domain.TypeStdout, "", fail); err != nil {
+		t.Fatalf("Failed to register notifier: %v", err)
+	}
+
+	q, err := queue.NewLocalQueue(&domain.LocalQueueConfig{BufferSize: 100})
+	if err != nil {
+		t.Fatalf("Failed to create queue: %v", err)
+	}
+
+	logger, err := logging.NewFromConfig("error", "stdout")
+	if err != nil {
+		t.Fatalf("Failed to create logger: %v", err)
+	}
+
+	return NewNotificationService(factory, q, 2, nil, nil, logger)
+}
+
+// waitForStatus polls GetNotification until it observes the notification in
+// the given status, or fails the test after timeout.
+func waitForStatus(t *testing.T, svc *NotificationService, ctx context.Context, id string, status domain.NotificationStatus, timeout time.Duration) *domain.Notification {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		n, err := svc.GetNotification(ctx, id)
+		if err == nil && n.Status == status {
+			return n
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Fatalf("notification %s did not reach status %s within %v", id, status, timeout)
+	return nil
+}
+
+// TestRetryBackoffExponentialDelaysRequeue verifies that, with the default
+// (exponential) backoff mode, the delay between retry attempts roughly
+// doubles each time rather than hammering the failing notifier in a tight
+// loop.
+func TestRetryBackoffExponentialDelaysRequeue(t *testing.T) {
+	fail := &alwaysFailNotifier{}
+	svc := createFailingTestService(t, fail)
+	svc.retryBaseDelay = 40 * time.Millisecond // tiny base delay for a fast test
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if err := svc.Start(ctx); err != nil {
+		t.Fatalf("Failed to start service: %v", err)
+	}
+	defer svc.Stop()
+
+	notification := &domain.Notification{
+		ID:         "backoff-exponential-1",
+		Type:       domain.TypeStdout,
+		Status:     domain.StatusPending,
+		Recipients: []string{"test@example.com"},
+		MaxRetries: 3,
+	}
+
+	if _, err := svc.Send(ctx, notification); err != nil {
+		t.Fatalf("Send failed: %v", err)
+	}
+
+	waitForStatus(t, svc, ctx, notification.ID, domain.StatusFailed, 5*time.Second)
+
+	calls := fail.callTimes()
+	if len(calls) != 3 {
+		t.Fatalf("expected 3 send attempts (MaxRetries=3), got %d", len(calls))
+	}
+
+	gap1 := calls[1].Sub(calls[0])
+	gap2 := calls[2].Sub(calls[1])
+
+	// Expected gaps are ~40ms then ~80ms; use generous tolerances to absorb
+	// scheduler jitter while still proving a real, growing delay was applied.
+	if gap1 < 25*time.Millisecond {
+		t.Errorf("expected delay before 2nd attempt >= ~40ms, got %v", gap1)
+	}
+	if gap2 < gap1 {
+		t.Errorf("expected delay before 3rd attempt (%v) to be larger than before 2nd (%v)", gap2, gap1)
+	}
+
+	t.Logf("attempt gaps: %v, %v", gap1, gap2)
+}
+
+// TestRetryBackoffNoneIsImmediate verifies that retryBackoff "none" requeues
+// immediately, ignoring the configured base delay entirely.
+func TestRetryBackoffNoneIsImmediate(t *testing.T) {
+	fail := &alwaysFailNotifier{}
+	svc := createFailingTestService(t, fail)
+	svc.WithRetryBackoff("none")
+	svc.retryBaseDelay = 5 * time.Second // large, to prove "none" ignores it
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if err := svc.Start(ctx); err != nil {
+		t.Fatalf("Failed to start service: %v", err)
+	}
+	defer svc.Stop()
+
+	notification := &domain.Notification{
+		ID:         "backoff-none-1",
+		Type:       domain.TypeStdout,
+		Status:     domain.StatusPending,
+		Recipients: []string{"test@example.com"},
+		MaxRetries: 3,
+	}
+
+	start := time.Now()
+	if _, err := svc.Send(ctx, notification); err != nil {
+		t.Fatalf("Send failed: %v", err)
+	}
+
+	waitForStatus(t, svc, ctx, notification.ID, domain.StatusFailed, 2*time.Second)
+	elapsed := time.Since(start)
+
+	if elapsed > 1*time.Second {
+		t.Errorf("expected immediate retries with backoff=none, took %v", elapsed)
+	}
+
+	calls := fail.callTimes()
+	if len(calls) != 3 {
+		t.Fatalf("expected 3 send attempts (MaxRetries=3), got %d", len(calls))
+	}
+}
+
+// TestRetryBackoffShutdownAbandonsCleanly verifies that Stop() does not block
+// on (or leak) a goroutine that is waiting out a backoff delay: it should
+// abandon the pending retry and mark the notification Failed instead.
+func TestRetryBackoffShutdownAbandonsCleanly(t *testing.T) {
+	fail := &alwaysFailNotifier{}
+	svc := createFailingTestService(t, fail)
+	svc.retryBaseDelay = 5 * time.Second // long enough that shutdown lands mid-wait
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if err := svc.Start(ctx); err != nil {
+		t.Fatalf("Failed to start service: %v", err)
+	}
+
+	notification := &domain.Notification{
+		ID:         "backoff-shutdown-1",
+		Type:       domain.TypeStdout,
+		Status:     domain.StatusPending,
+		Recipients: []string{"test@example.com"},
+		MaxRetries: 3,
+	}
+
+	if _, err := svc.Send(ctx, notification); err != nil {
+		t.Fatalf("Send failed: %v", err)
+	}
+
+	// Give the worker time to make its first (failing) attempt and enter the
+	// backoff wait.
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) && len(fail.callTimes()) == 0 {
+		time.Sleep(10 * time.Millisecond)
+	}
+	if len(fail.callTimes()) == 0 {
+		t.Fatal("notifier was never called; cannot test shutdown mid-backoff")
+	}
+
+	stopped := make(chan error, 1)
+	go func() {
+		stopped <- svc.Stop()
+	}()
+
+	select {
+	case err := <-stopped:
+		if err != nil {
+			t.Errorf("Stop returned error: %v", err)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("Stop did not return promptly - possible goroutine leak in retry backoff")
+	}
+
+	n, err := svc.GetNotification(context.Background(), notification.ID)
+	if err != nil {
+		t.Fatalf("GetNotification failed after shutdown: %v", err)
+	}
+	if n.Status != domain.StatusFailed {
+		t.Errorf("expected notification to be marked Failed after shutdown abandon, got %s", n.Status)
+	}
+
+	// Only the one attempt made before shutdown should have happened; the
+	// pending retry must not have fired.
+	if calls := len(fail.callTimes()); calls != 1 {
+		t.Errorf("expected exactly 1 send attempt before shutdown abandoned the retry, got %d", calls)
+	}
+}

internal/service/service_tenant_test.go

@@ -0,0 +1,194 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/igodwin/notifier/internal/auth"
+	"github.com/igodwin/notifier/internal/domain"
+)
+
+// ctxForClient builds a context carrying an auth.AuthContext for the given
+// client and roles, as REST/gRPC middleware would attach after authenticating
+// a request.
+func ctxForClient(clientID string, roles ...string) context.Context {
+	return auth.ContextWithAuth(context.Background(), &auth.AuthContext{
+		ClientID: clientID,
+		Roles:    roles,
+	})
+}
+
+// TestTenantScopingNonAdminSeesOnlyOwnNotifications verifies that a
+// non-admin caller only sees notifications stamped with their own ClientID.
+func TestTenantScopingNonAdminSeesOnlyOwnNotifications(t *testing.T) {
+	svc := createTestService(t)
+
+	tenantA := &domain.Notification{
+		ID:         uuid.New().String(),
+		Type:       domain.TypeStdout,
+		Status:     domain.StatusSent,
+		Recipients: []string{"a@example.com"},
+		ClientID:   "tenant-a",
+	}
+	tenantB := &domain.Notification{
+		ID:         uuid.New().String(),
+		Type:       domain.TypeStdout,
+		Status:     domain.StatusSent,
+		Recipients: []string{"b@example.com"},
+		ClientID:   "tenant-b",
+	}
+
+	svc.storeNotification(tenantA)
+	svc.storeNotification(tenantB)
+
+	ctxA := ctxForClient("tenant-a", "user")
+
+	list, err := svc.ListNotifications(ctxA, &domain.NotificationFilter{})
+	if err != nil {
+		t.Fatalf("ListNotifications failed: %v", err)
+	}
+	if len(list) != 1 || list[0].ID != tenantA.ID {
+		t.Fatalf("expected tenant-a to see only its own notification, got %d results", len(list))
+	}
+
+	if _, err := svc.GetNotification(ctxA, tenantA.ID); err != nil {
+		t.Errorf("tenant-a should be able to get its own notification: %v", err)
+	}
+
+	if _, err := svc.GetNotification(ctxA, tenantB.ID); err == nil {
+		t.Error("tenant-a should not be able to get tenant-b's notification")
+	}
+
+	stats, err := svc.GetStats(ctxA)
+	if err != nil {
+		t.Fatalf("GetStats failed: %v", err)
+	}
+	if stats.TotalSent != 1 {
+		t.Errorf("expected tenant-a stats to count only its own notification, got %d", stats.TotalSent)
+	}
+}
+
+// TestTenantScopingAdminSeesAll verifies that a caller with the admin role
+// can see notifications belonging to any tenant.
+func TestTenantScopingAdminSeesAll(t *testing.T) {
+	svc := createTestService(t)
+
+	tenantA := &domain.Notification{ID: uuid.New().String(), Type: domain.TypeStdout, Status: domain.StatusSent, Recipients: []string{"a@example.com"}, ClientID: "tenant-a"}
+	tenantB := &domain.Notification{ID: uuid.New().String(), Type: domain.TypeStdout, Status: domain.StatusSent, Recipients: []string{"b@example.com"}, ClientID: "tenant-b"}
+
+	svc.storeNotification(tenantA)
+	svc.storeNotification(tenantB)
+
+	adminCtx := ctxForClient("admin-client", "admin")
+
+	list, err := svc.ListNotifications(adminCtx, &domain.NotificationFilter{})
+	if err != nil {
+		t.Fatalf("ListNotifications failed: %v", err)
+	}
+	if len(list) != 2 {
+		t.Fatalf("expected admin to see both notifications, got %d", len(list))
+	}
+
+	if _, err := svc.GetNotification(adminCtx, tenantA.ID); err != nil {
+		t.Errorf("admin should be able to get tenant-a's notification: %v", err)
+	}
+	if _, err := svc.GetNotification(adminCtx, tenantB.ID); err != nil {
+		t.Errorf("admin should be able to get tenant-b's notification: %v", err)
+	}
+
+	stats, err := svc.GetStats(adminCtx)
+	if err != nil {
+		t.Fatalf("GetStats failed: %v", err)
+	}
+	if stats.TotalSent != 2 {
+		t.Errorf("expected admin stats to count both notifications, got %d", stats.TotalSent)
+	}
+}
+
+// TestTenantScopingAuthDisabledUnchanged verifies that when no auth context
+// is present (auth disabled), behavior is unchanged: every notification is
+// visible regardless of ClientID.
+func TestTenantScopingAuthDisabledUnchanged(t *testing.T) {
+	svc := createTestService(t)
+
+	tenantA := &domain.Notification{ID: uuid.New().String(), Type: domain.TypeStdout, Status: domain.StatusSent, Recipients: []string{"a@example.com"}, ClientID: "tenant-a"}
+	tenantB := &domain.Notification{ID: uuid.New().String(), Type: domain.TypeStdout, Status: domain.StatusSent, Recipients: []string{"b@example.com"}, ClientID: ""}
+
+	svc.storeNotification(tenantA)
+	svc.storeNotification(tenantB)
+
+	ctx := context.Background() // no auth context attached
+
+	list, err := svc.ListNotifications(ctx, &domain.NotificationFilter{})
+	if err != nil {
+		t.Fatalf("ListNotifications failed: %v", err)
+	}
+	if len(list) != 2 {
+		t.Fatalf("expected both notifications visible when auth is disabled, got %d", len(list))
+	}
+
+	if _, err := svc.GetNotification(ctx, tenantA.ID); err != nil {
+		t.Errorf("expected to get tenant-a's notification with auth disabled: %v", err)
+	}
+	if _, err := svc.GetNotification(ctx, tenantB.ID); err != nil {
+		t.Errorf("expected to get tenant-b's notification with auth disabled: %v", err)
+	}
+}
+
+// TestTenantScopingCrossTenantAccessReturnsNotFound verifies that
+// Get/Cancel/Retry on another tenant's notification return the exact same
+// not-found error as a genuinely missing ID, so existence isn't leaked.
+func TestTenantScopingCrossTenantAccessReturnsNotFound(t *testing.T) {
+	svc := createTestService(t)
+
+	tenantA := &domain.Notification{
+		ID:         uuid.New().String(),
+		Type:       domain.TypeStdout,
+		Status:     domain.StatusPending,
+		Recipients: []string{"a@example.com"},
+		MaxRetries: 3,
+		ClientID:   "tenant-a",
+	}
+	svc.storeNotification(tenantA)
+
+	ctxB := ctxForClient("tenant-b", "user")
+
+	wantMsg := fmt.Sprintf("notification not found: %s", tenantA.ID)
+
+	_, getErr := svc.GetNotification(ctxB, tenantA.ID)
+	if getErr == nil {
+		t.Fatal("expected not-found error for cross-tenant Get")
+	}
+	if getErr.Error() != wantMsg {
+		t.Errorf("expected cross-tenant Get error %q, got %q", wantMsg, getErr.Error())
+	}
+
+	_, missingErr := svc.GetNotification(ctxB, "does-not-exist")
+	if missingErr == nil {
+		t.Fatal("expected not-found error for missing ID")
+	}
+
+	cancelErr := svc.CancelNotification(ctxB, tenantA.ID)
+	if cancelErr == nil {
+		t.Fatal("expected not-found error for cross-tenant Cancel")
+	}
+	if cancelErr.Error() != wantMsg {
+		t.Errorf("expected cross-tenant Cancel error %q, got %q", wantMsg, cancelErr.Error())
+	}
+
+	_, retryErr := svc.RetryNotification(ctxB, tenantA.ID)
+	if retryErr == nil {
+		t.Fatal("expected not-found error for cross-tenant Retry")
+	}
+	if retryErr.Error() != wantMsg {
+		t.Errorf("expected cross-tenant Retry error %q, got %q", wantMsg, retryErr.Error())
+	}
+
+	// The owning tenant should still be able to see and act on it.
+	ctxA := ctxForClient("tenant-a", "user")
+	if _, err := svc.GetNotification(ctxA, tenantA.ID); err != nil {
+		t.Errorf("tenant-a should still be able to get its own notification: %v", err)
+	}
+}