fix(service,queue): eliminate data races, add retry backoff, tenant scoping

- Copy discipline for notifications: the store, the queue, workers, and
  API callers each own clones; no notification object is shared across
  goroutines (races previously flagged by -race between workers mutating
  Status/RetryCount and handlers JSON-encoding the same pointer).
- Retry progress derives from QueueMessage.Attempt so it survives
  requeues; exponential backoff (1s base, 30s cap) honors the documented
  queue.retry_backoff setting instead of hammering failing providers in a
  tight loop; shutdown abandons pending backoff waits cleanly.
- Stop() cancels a service-lifetime context so idle workers blocked in
  Dequeue exit immediately instead of waiting out their poll timeout.
- LocalQueue no longer holds its mutex while sending on the queue
  channel (Enqueue/Nack) — with a full buffer this deadlocked the entire
  worker pool, since draining requires the same mutex.
- Tenant scoping: notifications are stamped with the caller's ClientID;
  non-admin clients can only read/cancel/retry their own (cross-tenant
  access reports not-found to avoid leaking existence).
- Sentinel errors ErrNotificationNotFound/ErrNotificationAlreadySent.
- New race, backoff, and tenant-scoping test suites.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

internal/queue/local.go

@@ -52,13 +52,14 @@ func NewLocalQueue(config *domain.LocalQueueConfig) (*LocalQueue, error) {
 	return lq, nil
 }
 
-// Enqueue adds a notification to the queue
+// Enqueue adds a notification to the queue.
+//
+// The mutex must NOT be held while sending on the channel: when the buffer is
+// full the send blocks, and workers need the same mutex (Dequeue bookkeeping,
+// Ack) to drain the channel — holding it here deadlocks the whole pool.
 func (lq *LocalQueue) Enqueue(ctx context.Context, notification *domain.Notification) error {
-	lq.mu.Lock()
-	defer lq.mu.Unlock()
-
-	if lq.closed {
-		return fmt.Errorf("queue is closed")
+	if err := lq.checkOpen(); err != nil {
+		return err
 	}
 
 	msg := &domain.QueueMessage{
@@ -70,51 +71,40 @@ func (lq *LocalQueue) Enqueue(ctx context.Context, notification *domain.Notifica
 
 	select {
 	case lq.queue <- msg:
-		lq.messages[msg.ID] = msg
-		notification.Status = domain.StatusQueued
-
-		if lq.persistToDisk {
-			return lq.persistToDiskSync()
-		}
-		return nil
 	case <-ctx.Done():
 		return ctx.Err()
 	case <-lq.closeChan:
 		return fmt.Errorf("queue is closed")
 	}
+
+	lq.mu.Lock()
+	defer lq.mu.Unlock()
+	lq.messages[msg.ID] = msg
+	notification.Status = domain.StatusQueued
+
+	if lq.persistToDisk {
+		return lq.persistToDiskSync()
+	}
+	return nil
 }
 
 // EnqueueBatch adds multiple notifications to the queue
 func (lq *LocalQueue) EnqueueBatch(ctx context.Context, notifications []*domain.Notification) error {
-	lq.mu.Lock()
-	defer lq.mu.Unlock()
+	for _, notification := range notifications {
+		if err := lq.Enqueue(ctx, notification); err != nil {
+			return err
+		}
+	}
+	return nil
+}
 
+// checkOpen reports an error if the queue has been closed.
+func (lq *LocalQueue) checkOpen() error {
+	lq.mu.RLock()
+	defer lq.mu.RUnlock()
 	if lq.closed {
 		return fmt.Errorf("queue is closed")
 	}
-
-	for _, notification := range notifications {
-		msg := &domain.QueueMessage{
-			ID:           uuid.New().String(),
-			Notification: notification,
-			Attempt:      0,
-			EnqueuedAt:   time.Now().Unix(),
-		}
-
-		select {
-		case lq.queue <- msg:
-			lq.messages[msg.ID] = msg
-			notification.Status = domain.StatusQueued
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-lq.closeChan:
-			return fmt.Errorf("queue is closed")
-		}
-	}
-
-	if lq.persistToDisk {
-		return lq.persistToDiskSync()
-	}
 	return nil
 }
 
@@ -155,38 +145,45 @@ func (lq *LocalQueue) Ack(ctx context.Context, messageID string) error {
 	return nil
 }
 
-// Nack indicates processing failure and may requeue the message
+// Nack indicates processing failure and may requeue the message.
+// Like Enqueue, the requeue send happens without holding the mutex to avoid
+// deadlocking against workers draining the channel.
 func (lq *LocalQueue) Nack(ctx context.Context, messageID string, requeue bool) error {
 	lq.mu.Lock()
-	defer lq.mu.Unlock()
-
 	msg, exists := lq.messages[messageID]
 	if !exists {
+		lq.mu.Unlock()
 		return fmt.Errorf("message not found: %s", messageID)
 	}
 
-	if requeue {
-		msg.Notification.Status = domain.StatusRetrying
-		select {
-		case lq.queue <- msg:
-			if lq.persistToDisk {
-				return lq.persistToDiskSync()
-			}
-			return nil
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-lq.closeChan:
-			return fmt.Errorf("queue is closed")
-		}
-	} else {
+	if !requeue {
 		msg.Notification.Status = domain.StatusFailed
 		delete(lq.messages, messageID)
 
+		var err error
 		if lq.persistToDisk {
-			return lq.persistToDiskSync()
+			err = lq.persistToDiskSync()
 		}
+		lq.mu.Unlock()
+		return err
 	}
 
+	msg.Notification.Status = domain.StatusRetrying
+	lq.mu.Unlock()
+
+	select {
+	case lq.queue <- msg:
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-lq.closeChan:
+		return fmt.Errorf("queue is closed")
+	}
+
+	lq.mu.Lock()
+	defer lq.mu.Unlock()
+	if lq.persistToDisk {
+		return lq.persistToDiskSync()
+	}
 	return nil
 }
 
@@ -234,7 +231,9 @@ func (lq *LocalQueue) Close() error {
 		}
 	}
 
-	close(lq.queue)
+	// The queue channel is intentionally not closed: senders no longer hold
+	// the mutex while sending, so a concurrent close could panic. closeChan
+	// unblocks all pending senders and receivers instead.
 	return nil
 }