notification-service.config.yaml

Architecture Decisions

1. Message Broker: Use a durable queue (e.g., Kafka, RabbitMQ, or BullMQ) to decouple producers from consumers. This absorbs traffic spikes and allows independent scaling of workers.
2. Sharding Strategy: Partition queues by channel type or tenant to prevent noisy neighbors. High-volume channels (push) should not block critical channels (SMS alerts).
3. Worker Design: Workers must implement token-bucket rate limiting and dynamic batching. Batching should be adaptive, increasing batch size as queue depth grows to amortize connection overhead.
4. Idempotency: Implement a deduplication layer using a fast store (Redis) to prevent duplicate deliveries caused by broker redeliveries or client retries.

Implementation

The following TypeScript implementation demonstrates a robust worker pattern using a generic message broker interface, a token-bucket rate limiter, and dynamic batching.

1. Rate Limiter and Batcher

export class TokenBucketRateLimiter {
  private tokens: number;
  private lastRefill: number;

  constructor(
    private readonly capacity: number,
    private readonly refillRate: number // tokens per second
  ) {
    this.tokens = capacity;
    this.lastRefill = Date.now();
  }

  async consume(count: number): Promise<void> {
    this.refill();
    while (this.tokens < count) {
      const waitTime = ((count - this.tokens) / this.refillRate) * 1000;
      await new Promise((resolve) => setTimeout(resolve, waitTime));
      this.refill();
    }
    this.tokens -= count;
  }

  private refill(): void {
    const now = Date.now();
    const elapsed = (now - this.lastRefill) / 1000;
    this.tokens = Math.min(this.capacity, this.tokens + elapsed * this.refillRate);
    this.lastRefill = now;
  }
}

export class DynamicBatcher<T> {
  private buffer: T[] = [];
  private flushTimer: NodeJS.Timeout | null = null;

  constructor(
    private readonly maxBatchSize: number,
    private readonly maxWaitMs: number,
    private readonly flushCallback: (batch: T[]) => Promise<void>
  ) {}

  add(item: T): void {
    this.buffer.push(item);
    if (this.buffer.length >= this.maxBatchSize) {
      this.flush();
    } else if (!this.flushTimer) {
      this.flushTimer = setTimeout(() => this.flush(), this.maxWaitMs);
    }
  }

  private async flush(): Promise<void> {
    if (this.flushTimer) {
      clearTimeout(this.flushTimer);
      this.flushTimer = null;
    }
    if (this.buffer.length > 0) {
      const batch = this.buffer.splice(0, this.buffer.length);
      await this.flushCallback(batch);
    }
  }
}

2. Notification Worker

import { v4 as uuidv4 } from 'uuid';

export interface Notification {
  id: string;
  userId: string;
  channel: 'email' | 'push' | 'sms';
  payload: Record<string, unknown>;
  tenantId: string;
}

export class NotificationWorker {
  private rateLimiter: TokenBucketRateLimiter;
  private batcher: DynamicBatcher<Notification>;
  private idempotencyStore: Map<string, boolean>; // In prod, use Redis

  constructor(
    private readonly provider: { send: (batch: Notification[]) => Promise<void> },
    rateLimit: number,
    maxBatchSize: number
  ) {
    this.rateLimiter = new TokenBucketRateLimiter(rateLimit, rateLimit);
    this.batcher = new DynamicBatcher(maxBatchSize, 500, (batch) => this.processBatch(batch));
    this.idempotencyStore = new Map();
  }

  async handle(message: Notification): Promise<void> {
    // Idempotency check
    const dedupKey = `${message.id}:${message.channel}`;
    if (this.idempotencyStore.has(dedupKey)) {
      return;
    }

    try {
      this.batcher.add(message);
      this.idempotencyStore.set(dedupKey, true);
      
      // Cleanup idempotency store periodically or use TTL in Redis
      if (this.idempotencyStore.size > 10000) {
        const keys = Array.from(this.idempotencyStore.keys());
        keys.slice(0, 5000).forEach(k => this.idempotencyStore.delete(k));
      }
    } catch (error) {
      // Route to Dead Letter Queue
      console.error(`Failed to enqueue notification ${message.id}`, error);
    }
  }

  private async processBatch(batch: Notification[]): Promise<void> {
    // Consume rate limit tokens based on batch size
    await this.rateLimiter.consume(batch.length);

    try {
      await this.provider.send(batch);
    } catch (error) {
      // Implement retry with jitter for transient failures
      if (this.isRetryable(error)) {
        await this.retryWithJitter(batch);
      } else {
        // Permanent failure, route to DLQ
        console.error(`Permanent failure for batch of ${batch.length} notifications`, error);
      }
    }
  }

  private async retryWithJitter(batch: Notification[]): Promise<void> {
    const retries = 3;
    for (let i = 0; i < retries; i++) {
      const delay = Math.min(1000 * Math.pow(2, i) + Math.random() * 1000, 10000);
      await new Promise(resolve => setTimeout(resolve, delay));
      try {
        await this.provider.send(batch);
        return;
      } catch (error) {
        if (i === retries - 1) throw error;
      }
    }
  }

  private isRetryable(error: unknown): boolean {
    // Check for 429, 5xx, network timeouts
    return true; // Simplified for example
  }
}

3. Worker Initialization and Sharding

// Example setup for sharded workers based on channel
const channels = ['email', 'push', 'sms'] as const;

const workers = channels.map(channel => {
  const provider = createProvider(channel); // Factory for channel-specific provider
  const rateLimit = channel === 'sms' ? 10 : 100; // SMS has stricter limits
  const maxBatch = channel === 'email' ? 50 : 10; // Email supports larger batches

  const worker = new NotificationWorker(provider, rateLimit, maxBatch);
  
  // Subscribe to channel-specific queue
  subscribeToQueue(`notifications-${channel}`, async (msg) => {
    await worker.handle(msg);
  });
  
  return worker;
});

Rationale

The architecture separates concerns to allow independent scaling. The TokenBucketRateLimiter ensures strict adherence to provider constraints, preventing 429 errors. The DynamicBatcher optimizes for throughput by grouping requests, reducing connection overhead. Sharding by channel ensures that rate limits and batch sizes can be tuned per provider without affecting others. The idempotency check prevents duplicate deliveries, which is critical for user trust and cost control.

Pitfall Guide

1. Ignoring Provider Rate Limits: Firing requests without rate limiting leads to immediate throttling. Providers often impose limits per API key, IP, or tenant. Implement token buckets or leaky buckets specific to each provider's documentation.
2. Blocking the Critical Path: Sending notifications synchronously within request handlers increases latency and risks timeouts. Always offload to a message broker. If using Node.js, ensure the broker client is non-blocking.
3. Lack of Idempotency: Message brokers may redeliver messages on worker crashes or network issues. Without idempotency keys, users receive duplicate notifications. Store processed message IDs with a TTL to handle redeliveries safely.
4. Retry Storms: Retrying failed requests without exponential backoff and jitter amplifies load on failing providers, potentially causing outages. Implement randomized backoff to desynchronize retry attempts across workers.
5. Storing PII in Queue Payloads: Queues may be persisted or accessible by multiple services. Avoid storing sensitive user data (phone numbers, emails) directly in the queue payload. Use references or encrypted payloads, and resolve data within the worker securely.
6. Monolithic Provider Logic: Hardcoding provider logic makes it difficult to switch vendors or handle multi-provider failover. Abstract providers behind a common interface and implement a strategy pattern to route traffic based on cost, latency, or provider health.
7. Neglecting Quiet Hours and Timezones: Sending notifications during user sleep hours increases churn and spam reports. Implement timezone-aware scheduling and quiet hour rules at the worker level to defer delivery until appropriate times.

Production Bundle

Action Checklist

• Implement Dead Letter Queues: Configure DLQs for all notification channels to capture permanent failures for analysis and manual retry.
• Add Distributed Tracing: Instrument notification flow with trace IDs to track latency and failures across producer, broker, and worker.
• Configure Rate Limits per Provider: Define rate limits based on provider contracts and implement token buckets in workers.
• Enable Idempotency: Set up a deduplication store with TTLs to prevent duplicate deliveries from broker redeliveries.
• Test Chaos Scenarios: Simulate provider outages and rate limit spikes to validate retry logic and fallback strategies.
• Monitor Cost Metrics: Track cost per notification and aggregate spend by channel to identify optimization opportunities.
• Implement Quiet Hours: Add timezone and quiet hour logic to defer non-critical notifications during user sleep times.
• Shard Queues by Tenant/Channel: Partition queues to isolate noisy tenants and allow channel-specific scaling.

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
High-Volume Marketing Email	Sharded Batching with Provider Failover	Maximizes throughput, reduces API calls, ensures delivery via fallback providers.	Lowers cost per msg via batching; failover may increase cost slightly but improves ROI.
Real-Time Push Notifications	Streaming with Priority Queuing	Low latency is critical; batching adds unacceptable delay. Prioritize critical alerts.	Higher cost per msg due to individual API calls, but necessary for UX.
SMS Critical Alerts	Dedicated Low-Latency Queue	SMS has strict rate limits and high cost; isolation prevents blocking by other traffic.	High cost per msg; isolation ensures delivery reliability for critical paths.
Multi-Channel User Alerts	Fan-Out with Deduplication	Send across channels once per event; deduplication prevents spam if user engages on one channel.	Increases initial send cost; deduplication reduces wasted sends and churn.

Configuration Template

# notification-service.config.yaml
broker:
  type: kafka
  topics:
    - name: notifications-email
      partitions: 12
      replication: 3
    - name: notifications-push
      partitions: 6
      replication: 3
    - name: notifications-sms
      partitions: 4
      replication: 3
  dlq:
    enabled: true
    retention_days: 30

workers:
  email:
    rate_limit: 1000 # requests per second
    batch_size: 50
    max_wait_ms: 500
    retry:
      max_attempts: 3
      backoff_base_ms: 1000
      jitter: true
  push:
    rate_limit: 5000
    batch_size: 10
    max_wait_ms: 100
    retry:
      max_attempts: 2
      backoff_base_ms: 500
      jitter: true
  sms:
    rate_limit: 100
    batch_size: 5
    max_wait_ms: 2000
    retry:
      max_attempts: 4
      backoff_base_ms: 2000
      jitter: true

deduplication:
  store: redis
  ttl_seconds: 86400 # 24 hours
  key_prefix: notif:dedup

observability:
  metrics:
    enabled: true
    labels: [channel, tenant_id, status]
  tracing:
    enabled: true
    export: jaeger

Quick Start Guide

1. Initialize Broker: Deploy Kafka or RabbitMQ with the topics defined in the configuration template. Ensure partitions are sized for expected throughput.
2. Deploy Worker Service: Run the notification worker service with the configuration. Scale worker replicas based on queue lag and CPU utilization.
3. Integrate Producer: Replace synchronous notification calls in your application with async messages to the broker. Include tenant_id, channel, and idempotency_key in the payload.
4. Verify Idempotency: Test by sending duplicate messages to ensure the worker deduplicates correctly using the Redis store.
5. Monitor Metrics: Check Prometheus/Grafana for throughput, latency, and error rates. Validate that rate limits are respected and DLQs are empty under normal load.