tionMeta>();
private redis: Redis;
constructor(redisUrl: string) {
this.redis = new Redis(redisUrl);
}
register(ws: WebSocket, userId: string, roomId: string): void {
const key = ${roomId}:${userId};
this.connections.set(key, {
ws,
userId,
roomId,
lastHeartbeat: Date.now()
});
// Publish presence for cross-node awareness
this.redis.publish('ws:presence', JSON.stringify({ action: 'join', key, nodeId: process.env.NODE_ID }));
}
getConnections(roomId: string): ConnectionMeta[] {
const result: ConnectionMeta[] = [];
for (const [key, meta] of this.connections) {
if (key.startsWith(${roomId}:)) result.push(meta);
}
return result;
}
remove(ws: WebSocket): void {
for (const [key, meta] of this.connections) {
if (meta.ws === ws) {
this.connections.delete(key);
this.redis.publish('ws:presence', JSON.stringify({ action: 'leave', key, nodeId: process.env.NODE_ID }));
break;
}
}
}
async routeMessage(roomId: string, payload: string): Promise<void> {
const local = this.getConnections(roomId);
if (local.length > 0) {
local.forEach(m => {
if (m.ws.readyState === WebSocket.OPEN) m.ws.send(payload);
});
}
// Forward to other nodes if room spans multiple instances
await this.redis.publish(ws:room:${roomId}, payload);
}
}
### Step 2: Cross-Node Message Synchronization
Redis Pub/Sub handles fan-out when recipients span multiple nodes. For high-throughput scenarios, switch to Redis Streams to guarantee ordering and enable consumer groups.
```typescript
import { Redis } from 'ioredis';
export class CrossNodeSync {
private pub: Redis;
private sub: Redis;
private registry: ConnectionRegistry;
constructor(redisUrl: string, registry: ConnectionRegistry) {
this.pub = new Redis(redisUrl);
this.sub = new Redis(redisUrl);
this.registry = registry;
this.subscribeToRooms();
}
private subscribeToRooms(): void {
this.sub.psubscribe('ws:room:*', (err, count) => {
if (err) throw new Error(`Redis subscribe failed: ${err.message}`);
});
this.sub.on('pmessage', (pattern, channel, message) => {
const roomId = channel.split(':').pop()!;
// Avoid echoing back to sender node
const parsed = JSON.parse(message);
if (parsed.originNode !== process.env.NODE_ID) {
this.registry.getConnections(roomId).forEach(m => {
if (m.ws.readyState === WebSocket.OPEN) m.ws.send(parsed.data);
});
}
});
}
async broadcast(roomId: string, data: any): Promise<void> {
const payload = JSON.stringify({
originNode: process.env.NODE_ID,
data
});
await this.pub.publish(`ws:room:${roomId}`, payload);
}
}
Step 3: Heartbeat & Connection Lifecycle
Persistent connections decay without maintenance. Implement server-driven pings and client-driven pong responses. Drop stale connections to free file descriptors.
export function attachHeartbeat(ws: WebSocket, registry: ConnectionRegistry): void {
const interval = setInterval(() => {
if (ws.readyState === WebSocket.CLOSED) {
clearInterval(interval);
registry.remove(ws);
return;
}
if (ws.isAlive === false) {
ws.terminate();
return;
}
ws.isAlive = false;
ws.ping();
}, 30000);
ws.isAlive = true;
ws.on('pong', () => { ws.isAlive = true; });
ws.on('close', () => {
clearInterval(interval);
registry.remove(ws);
});
}
Step 4: Server Initialization
import { WebSocketServer } from 'ws';
import { createServer } from 'http';
import { ConnectionRegistry } from './ConnectionRegistry';
import { CrossNodeSync } from './CrossNodeSync';
import { attachHeartbeat } from './Heartbeat';
const server = createServer();
const wss = new WebSocketServer({ server });
const registry = new ConnectionRegistry(process.env.REDIS_URL!);
const sync = new CrossNodeSync(process.env.REDIS_URL!, registry);
wss.on('connection', (ws, req) => {
const url = new URL(req.url!, `http://${req.headers.host}`);
const userId = url.searchParams.get('user_id')!;
const roomId = url.searchParams.get('room_id')!;
registry.register(ws, userId, roomId);
attachHeartbeat(ws, registry);
ws.on('message', async (raw) => {
const msg = JSON.parse(raw.toString());
await sync.broadcast(roomId, msg);
});
});
server.listen(8080, () => {
console.log(`WebSocket node ${process.env.NODE_ID} listening on 8080`);
});
Architecture Decisions & Rationale
- Redis over Kafka/NATS: WebSocket traffic is typically low-throughput, high-concurrency, and requires sub-10ms delivery. Kafka’s batching and partitioning add latency. NATS is viable but lacks built-in TTLs and connection tracking primitives that simplify room state management.
- Sticky Sessions over Full Mesh: Routing connections to a single node per room/user eliminates O(n²) synchronization. Cross-node pub/sub only triggers when a room spans multiple nodes, which is rare in well-partitioned workloads.
- In-Memory Registry + Redis Pub/Sub: Keeps routing fast (O(1) local lookup) while providing eventual consistency across nodes. Full state replication is unnecessary and memory-prohibitive.
- Separation of Control vs Data Channels: Presence/heartbeat traffic uses a dedicated Redis channel. Message payloads use room-scoped channels. Prevents control traffic from starving data delivery during spikes.
Pitfall Guide
-
Treating WebSockets as Stateless HTTP Endpoints
WebSockets maintain TCP state. Load balancers that terminate TLS and re-establish connections to backend nodes will break the upgrade handshake or drop frames. Always configure L7 proxies to pass through the Upgrade: websocket header and maintain TCP affinity.
-
Blocking the Event Loop with Synchronous Processing
Node.js single-threaded architecture stalls when message handlers perform CPU-bound work or synchronous I/O. A single blocking JSON.parse on a 10MB payload can freeze all connections. Offload heavy processing to worker threads or async queues.
-
Ignoring Connection Lifecycle Management
Networks drop silently. Mobile clients switch networks. Firewalls kill idle TCP streams. Without server-driven pings and client pong responses, connections remain in OPEN state while the underlying socket is dead. This leaks file descriptors and causes message delivery failures.
-
Naive Connection Counting (1 Connection = 1 User)
Users open multiple tabs, mobile apps reconnect aggressively, and bots spawn parallel sessions. Connection counts rarely map 1:1 to active users. Scale based on concurrent TCP connections, not user metrics. Monitor netstat or /proc/sys/fs/file-nr for accurate capacity planning.
-
Over-Pub/Sub-ing High-Frequency Data
Broadcasting every frame of a multiplayer game tick or real-time chart update through Redis saturates the broker and introduces queueing delay. Sample data on the client, use delta compression, or switch to UDP/QUIC for sub-16ms requirements. Pub/Sub is for event-driven state, not continuous streams.
-
Missing Backpressure Handling
Sending messages faster than the client can process causes TCP buffer exhaustion. The ws library queues messages in memory by default. Check ws.bufferedAmount before sending. Drop or batch messages when the buffer exceeds a threshold.
-
Assuming Cloud Load Balancers Handle WebSocket Upgrades Automatically
AWS ALB, GCP Cloud Load Balancing, and Azure Application Gateway require explicit WebSocket configuration. Health checks must use TCP or HTTP with Upgrade header awareness. Without proper health check configuration, nodes appear healthy while WebSocket connections fail silently.
Production Best Practices:
- Set
fs.file-max and ulimit -n to 100k+ per node.
- Use
cluster mode or PM2 with shared memory for registry sync if avoiding Redis.
- Implement exponential backoff on client reconnection (1s, 2s, 4s, 8s, max 30s).
- Separate control plane (auth, presence) from data plane (messages, events).
- Monitor
connections_in_use, redis_pubsub_channels, and event_loop_delay via OpenTelemetry.
Production Bundle
Action Checklist
Decision Matrix
| Scenario | Recommended Approach | Why | Cost Impact |
|---|
| Real-time chat / collaboration | Proxy + Sticky Routing + Redis Pub/Sub | Low message frequency, high concurrency, room-based routing minimizes cross-node hops | Low-Medium (Redis cluster) |
| Live dashboards / telemetry | Centralized Pub/Sub (NATS/Redis Streams) | Broadcast-heavy, stateless routing acceptable, fan-out optimized | Medium (managed broker) |
| Multiplayer games / high-frequency ticks | UDP/QUIC or dedicated game server mesh | Sub-16ms requirement, Redis latency unacceptable, need client-side interpolation | High (custom infra) |
| IoT device fleet | MQTT over WebSockets + broker clustering | Protocol-native QoS, offline buffering, device lifecycle management | Medium-High (managed IoT core) |
| Multi-region deployment | Connection proxy with region-local Redis + cross-region replication | Reduces latency by keeping connections regional, replication handles failover | High (multi-region infra) |
Configuration Template
Nginx L7 Proxy (WebSocket-aware)
upstream ws_backend {
ip_hash; # Sticky sessions by client IP
server 10.0.1.10:8080;
server 10.0.1.11:8080;
server 10.0.1.12:8080;
}
server {
listen 443 ssl http2;
server_name ws.example.com;
ssl_certificate /etc/ssl/certs/ws.pem;
ssl_certificate_key /etc/ssl/private/ws.key;
location /ws/ {
proxy_pass http://ws_backend;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
proxy_buffering off;
}
}
Node.js Environment & Limits
# /etc/security/limits.conf
* soft nofile 100000
* hard nofile 100000
# /etc/sysctl.conf
fs.file-max = 200000
net.core.somaxconn = 65535
net.ipv4.tcp_keepalive_time = 30
net.ipv4.tcp_keepalive_intvl = 10
net.ipv4.tcp_keepalive_probes = 3
# .env
NODE_ENV=production
NODE_ID=ws-node-$(hostname -s)
REDIS_URL=redis://redis-cluster:6379
WS_PORT=8080
HEARTBEAT_INTERVAL=30000
MAX_BUFFERED_AMOUNT=1048576
Quick Start Guide
- Initialize the project:
npm init -y && npm i ws ioredis @types/ws @types/node typescript ts-node
- Create
tsconfig.json: Set module: "commonjs", target: "ES2020", outDir: "./dist", strict: true
- Spin up Redis locally:
docker run -d -p 6379:6379 --name ws-redis redis:7-alpine
- Run the server:
NODE_ID=local-1 REDIS_URL=redis://localhost:6379 npx ts-node src/server.ts
- Test connectivity: Open browser console or use
wscat -c ws://localhost:8080/ws/?user_id=1&room_id=general and send a JSON payload. Verify cross-node delivery by running a second instance with NODE_ID=local-2 and routing to the same room.
