ibute: string;
operator: 'equals' | 'in' | 'gt' | 'lt';
value: string | number | string[];
}
export interface GrowthExperiment {
id: string;
name: string;
status: 'draft' | 'active' | 'paused' | 'completed';
variants: Variant[];
targeting: TargetingRule[];
metrics: MetricId[];
seed: string; // Used for hashing to ensure stability
trafficAllocation: number; // 0 to 100
}
export interface EvaluationContext {
userId: string;
sessionId: string;
attributes: Record<string, string | number>;
}
export interface EvaluationResult {
experimentId: string;
variantId: VariantId;
isEligible: boolean;
payload: Record<string, unknown>;
}
#### 2. Deterministic Decision Engine
The decision engine uses MurmurHash for performance and distribution quality. It evaluates targeting rules first, then hashes the user ID to determine variant allocation.
```typescript
// engine/decision-engine.ts
import murmur from 'murmurhash-native';
export class GrowthDecisionEngine {
private experiments: Map<string, GrowthExperiment> = new Map();
registerExperiment(experiment: GrowthExperiment): void {
this.experiments.set(experiment.id, experiment);
}
evaluate(
experimentId: string,
context: EvaluationContext
): EvaluationResult {
const experiment = this.experiments.get(experimentId);
if (!experiment) {
throw new Error(`Experiment ${experimentId} not found`);
}
// 1. Check Status
if (experiment.status !== 'active') {
return { experimentId, variantId: 'control', isEligible: false, payload: {} };
}
// 2. Check Traffic Allocation
const hashInput = `${experiment.seed}:${context.userId}`;
const hashValue = murmur.v3(hashInput) % 100;
if (hashValue >= experiment.trafficAllocation) {
return { experimentId, variantId: 'control', isEligible: false, payload: {} };
}
// 3. Check Targeting Rules
if (!this.matchesTargeting(experiment.targeting, context.attributes)) {
return { experimentId, variantId: 'control', isEligible: false, payload: {} };
}
// 4. Resolve Variant
const variantIndex = this.selectVariant(hashValue, experiment.variants, experiment.trafficAllocation);
const variant = experiment.variants[variantIndex];
return {
experimentId,
variantId: variant.id,
isEligible: true,
payload: variant.payload,
};
}
private matchesTargeting(rules: TargetingRule[], attrs: Record<string, unknown>): boolean {
return rules.every(rule => {
const attrValue = attrs[rule.attribute];
switch (rule.operator) {
case 'equals': return attrValue === rule.value;
case 'in': return Array.isArray(rule.value) && rule.value.includes(attrValue);
case 'gt': return (attrValue as number) > (rule.value as number);
case 'lt': return (attrValue as number) < (rule.value as number);
default: return true;
}
});
}
private selectVariant(
hashValue: number,
variants: Variant[],
totalAllocation: number
): number {
// Normalize hash to the allocated range
const normalizedHash = (hashValue / totalAllocation) * 100;
let cumulativeWeight = 0;
for (let i = 0; i < variants.length; i++) {
cumulativeWeight += variants[i].weight;
if (normalizedHash <= cumulativeWeight) {
return i;
}
}
return variants.length - 1;
}
}
3. Middleware Integration
Integrate the decision engine into the request lifecycle. This example uses a generic middleware pattern compatible with Express/Fastify.
// middleware/growth-middleware.ts
import { GrowthDecisionEngine, EvaluationContext } from './engine/decision-engine';
export function growthMiddleware(engine: GrowthDecisionEngine) {
return async (req: any, res: any, next: any) => {
const userId = req.user?.id || req.headers['x-anonymous-id'];
if (!userId) return next();
const context: EvaluationContext = {
userId,
sessionId: req.headers['x-session-id'],
attributes: {
plan: req.user?.plan,
region: req.headers['x-geo-region'],
device: req.headers['x-device-type'],
},
};
try {
// Parallel evaluation for multiple experiments
const experimentIds = req.growthExperiments || [];
const results = await Promise.all(
experimentIds.map(id => engine.evaluate(id, context))
);
// Attach results to request for downstream use
req.growthContext = results.reduce((acc, result) => {
acc[result.experimentId] = result;
return acc;
}, {} as Record<string, any>);
next();
} catch (error) {
console.error('Growth evaluation failed:', error);
next(); // Fail open to prevent blocking requests
}
};
}
4. Automation Controller
The automation controller periodically checks metric aggregates against success criteria. This removes manual analysis from the loop.
// controller/automation-controller.ts
import { GrowthExperiment, MetricId } from './types/experiment';
export interface SuccessCriteria {
metric: MetricId;
target: number;
direction: 'increase' | 'decrease';
minSampleSize: number;
confidenceLevel: number; // e.g., 0.95
}
export class AutomationController {
constructor(
private analyticsService: any,
private experimentRegistry: any
) {}
async evaluateExperiments(): Promise<void> {
const activeExperiments = await this.experimentRegistry.getActive();
for (const exp of activeExperiments) {
const metrics = await this.analyticsService.getAggregates(exp.id);
if (this.meetsCriteria(metrics, exp.criteria)) {
await this.experimentRegistry.promote(exp.id);
await this.notifySlack(`Experiment ${exp.id} auto-promoted.`);
} else if (this.isFailing(metrics, exp.criteria)) {
await this.experimentRegistry.rollback(exp.id);
await this.notifySlack(`Experiment ${exp.id} auto-rolled back.`);
}
}
}
private meetsCriteria(metrics: any, criteria: SuccessCriteria): boolean {
const value = metrics[criteria.metric];
if (metrics.sampleSize < criteria.minSampleSize) return false;
const passed = criteria.direction === 'increase'
? value > criteria.target
: value < criteria.target;
return passed && metrics.confidence >= criteria.confidenceLevel;
}
}
Pitfall Guide
1. Coupling Growth Logic to Business Logic
Mistake: Embedding experiment variants directly into business rules or database transactions.
Impact: Creates tight coupling, making it impossible to run experiments on core flows without risking data integrity. Refactoring becomes painful.
Best Practice: Use the decision engine to return a payload or variant ID. Business logic should consume the result via a strategy pattern or switch statement that is isolated from the core domain model.
2. Ignoring Latency in the Decision Engine
Mistake: Making synchronous database calls for every experiment evaluation.
Impact: Adds significant latency to user requests, degrading UX and increasing infrastructure costs.
Best Practice: The decision engine must operate in-memory. Experiment configurations should be loaded into memory or cached via a distributed cache (e.g., Redis) with a short TTL. Use a background sync process to update the cache when configurations change.
3. Data Leakage Across Sessions
Mistake: Using session IDs or cookies for allocation instead of persistent user IDs.
Impact: Users may see different variants across devices or after clearing cookies, corrupting data and frustrating users.
Best Practice: Always hash based on a stable, persistent identifier (userId). For anonymous users, use a deterministic anonymous ID that survives cookie resets, or accept the limitation and exclude anonymous traffic from critical experiments.
4. Peeking at Results
Mistake: Stopping an experiment early because a variant looks like a winner, or continuing because it looks close.
Impact: Inflates false positive rates. Statistical significance is only valid at the pre-calculated sample size.
Best Practice: Implement "blind" automation. The controller should only check results after the minimum sample size is reached. Use Bayesian methods if early stopping is required, but document the risk.
5. Inconsistent Hashing Algorithms
Mistake: Changing the hashing algorithm or seed mid-experiment.
Impact: Causes "switching" where users are reassigned to different variants, violating the stability assumption of A/B testing.
Best Practice: The seed is immutable for the life of an experiment. If you must change allocation logic, archive the experiment and create a new one. Ensure the hash function is consistent across all services (e.g., use the same MurmurHash implementation in Node, Go, and Python).
6. Over-Engineering Analytics Before Validation
Mistake: Building complex data pipelines for every experiment before proving the hypothesis.
Impact: Wastes engineering resources on experiments that fail quickly.
Best Practice: Start with lightweight instrumentation. Use a generic track_experiment_event that includes experiment ID, variant ID, and metric name. Aggregate metrics in a flexible store (e.g., ClickHouse or BigQuery) rather than hardcoding schemas for every experiment.
7. No Fallback Strategy
Mistake: Assuming the decision engine is always available and returning errors on failure.
Impact: Blocks user requests or shows broken UI when the growth service is down.
Best Practice: Implement a fail-open strategy. If the decision engine times out or errors, default to the control variant. Ensure the control variant represents the stable, production-ready state.
Production Bundle
Action Checklist
Decision Matrix
| Scenario | Recommended Approach | Why | Cost Impact |
|---|
| High Traffic, Low Risk | Client-side Resolution | Reduces server load; faster response times. | Low server cost; higher bandwidth. |
| Low Traffic, High Risk | Server-side Resolution | Ensures security and strict allocation control. | Medium server cost; high reliability. |
| Multi-step Funnel | Contextual Resolution | Maintains state across steps; prevents leakage. | High complexity; medium cost. |
| Real-time Personalization | Edge Decision | Ultra-low latency; geographic optimization. | High infra cost; high performance. |
| Compliance-Sensitive | Server-side with Audit | Ensures data privacy and regulatory adherence. | Medium cost; high compliance. |
Configuration Template
Use this JSON template to define experiments in your registry. This structure supports versioning and rollback.
{
"experimentId": "exp-checkout-v2",
"version": 1,
"status": "active",
"name": "Checkout Flow Redesign",
"createdAt": "2024-01-15T10:00:00Z",
"seed": "checkout-flow-2024",
"trafficAllocation": 50,
"variants": [
{
"id": "control",
"weight": 50,
"payload": { "layout": "legacy" }
},
{
"id": "variant-a",
"weight": 50,
"payload": { "layout": "modern", "showUpsell": true }
}
],
"targeting": [
{ "attribute": "plan", "operator": "in", "value": ["pro", "enterprise"] }
],
"metrics": ["conversion_rate", "average_order_value"],
"criteria": {
"conversion_rate": {
"target": 0.05,
"direction": "increase",
"minSampleSize": 10000,
"confidenceLevel": 0.95
}
},
"automation": {
"enabled": true,
"checkInterval": "15m",
"actions": {
"promote": "auto-deploy variant-a",
"rollback": "revert to control"
}
}
}
Quick Start Guide
-
Initialize the Engine:
Install dependencies and instantiate the GrowthDecisionEngine. Load initial configurations from your registry.
npm install murmurhash-native @types/murmurhash-native
const engine = new GrowthDecisionEngine();
engine.registerExperiment(require('./experiments/exp-checkout-v2.json'));
-
Add Middleware:
Apply the growth middleware to your application router. Define which experiments to evaluate per route.
app.use('/checkout', growthMiddleware(engine), (req, res) => {
const exp = req.growthContext['exp-checkout-v2'];
if (exp.isEligible && exp.variantId === 'variant-a') {
res.render('checkout/modern');
} else {
res.render('checkout/legacy');
}
});
-
Track Events:
Emit events with experiment context. Use your analytics SDK to send data.
analytics.track('checkout_completed', {
experiment_id: 'exp-checkout-v2',
variant_id: req.growthContext['exp-checkout-v2'].variantId,
value: orderTotal,
});
-
Monitor Results:
Query your analytics backend to view performance.
SELECT
variant_id,
COUNT(*) as conversions,
SUM(value) as revenue
FROM events
WHERE event_name = 'checkout_completed'
AND experiment_id = 'exp-checkout-v2'
GROUP BY variant_id;
-
Iterate:
Update the experiment configuration in the registry. The decision engine will pick up changes via the cache sync process, allowing real-time adjustments without redeployment.
