Digital product experimentation

config.status !== 'running') return null;

// Create a stable hash for the allocation
const hashInput = `${experimentId}:${targetValue}`;
const hash = createHash('sha256').update(hashInput).digest('hex');

// Convert first 8 hex chars to a number between 0 and 1
const hashValue = parseInt(hash.substring(0, 8), 16) / 0xFFFFFFFF;

let cumulativeWeight = 0;
for (const variant of config.variants) {
  cumulativeWeight += variant.weight;
  if (hashValue < cumulativeWeight) {
    return variant.id;
  }
}

// Fallback to control if floating point issues occur
return config.variants[0].id;

} }

#### 3. React Integration Hook

Provide a React hook that abstracts allocation and exposure tracking. This hook integrates with the allocation engine and emits exposure events.

```typescript
import { useState, useEffect, useCallback } from 'react';

interface UseExperimentResult<T> {
  variant: T | null;
  isLoading: boolean;
  trackExposure: () => void;
}

export function useExperiment<T extends string>(
  experimentId: string,
  userId: string | null,
  allocationEngine: AllocationEngine
): UseExperimentResult<T> {
  const [variant, setVariant] = useState<T | null>(null);
  const [isLoading, setIsLoading] = useState(true);

  const trackExposure = useCallback(() => {
    if (variant) {
      // Emit to event bus/analytics pipeline
      window.analytics?.track('experiment_exposed', {
        experimentId,
        variantId: variant,
        userId,
        timestamp: Date.now()
      });
    }
  }, [experimentId, variant, userId]);

  useEffect(() => {
    if (!userId) {
      setIsLoading(false);
      return;
    }

    // Simulate async config fetch if needed, or use local cache
    const allocatedVariant = allocationEngine.allocate(experimentId, userId);
    setVariant(allocatedVariant as T);
    setIsLoading(false);
  }, [experimentId, userId, allocationEngine]);

  return { variant, isLoading, trackExposure };
}

4. SRM Detection Utility

Include a utility to detect Sample Ratio Mismatch during implementation testing. This helps engineers catch routing bugs before launching.

export function detectSRM(
  expectedRatio: number[],
  observedCounts: number[]
): { isMismatch: boolean; pValue: number } {
  // Chi-squared test approximation
  const totalObserved = observedCounts.reduce((a, b) => a + b, 0);
  const totalExpected = expectedRatio.reduce((a, b) => a + b, 0);
  
  let chiSquare = 0;
  for (let i = 0; i < expectedRatio.length; i++) {
    const expected = (expectedRatio[i] / totalExpected) * totalObserved;
    const observed = observedCounts[i];
    chiSquare += Math.pow(observed - expected, 2) / expected;
  }

  // Simplified p-value lookup (in production, use a statistical library)
  const degreesOfFreedom = expectedRatio.length - 1;
  const pValue = chiSquare > 3.84 && degreesOfFreedom === 1 ? 0.049 : 0.85;

  return {
    isMismatch: pValue < 0.05,
    pValue
  };
}

Pitfall Guide

Common Mistakes and Best Practices

1. Peeking at Results Early

   • Mistake: Checking statistical significance repeatedly during the experiment and stopping early when a p-value drops below 0.05. This inflates the false positive rate up to 30%.
   • Best Practice: Pre-register the sample size based on power analysis. Use sequential testing methods (e.g., SPRT) if early stopping is required.

2. Ignoring Sample Ratio Mismatch (SRM)

   • Mistake: Launching experiments without verifying that the traffic split matches the configuration. SRM indicates a bug in hashing, routing, or client-side filtering.
   • Best Practice: Implement automated SRM checks in the CI/CD pipeline and dashboard. Block analysis if SRM is detected.

3. Metric Definition Ambiguity

   • Mistake: Defining metrics loosely (e.g., "engagement") without a precise SQL or event definition. Different teams calculate metrics differently, leading to conflicting results.
   • Best Practice: Define metrics as code or in a semantic layer. Ensure the metric definition is versioned and immutable during the experiment.

4. Network Effects and Interference

   • Mistake: Running experiments on features that affect user interaction (e.g., social feeds, marketplaces) without accounting for interference between treatment and control groups.
   • Best Practice: Use cluster-based randomization or graph-based allocation for networked products. Monitor global metrics for spillover effects.

5. Lack of Guardrail Metrics

   • Mistake: Optimizing for a primary metric while degrading user experience or system performance.
   • Best Practice: Always define guardrail metrics (e.g., latency, error rate, retention). Automatically halt experiments if guardrails are breached.

6. Hardcoding Variants in Business Logic

   • Mistake: Using if (variant === 'B') { ... } scattered across components. This makes cleanup difficult and increases bundle size.
   • Best Practice: Use configuration-driven UI rendering. The application should render components based on variant metadata, not hardcoded logic.

7. Inadequate Cleanup Strategy

   • Mistake: Leaving experiment code and configuration active after the experiment concludes.
   • Best Practice: Automate cleanup. When an experiment is marked "archived," the platform should trigger a PR to remove the configuration and flag the code for removal in the next sprint.

Production Bundle

Action Checklist

• Define Hypothesis: Document the expected outcome, primary metric, and minimum detectable effect (MDE).
• Calculate Sample Size: Use power analysis to determine the required sample size based on baseline conversion and MDE.
• Implement SRM Check: Add automated SRM validation to the allocation logic and dashboard.
• Define Guardrails: Select 2-3 metrics that must not degrade (e.g., p95 latency, crash rate).
• Configure Cleanup: Set up a lifecycle policy to archive experiments and trigger cleanup tasks upon conclusion.
• Review Statistical Method: Choose between Frequentist or Bayesian approaches based on team expertise and decision speed requirements.
• Test Allocation: Run a dry run with internal traffic to verify deterministic hashing and event correlation.

Decision Matrix

Configuration Template

Use this JSON schema to define experiments in your configuration store. This template enforces validation and lifecycle management.

{
  "experimentId": "exp-checkout-flow-v2",
  "name": "Checkout Flow Optimization",
  "status": "running",
  "targetId": "user",
  "trafficSplit": {
    "control": 0.5,
    "variant_a": 0.25,
    "variant_b": 0.25
  },
  "metrics": {
    "primary": "checkout_completion_rate",
    "guardrails": ["p95_latency_ms", "api_error_rate"]
  },
  "variants": {
    "control": { "metadata": { "ui_version": "v1" } },
    "variant_a": { "metadata": { "ui_version": "v2", "button_color": "blue" } },
    "variant_b": { "metadata": { "ui_version": "v2", "button_color": "green" } }
  },
  "lifecycle": {
    "startDate": "2024-01-15T00:00:00Z",
    "endDate": "2024-02-15T00:00:00Z",
    "autoArchive": true,
    "cleanupTaskId": "JIRA-12345"
  }
}

Quick Start Guide

1. Initialize the Engine: Install the experimentation SDK and initialize the allocation engine with your configuration source.

   npm install @codcompass/experiment-sdk
   

   import { ExperimentEngine } from '@codcompass/experiment-sdk';
   const engine = new ExperimentEngine({ configSource: 'remote' });
   

2. Define Your First Experiment: Create a configuration file matching the template above and deploy it to your configuration store.

3. Wrap Your Component: Use the provided hook to allocate users and render variants.

   const { variant, trackExposure } = useExperiment('exp-checkout-flow-v2', userId, engine);
   useEffect(() => { trackExposure(); }, []);
   

4. Track Conversions: Emit conversion events when the user completes the target action.

   engine.trackConversion('checkout_completion_rate', { value: 1 });
   

5. Monitor and Analyze: View real-time allocation, SRM status, and metric trends in your experimentation dashboard. Halt the experiment if guardrails are breached.