docker-compose.yml

eScript for compile-time safety.

import { z } from 'zod';

export const InvoicePaidEvent = z.object({
  id: z.string().uuid(),
  customer_id: z.string().uuid(),
  amount_cents: z.number().int().positive(),
  currency: z.string().length(3),
  paid_at: z.string().datetime(),
  metadata: z.record(z.unknown()).optional()
});

export type InvoicePaidEvent = z.infer<typeof InvoicePaidEvent>;

2. Build an Idempotent Handler Idempotency prevents duplicate processing on network retries or webhook redelivery. Store execution state in PostgreSQL.

import { Hono } from 'hono';
import { db } from './db';
import { InvoicePaidEvent } from './events';

const app = new Hono();

app.post('/webhooks/stripe', async (c) => {
  const payload = await c.req.json();
  const parsed = InvoicePaidEvent.safeParse(payload);
  if (!parsed.success) return c.json({ error: 'Invalid payload' }, 400);

  const event = parsed.data;
  const { idempotency_key } = event;

  // Check execution state
  const existing = await db.query.executions.findFirst({
    where: (eq, { and }) => and(eq(executions.idempotencyKey, idempotency_key))
  });

  if (existing) return c.json({ status: 'already_processed' }, 200);

  // Mark as processing to prevent race conditions
  await db.insert(executions).values({
    idempotency_key,
    event_type: 'invoice.paid',
    status: 'processing',
    payload: event,
    created_at: new Date()
  });

  try {
    await processPayment(event);
    await db.update(executions).set({ status: 'completed' })
      .where(eq(executions.idempotency_key, idempotency_key));
    return c.json({ status: 'ok' }, 200);
  } catch (err) {
    await db.update(executions).set({ status: 'failed', error: String(err) })
      .where(eq(executions.idempotency_key, idempotency_key));
    return c.json({ error: 'processing_failed' }, 500);
  }
});

3. Implement Scheduled Tasks Use a lightweight cron runner with deterministic execution windows and failure alerts.

import { Cron } from 'croner';
import { logger } from './observability';

const dailyReconciliation = new Cron('0 2 * * *', async () => {
  logger.info('starting_daily_reconciliation');
  try {
    await reconcileAccounts();
    logger.info('daily_reconciliation_completed');
  } catch (err) {
    logger.error('reconciliation_failed', { error: err });
    await sendAlert('reconciliation_failure', err);
  }
});

4. Add Observability Structured logging replaces console.debug. Use OpenTelemetry-compatible JSON logs with correlation IDs.

import pino from 'pino';

export const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  formatters: { level: (label) => ({ level: label }) },
  timestamp: pino.stdTimeFunctions.isoTime,
  base: { service: 'solo-os' }
});

5. Deploy with Infrastructure-as-Code Containerize the runtime or deploy as serverless functions. Use Terraform or Pulumi for reproducible environments.

resource "aws_lambda_function" "automation_runtime" {
  function_name = "solo-os-webhooks"
  runtime       = "nodejs20.x"
  handler       = "dist/index.handler"
  filename      = "dist/lambda.zip"
  memory_size   = 256
  timeout       = 30
  environment {
    variables = {
      DATABASE_URL = var.db_url
      LOG_LEVEL    = "info"
    }
  }
}

Architecture Decisions and Rationale

• Event-driven over polling: Webhooks eliminate unnecessary API calls and reduce latency. Scheduled tasks handle reconciliation and cleanup.
• TypeScript + Zod: Runtime validation prevents malformed payloads from corrupting state. Compile-time types catch integration mismatches early.
• PostgreSQL + JSONB: Relational integrity for core entities, JSONB for flexible event payloads. Avoids NoSQL trade-offs while maintaining schema agility.
• Idempotency-first design: Network failures, webhook retries, and cloud function cold starts are guaranteed. Execution state is the source of truth.
• Code-first over visual builders: Version control, code review, testing, and rollback capabilities are non-negotiable for production reliability.

Pitfall Guide

1. Ignoring idempotency guarantees Webhook providers retry on 5xx or timeout responses. Without idempotency keys and execution state tracking, a single payment triggers duplicate fulfillment, double charges, or corrupted accounting. Always store execution state before processing, and check it on every invocation.

2. Hardcoding secrets or embedding credentials in logic Automation handlers often interact with payment gateways, email providers, and CRMs. Embedding API keys in source code or environment files without rotation policies creates blast radius exposure. Use a secrets manager (AWS Secrets Manager, Doppler, or 1Password CLI) with short-lived tokens and automatic rotation.

3. Over-engineering the event bus Building custom pub/sub systems, message queues, or distributed state machines for a solo operation introduces unnecessary failure surfaces. Stick to direct HTTP webhooks + PostgreSQL state. Introduce RabbitMQ/Kafka only when throughput exceeds 10k events/minute or multi-tenant isolation becomes mandatory.

4. Skipping rate limit and backoff handling External APIs enforce strict rate limits. Automated handlers that fire synchronously without exponential backoff or queueing will trigger 429 responses and temporary bans. Implement token bucket or leaky bucket rate limiters, and respect Retry-After headers.

5. Coupling automation to frontend or UI state Business logic embedded in React components or client-side scripts breaks when users close tabs or lose connectivity. Isolate automation in server-side handlers. The UI should only trigger actions; it should never contain business rules or state mutations.

6. No retry or dead-letter strategy Transient failures (network timeouts, temporary API outages) become permanent data loss without retry policies. Implement exponential backoff with a maximum retry count. Route exhausted events to a dead-letter queue for manual inspection, not silent drops.

7. Ignoring local development parity "Works on my machine" automation fails in production due to environment drift. Run PostgreSQL, Redis, and webhook simulators locally via Docker Compose. Use .env files with strict validation. Test webhook signatures and TLS termination in staging before production deployment.

Best practices from production experience:

• Treat automation handlers as pure functions where possible; isolate side effects.
• Use contract testing between event producers and consumers.
• Implement correlation IDs across all logs for traceability.
• Version your event schemas; deprecate gracefully with migration scripts.
• Run weekly reconciliation jobs to detect state drift.

Production Bundle

Action Checklist

• Define event schemas with Zod and enforce runtime validation on all webhook endpoints
• Implement idempotency tracking with execution state table and unique constraint on idempotency_key
• Configure structured logging with correlation IDs and log levels aligned to environment
• Rotate secrets via CLI or secrets manager; never commit credentials to version control
• Add exponential backoff retry logic with dead-letter queue for exhausted events
• Schedule weekly reconciliation jobs to validate state consistency across systems
• Document runbooks for common failure modes and manual intervention procedures

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
Validating idea or <100 customers	No-Code (Make/Zapier)	Rapid iteration, zero infrastructure overhead	$30–$50/mo
Scaling to 1k+ customers, recurring revenue	Code-First TypeScript	Deterministic execution, full auditability, cost predictability	$15–$35/mo
Multi-service integration with complex state	Low-Code + Custom Handlers	Balance speed with control; use no-code for routing, code for business logic	$80–$150/mo
High-volume events (>5k/day)	Code-First + Queue (Redis/SQS)	Prevents handler overload, enables backpressure and parallel processing	$25–$60/mo
Regulatory compliance (SOC2, GDPR)	Code-First + IaC	Version control, audit trails, explicit data retention policies	$40–$90/mo

Configuration Template

// config/env.ts
import { z } from 'zod';

const envSchema = z.object({
  DATABASE_URL: z.string().url(),
  STRIPE_WEBHOOK_SECRET: z.string().min(32),
  LOG_LEVEL: z.enum(['debug', 'info', 'warn', 'error']).default('info'),
  MAX_RETRY_ATTEMPTS: z.coerce.number().min(1).max(5).default(3),
  RETRY_BASE_DELAY_MS: z.coerce.number().min(1000).default(2000)
});

export const env = envSchema.parse(process.env);

// middleware/idempotency.ts
import { Context, Next } from 'hono';
import { db } from '../db';
import { executions } from '../db/schema';
import { eq } from 'drizzle-orm';

export const idempotency = async (c: Context, next: Next) => {
  const key = c.req.header('x-idempotency-key');
  if (!key) return c.json({ error: 'missing_idempotency_key' }, 400);

  const existing = await db.query.executions.findFirst({
    where: eq(executions.idempotency_key, key)
  });

  if (existing) {
    return c.json({ status: 'duplicate', execution_id: existing.id }, 200);
  }

  await db.insert(executions).values({
    idempotency_key: key,
    status: 'processing',
    created_at: new Date()
  });

  c.set('idempotency_key', key);
  await next();
};

# docker-compose.yml
version: '3.9'
services:
  db:
    image: postgres:16-alpine
    environment:
      POSTGRES_DB: solo_os
      POSTGRES_USER: dev
      POSTGRES_PASSWORD: dev
    ports:
      - "5432:5432"
    volumes:
      - pgdata:/var/lib/postgresql/data
  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"
volumes:
  pgdata:

Quick Start Guide

1. Initialize project with npm create hono@latest solo-os and install dependencies: npm i drizzle-orm zod pino cronier
2. Configure environment variables using the template above, then run docker compose up -d to start local PostgreSQL and Redis
3. Generate database schema with Drizzle Kit (npx drizzle-kit generate) and apply migrations (npx drizzle-kit push)
4. Start the runtime locally with npm run dev, test webhook delivery using ngrok http 3000, and verify idempotency by replaying the same payload twice
5. Deploy to Cloudflare Workers, Vercel, or AWS Lambda using the provided IaC template; configure environment secrets via your provider's dashboard