import { fetch, Agent, setGlobalDispatcher } from 'undici';
import { Logger } from './logger';

const agent = new Agent({ connections: 100, pipelining: 1 });
setGlobalDispatcher(agent);

export interface FetchConfig {
  url: string;
  maxRetries?: number;
  timeoutMs?: number;
  headers?: Record<string, string>;
}

export async function resilientFetch(config: FetchConfig): Promise<string> {
  const { url, maxRetries = 3, timeoutMs = 8000, headers = {} } = config;
  let attempt = 0;

  while (attempt < maxRetries) {
    try {
      const response = await fetch(url, {
        method: 'GET',
        headers: {
          'User-Agent': 'DataPipeline/1.0 (Commercial)',
          'Accept': 'text/html,application/xhtml+xml',
          ...headers,
        },
        signal: AbortSignal.timeout(timeoutMs),
      });

      if (!response.ok) {
        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
      }

      return await response.text();
    } catch (error) {
      attempt++;
      const delay = Math.min(1000 * 2 ** attempt, 10000);
      Logger.warn(`Fetch failed for ${url}. Attempt ${attempt}/${maxRetries}. Retrying in ${delay}ms.`);
      await new Promise(resolve => setTimeout(resolve, delay));
    }
  }

  throw new Error(`Exhausted ${maxRetries} retries for ${url}`);
}

import * as cheerio from 'cheerio';
import { z } from 'zod';

const ProductSchema = z.object({
  id: z.string().uuid(),
  title: z.string().min(1).max(255),
  price: z.number().positive(),
  currency: z.enum(['USD', 'EUR', 'GBP']),
  lastUpdated: z.string().datetime(),
});

export typ

e Product = z.infer<typeof ProductSchema>;

export function parseProductPage(html: string): Product[] { const $ = cheerio.load(html); const rawItems: Partial<Product>[] = [];

$('.catalog-item').each((_, el) => { const id = $(el).attr('data-sku') ?? ''; const title = $(el).find('.item-title').text().trim(); const priceRaw = $(el).find('.price-value').text().replace(/[^0-9.]/g, ''); const price = parseFloat(priceRaw);

rawItems.push({
  id,
  title,
  price: isNaN(price) ? 0 : price,
  currency: 'USD',
  lastUpdated: new Date().toISOString(),
});

});

const validated: Product[] = []; for (const item of rawItems) { const result = ProductSchema.safeParse(item); if (result.success) { validated.push(result.data); } else { Logger.error(Schema validation failed: ${JSON.stringify(result.error.format())}); } }

return validated; }

**Rationale**: Cheerio avoids browser automation overhead. Zod enforces strict contracts at the ingestion boundary, preventing garbage data from corrupting downstream analytics or API responses.

#### 3. Monetization-Ready API Endpoint

```typescript
import Fastify from 'fastify';
import { fastifyRateLimit } from '@fastify/rate-limit';
import { fastifyRedis } from '@fastify/redis';
import { fetchProductData } from './scraper';
import { db } from './database';

const app = Fastify({ logger: true });

await app.register(fastifyRedis, { url: 'redis://localhost:6379' });
await app.register(fastifyRateLimit, {
  max: 100,
  timeWindow: '1 minute',
  keyGenerator: (req) => req.ip,
});

app.get('/v1/products', async (req, reply) => {
  const cacheKey = 'feed:products:latest';
  const cached = await app.redis.get(cacheKey);

  if (cached) {
    return reply.code(200).header('X-Cache', 'HIT').send(JSON.parse(cached));
  }

  try {
    const html = await fetchProductData('https://target-retailer.com/catalog');
    const products = parseProductPage(html);

    await db.products.upsertMany(products);
    await app.redis.set(cacheKey, JSON.stringify(products), 'EX', 300);

    return reply.code(200).header('X-Cache', 'MISS').send(products);
  } catch (error) {
    req.log.error(error);
    return reply.code(503).send({ error: 'Data pipeline temporarily unavailable' });
  }
});

const PORT = process.env.PORT ?? 3000;
app.listen({ port: PORT, host: '0.0.0.0' }, (err) => {
  if (err) {
    app.log.error(err);
    process.exit(1);
  }
  app.log.info(`DaaS API listening on port ${PORT}`);
});

Rationale: Rate limiting protects infrastructure and enables tiered commercial plans. Redis caching reduces scraper load and improves latency. Database upserts ensure historical tracking without duplication. The X-Cache header provides transparency for enterprise clients.

Pitfall Guide

1. Selector Fragility & DOM Drift

Explanation: Relying on exact CSS classes or XPath expressions causes immediate breakage when target sites update their frontend frameworks or A/B test layouts. Fix: Implement fallback extraction strategies. Use data attributes (data-sku, data-price) when available. Deploy automated drift detection that triggers alerts when extraction yield drops below a threshold.

2. Ignoring Rate Limits & Throttling

Explanation: Aggressive request patterns trigger IP bans, CAPTCHAs, or legal action. Many teams assume "it worked locally" translates to production. Fix: Implement adaptive throttling based on Retry-After headers and HTTP 429 responses. Rotate residential or datacenter proxies. Respect robots.txt directives and commercial terms of service.

3. Missing Data Validation & Schema Drift

Explanation: Unvalidated data enters pipelines, causing downstream analytics failures, pricing engine miscalculations, or API contract violations. Fix: Enforce schema validation at the ingestion boundary using tools like Zod or JSON Schema. Implement versioned data contracts and migration strategies for structural changes.

4. Legal & Compliance Blind Spots

Explanation: Scraping personal data, copyrighted content, or restricted APIs violates GDPR, CCPA, or CFAA provisions. Commercial resale amplifies liability. Fix: Conduct a data governance audit before deployment. Exclude PII, implement data retention policies, and consult legal counsel regarding target site terms. Use public APIs when available.

5. Single-Threaded Blocking Execution

Explanation: Sequential scraping blocks worker threads, causing memory leaks and degraded throughput under concurrent API requests. Fix: Use async/await patterns with bounded concurrency. Implement worker pools or message queues (e.g., BullMQ, RabbitMQ) for distributed extraction.

6. Inadequate Error Handling & Silent Failures

Explanation: Uncaught exceptions or swallowed errors result in data gaps that go unnoticed until clients report missing records. Fix: Implement structured logging with correlation IDs. Deploy health checks, dead-letter queues for failed extractions, and automated alerting via PagerDuty or Slack webhooks.

7. Overlooking Caching & Idempotency

Explanation: Repeated requests to the same endpoint waste bandwidth and trigger anti-bot defenses. Non-idempotent writes cause duplicate records. Fix: Implement HTTP caching headers, Redis TTLs, and database upsert operations. Design extraction jobs to be idempotent by using deterministic keys and conflict resolution strategies.

Production Bundle

Action Checklist

• Compliance Review: Verify target site terms, exclude PII, document data lineage
• Schema Contract: Define Zod/JSON schema with versioning and migration path
• Resilience Layer: Implement retry/backoff, proxy rotation, and circuit breakers
• Observability: Deploy structured logging, metrics (Prometheus), and alerting thresholds
• Caching Strategy: Configure Redis TTLs, cache invalidation rules, and fallback responses
• API Tiering: Design rate limits, subscription tiers, and SLA tracking for commercial access
• Testing Pipeline: Add contract tests, selector drift monitors, and synthetic load testing
• Data Retention: Implement automated archival, GDPR deletion workflows, and backup rotation

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
High-frequency pricing updates (<5 min)	Headless browser + proxy pool + Redis cache	Handles JS rendering, avoids bans, reduces scraper load	High (Proxy + compute)
Static catalog extraction (Daily)	HTTP client + Cheerio + PostgreSQL	Lightweight, fast, low infrastructure overhead	Low (Single VM + DB)
Enterprise DaaS API (10k+ req/min)	Fastify + rate limiting + CDN + auto-scaling	Guarantees SLA, handles traffic spikes, monetization-ready	Medium-High (Cloud infra)
Compliance-heavy industry (Finance/Health)	Public API first, manual audit fallback	Avoids legal risk, ensures data accuracy, meets audit requirements	Medium (Legal + manual ops)

Configuration Template

# .env.production
DATABASE_URL=postgresql://user:pass@db-host:5432/daas_db
REDIS_URL=redis://cache-host:6379
SCRAPER_PROXY_POOL=https://proxy-provider.com/rotate
LOG_LEVEL=info
API_RATE_LIMIT_MAX=100
API_RATE_LIMIT_WINDOW=60000
CACHE_TTL_SECONDS=300
SCHEMA_VERSION=v1

// config.ts
import { z } from 'zod';

const envSchema = z.object({
  DATABASE_URL: z.string().url(),
  REDIS_URL: z.string().url(),
  SCRAPER_PROXY_POOL: z.string().url(),
  LOG_LEVEL: z.enum(['debug', 'info', 'warn', 'error']).default('info'),
  API_RATE_LIMIT_MAX: z.coerce.number().default(100),
  API_RATE_LIMIT_WINDOW: z.coerce.number().default(60000),
  CACHE_TTL_SECONDS: z.coerce.number().default(300),
  SCHEMA_VERSION: z.string().default('v1'),
});

export const config = envSchema.parse(process.env);

Quick Start Guide

1. Initialize Project: Run npm init -y && npm install undici cheerio zod fastify @fastify/rate-limit @fastify/redis pg
2. Configure Environment: Copy .env.example to .env.production and populate database, Redis, and proxy credentials.
3. Deploy Infrastructure: Start PostgreSQL and Redis via Docker Compose. Verify connectivity with docker compose up -d.
4. Run Pipeline: Execute node dist/server.js. Monitor logs for successful fetches, schema validation, and cache hits.
5. Validate API: Test GET /v1/products with curl -H "Authorization: Bearer <token>" http://localhost:3000/v1/products. Verify response structure and X-Cache header behavior.

Building a commercial data extraction pipeline requires treating web scraping as a data engineering discipline rather than a scripting exercise. By enforcing schema contracts, implementing resilience patterns, and designing API layers with monetization in mind, developers can transform fragile scrapers into reliable, revenue-generating data services. The architectural choices outlined here prioritize uptime, compliance, and scalability—ensuring that your data feeds remain accurate, accessible, and commercially viable long after initial deployment.