import fs from 'fs/promises';
import path from 'path';

const BASE_STORAGE_ROOT = path.resolve(__dirname, '../data/assets');
const SUBDIRECTORIES = ['public', 'private', 'temp'];

async function initializeStorageLayout(): Promise<void> {
  await fs.mkdir(BASE_STORAGE_ROOT, { recursive: true });
  
  for (const dir of SUBDIRECTORIES) {
    const target = path.join(BASE_STORAGE_ROOT, dir);
    await fs.mkdir(target, { recursive: true });
  }
}

export { BASE_STORAGE_ROOT, SUBDIRECTORIES, initializeStorageLayout };

import multer from 'multer';
import { Request } from 'express';
import { v4 as uuidv4 } from 'uuid';

const ALLOWED_SIGNATURES: Record<string, Buffer> = {
  'image/jpeg': Buffer.from([0xFF, 0xD8, 0xFF]),
  'image/png': Buffer.from([0x89, 0x50, 0x4E, 0x47]),
  'application/pdf': Buffer.from([0x25, 0x50, 0x44, 0x46]),
};

const MAX_PAYLOAD_SIZE = 10 * 1024 * 1024; // 10MB

function verifyStreamSignature(req: Request, file: Express.Multer.File, callback: multer.FileFilterCallback) {
  const expectedSig = ALLOWED_SIGNATURES[file.mimetype];
  if (!expectedSig) {
    return callback(new Error('Unsupported media type'), false);
  }

  const stream = file.stream;
  let headerBytes = Buffer.alloc(0);

  stream.on('data', (chunk: Buffer) => {
    headerBytes = Buffer.concat([headerBytes, chunk]);
    if (headerBytes.length >= expectedSig.length) {
      stream.removeAllListeners('data');
      const isValid = headerBytes.slice(0, expectedSig.length).equals(expectedSig);
      callback(isValid ? null : new Error('Signature mismatch'), isValid);
    }
  });

  stream.on('error', () => callback(new Error('Stream read failure'), false));
}

export const secureUploadMiddleware = multer({
  storage: multer.memoryStorage(),
  limits: { fileSize: MAX_PAYLOAD_SIZE },
  fileFilter: verifyStreamSignature,
});

model. An interface-based adapter pattern enables seamless migration to cloud providers.

export interface AssetRepository {
  persist(buffer: Buffer, key: string, tier: 'public' | 'private'): Promise<string>;
  retrieve(key: string, tier: 'public' | 'private'): Promise<Buffer>;
  remove(key: string, tier: 'public' | 'private'): Promise<void>;
  generateAccessUrl(key: string, tier: 'public' | 'private'): string;
}

import { BASE_STORAGE_ROOT } from './layout';

export class LocalDiskRepository implements AssetRepository {
  async persist(buffer: Buffer, key: string, tier: 'public' | 'private'): Promise<string> {
    const targetPath = path.join(BASE_STORAGE_ROOT, tier, key);
    await fs.writeFile(targetPath, buffer);
    return targetPath;
  }

  async retrieve(key: string, tier: 'public' | 'private'): Promise<Buffer> {
    const targetPath = path.join(BASE_STORAGE_ROOT, tier, key);
    return fs.readFile(targetPath);
  }

  async remove(key: string, tier: 'public' | 'private'): Promise<void> {
    const targetPath = path.join(BASE_STORAGE_ROOT, tier, key);
    await fs.unlink(targetPath);
  }

  generateAccessUrl(key: string, tier: 'public' | 'private'): string {
    return tier === 'public' ? `/assets/${key}` : `/api/secure/${key}`;
  }
}

Rationale: The repository contract isolates I/O operations. LocalDiskRepository handles the current deployment, while a future CloudObjectRepository can implement the same interface using AWS SDK or Google Cloud Storage client libraries. URL generation is centralized, ensuring consistent routing across the application.

Step 4: Static Serving & Route Configuration

Express static middleware must be mounted with explicit prefixes and strict path boundaries. Public assets and authenticated resources require separate mounting strategies.

import express from 'express';
import { BASE_STORAGE_ROOT } from './layout';

export function mountStaticLayers(app: express.Application): void {
  // Public tier: cached, no authentication
  app.use('/assets', express.static(path.join(BASE_STORAGE_ROOT, 'public'), {
    maxAge: '1d',
    etag: true,
    dotfiles: 'ignore',
  }));

  // Private tier: requires middleware validation
  app.use('/api/secure', (req, res, next) => {
    if (!req.user?.hasAccess) {
      return res.status(403).json({ error: 'Insufficient privileges' });
    }
    next();
  }, express.static(path.join(BASE_STORAGE_ROOT, 'private'), {
    dotfiles: 'ignore',
    index: false,
  }));
}

Rationale: maxAge and etag enable browser and CDN caching for public assets. dotfiles: 'ignore' prevents exposure of .env or .git files. Private resources are wrapped in an authentication guard before static serving, ensuring authorization checks execute on every request. index: false disables directory listing, a common misconfiguration.

Pitfall Guide

1. Relative Path Resolution in Production

Explanation: Using './uploads' or 'uploads/' relies on process.cwd(), which changes based on how the Node process is launched. PM2, Docker, and systemd often set different working directories, causing silent ENOENT errors. Fix: Always resolve paths using path.resolve(__dirname, '../relative/path') at startup. Cache the absolute path in a module-level constant.

2. Trusting Client-Supplied MIME Types

Explanation: Browsers and curl clients can arbitrarily set Content-Type headers. An attacker can upload a PHP or shell script disguised as image/png. Fix: Validate magic bytes (file signatures) from the raw stream. Never rely on file.mimetype or file.originalname for security decisions.

3. Synchronous File System Operations in Request Handlers

Explanation: Using fs.readFileSync() or fs.existsSync() inside route handlers blocks the single-threaded event loop. Under concurrent load, this causes request queuing and timeout cascades. Fix: Use fs/promises or callback-based async methods. Pre-initialize directories during application bootstrap, not per-request.

4. Missing Cleanup for Failed Uploads

Explanation: When validation fails mid-stream, partially written files or temporary buffers remain on disk. Over time, this consumes storage and creates orphaned data. Fix: Implement a background garbage collector that scans the temp/ tier for files older than a threshold (e.g., 15 minutes). Use fs.unlink() with error suppression for missing files.

5. Exposing Internal Directory Structures via Static Routes

Explanation: Mounting express.static() without index: false or proper prefix mapping allows directory traversal or listing. Attackers can enumerate filenames and guess access patterns. Fix: Disable directory indexing explicitly. Use strict URL prefixes that don't mirror internal folder names. Apply dotfiles: 'ignore' to hide system files.

6. Ignoring Concurrent Upload Race Conditions

Explanation: Using predictable filenames (e.g., user-${id}.jpg) causes collisions when users upload simultaneously. The last write wins, silently overwriting previous data. Fix: Generate cryptographically random identifiers (UUIDv4 or nanoid) combined with timestamps. Store the mapping in your database; never rely on filesystem naming for uniqueness.

7. Bypassing Rate Limiting on Upload Endpoints

Explanation: File ingestion consumes CPU, memory, and disk I/O. Unrestricted endpoints are prime targets for resource exhaustion attacks. Fix: Apply aggressive rate limiting specifically to upload routes. Use token bucket algorithms with lower thresholds than standard API endpoints. Consider queueing large uploads via background workers.

Production Bundle

Action Checklist

• Resolve all storage paths using path.resolve(__dirname, ...) at application startup
• Implement stream-level magic byte validation before disk persistence
• Enforce strict file size limits at the parser level, not in application logic
• Abstract storage operations behind a repository interface to enable cloud migration
• Mount public and private static routes with separate middleware chains
• Disable directory indexing and dotfile exposure on all static mounts
• Schedule automated cleanup jobs for temporary and failed upload directories
• Apply dedicated rate limiting to upload endpoints with lower thresholds than read routes

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
Prototype / Internal Tool	Local Disk + Absolute Paths	Zero external dependencies, fastest iteration	$0 infrastructure, high manual ops
Multi-Node Load Balanced	Cloud Object Storage (S3/GCS)	Shared state across instances, built-in durability	$23/TB/month, eliminates backup engineering
Compliance / Audit Heavy	Cloud Storage + Versioning + IAM	Immutable history, granular access policies, automated retention	+15% storage cost, reduces legal/compliance risk
High-Throughput Media	CDN + Signed URLs + Edge Caching	Offloads origin bandwidth, reduces latency globally	CDN egress fees, but cuts origin compute costs by 60%+

Configuration Template

// config/storage.ts
import path from 'path';
import multer from 'multer';
import { v4 as uuidv4 } from 'uuid';

export const STORAGE_CONFIG = {
  root: path.resolve(__dirname, '../data/assets'),
  tiers: ['public', 'private', 'temp'] as const,
  limits: { fileSize: 10 * 1024 * 1024 },
  allowedSignatures: {
    'image/jpeg': Buffer.from([0xFF, 0xD8, 0xFF]),
    'image/png': Buffer.from([0x89, 0x50, 0x4E, 0x47]),
    'application/pdf': Buffer.from([0x25, 0x50, 0x44, 0x46]),
  },
};

export const uploadPipeline = multer({
  storage: multer.memoryStorage(),
  limits: STORAGE_CONFIG.limits,
  fileFilter: (req, file, cb) => {
    const sig = STORAGE_CONFIG.allowedSignatures[file.mimetype as keyof typeof STORAGE_CONFIG.allowedSignatures];
    if (!sig) return cb(new Error('Unsupported type'), false);
    
    let header = Buffer.alloc(0);
    file.stream.on('data', (chunk) => {
      header = Buffer.concat([header, chunk]);
      if (header.length >= sig.length) {
        file.stream.removeAllListeners('data');
        cb(header.slice(0, sig.length).equals(sig) ? null : new Error('Invalid signature'), header.slice(0, sig.length).equals(sig));
      }
    });
    file.stream.on('error', () => cb(new Error('Stream error'), false));
  },
});

export function generateAssetKey(originalName: string): string {
  const ext = path.extname(originalName).toLowerCase();
  return `${uuidv4()}${ext}`;
}

Quick Start Guide

1. Initialize Layout: Run initializeStorageLayout() during your Express app bootstrap sequence. Verify directories exist before mounting routes.
2. Attach Pipeline: Import uploadPipeline and apply it to your POST route: router.post('/upload', uploadPipeline.single('file'), handler).
3. Persist & Map: In the handler, read req.file.buffer, generate a safe key using generateAssetKey(), and call your repository's persist() method. Store the returned key in your database.
4. Mount Static Layers: Call mountStaticLayers(app) after defining your API routes. Verify public assets load without authentication and private assets return 403 when unauthenticated.
5. Validate End-to-End: Use curl or Postman to upload a valid file, capture the returned key, and request it via the static prefix. Confirm caching headers appear for public assets and authorization gates trigger for private routes.