import { QdrantClient } from "@qdrant/js-client-rest";
import { MilvusClient } from "@zilliz/milvus2-sdk-node";
import { Pool } from "pg";

// Abstract Benchmark Interface
interface VectorDBClient {
  name: string;
  search(vectors: number[], limit: number, filter?: Record<string, any>): Promise<SearchResult[]>;
  insert(vectors: number[][], metadata: Record<string, any>[]): Promise<void>;
}

interface SearchResult {
  id: string;
  score: number;
  metadata: Record<string, any>;
}

// Benchmark Runner
async function runBenchmark(
  client: VectorDBClient,
  groundTruth: { vector: number[]; id: string; metadata: Record<string, any> }[],
  queryVector: number[],
  k: number = 10
) {
  const start = performance.now();
  const results = await client.search(queryVector, k, { tenant_id: "tenant_42" });
  const latency = performance.now() - start;

  // Calculate Recall
  const groundTruthIds = new Set(groundTrut

h.map((g) => g.id)); const retrievedIds = new Set(results.map((r) => r.id)); const intersection = [...groundTruthIds].filter((id) => retrievedIds.has(id)); const recall = intersection.length / Math.min(groundTruthIds.size, k);

return { db: client.name, latency_ms: latency, recall_at_k: recall, result_count: results.length, }; }

// Usage Example async function main() { // Initialize clients (configuration omitted for brevity) const qdrant = new QdrantClient({ url: "http://localhost:6333" }); const milvus = new MilvusClient({ address: "localhost:19530" });

// Load synthetic dataset matching production distribution // Run benchmark with and without filters // Compare results }

### Step 3: Index Configuration Strategy

Optimize index parameters based on the benchmark results.

**HNSW Configuration (Qdrant, Weaviate, pgvector HNSW):**
*   `m`: Number of bidirectional links. Higher `m` increases recall and memory usage but decreases latency. Default is often 16; increase to 32-64 for high recall.
*   `ef_construction`: Quality of index build. Higher values yield better recall but longer build times. Set to 100-200.
*   `ef_search`: Query-time trade-off. Higher values increase recall at cost of latency. Tune dynamically based on query complexity.

**IVF Configuration (Milvus, pgvector IVFFlat):**
*   `nlist`: Number of clusters. Rule of thumb: `nlist ≈ 4 * sqrt(N)`.
*   `nprobe`: Number of clusters to search. Higher `nprobe` improves recall but increases latency. Start with `nprobe = 8` and scale based on recall requirements.

**Quantization Strategy:**
*   **Scalar Quantization (SQ):** Reduces memory by 4x (FP32 to INT8). Recall loss is minimal (<1%) for most embeddings. Enable by default for scale.
*   **Product Quantization (PQ):** Aggressive compression. Use only if memory is constrained and recall loss is acceptable.
*   **Binary Quantization:** Extreme compression. Only suitable for very high-dimensional vectors where precision is less critical.

### Step 4: Hybrid Search Architecture

Pure vector search fails on exact matches and keyword-heavy queries. Implement hybrid search combining BM25 (keyword) and dense vector retrieval.

```typescript
// Hybrid Search Logic
const vectorResults = await db.searchDense(queryEmbedding, { limit: 50 });
const keywordResults = await db.searchBM25(queryText, { limit: 50 });

// Recombine using RRF (Reciprocal Rank Fusion)
const combined = reciprocalRankFusion(vectorResults, keywordResults, k: 60);
const finalResults = combined.slice(0, 10);

• Rationale: RRF is parameter-free and robust. It balances semantic relevance with keyword precision, significantly improving RAG accuracy.

Pitfall Guide

1. Ignoring Metadata Filter Tax

• Mistake: Selecting a database based on unfiltered latency benchmarks.
• Impact: Production queries with filters experience 3-5x latency increase, violating SLAs.
• Best Practice: Always benchmark with production-like filter distributions. Prioritize databases with optimized inverted indexes or separate filter engines.

2. Misconfiguring HNSW Parameters

• Mistake: Using default m and ef_search values.
• Impact: Suboptimal recall or excessive memory usage.
• Best Practice: Tune ef_search per query. Use lower values for simple queries and higher values for complex semantic searches. Monitor memory growth as m increases.

3. Assuming Quantization is Free

• Mistake: Enabling quantization without measuring recall impact.
• Impact: Degraded retrieval quality leads to hallucinations in LLM responses.
• Best Practice: Validate recall delta after enabling quantization. Use Scalar Quantization as a safe default; avoid Product Quantization for critical retrieval tasks.

4. Neglecting Multi-Tenancy Isolation

• Mistake: Storing all tenant data in a single collection without proper sharding.
• Impact: Security leaks, noisy neighbor performance issues, and inefficient filtering.
• Best Practice: Use payload indexes for tenant IDs. For high-scale multi-tenancy, consider separate collections or sharding strategies supported by the database.

5. Overlooking Update/Delete Latency

• Mistake: Assuming vector updates are instantaneous.
• Impact: HNSW indices can be slow to update. Delete operations may leave "tombstones" that degrade performance over time.
• Best Practice: Profile update patterns. Use databases with efficient update mechanisms or implement periodic index rebuilding for high-churn workloads.

6. Embedding Normalization Errors

• Mistake: Failing to normalize embeddings before storage or query.
• Impact: Cosine distance calculations fail; retrieval returns irrelevant results.
• Best Practice: Normalize vectors to unit length before insertion. Ensure the distance metric matches the embedding model's training (e.g., Cosine vs. Dot Product).

7. Treating Vector DB as Primary Storage

• Mistake: Storing full document payloads in the vector database.
• Impact: Increased storage costs, slower retrieval, and data consistency issues.
• Best Practice: Use vector databases solely for retrieval. Store metadata and payloads in a primary database. Fetch full content by ID after retrieval.

Production Bundle

Action Checklist

• Define SLA: Establish target recall@10 and p99 latency for both filtered and unfiltered queries.
• Profile Workload: Analyze vector count, dimensionality, filter cardinality, and update frequency.
• Synthetic Benchmark: Create a dataset matching production distribution and run comparative benchmarks.
• Test Filter Tax: Measure latency impact of metadata filters with high selectivity.
• Evaluate Quantization: Test scalar quantization impact on recall; enable if delta is acceptable.
• Configure Hybrid Search: Implement BM25 + Vector retrieval with RRF re-ranking.
• Plan Multi-Tenancy: Design sharding or payload indexing strategy for tenant isolation.
• Monitor Drift: Implement monitoring for embedding drift and index health.

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
Small Team / Simple RAG	pgvector	Low ops overhead, integrates with existing Postgres. Sufficient for <1M vectors.	Low (Shared infra)
High Scale / Filter Heavy	Milvus	Superior filter performance, disk-based storage, high throughput.	Medium-High (Compute/Storage)
Low Latency / Managed	Pinecone	Fully managed, optimized performance, no ops.	High (Per-vector pricing)
Multi-Tenant SaaS	Qdrant	Excellent multi-tenancy support, Rust performance, cost-efficient.	Medium (Self-hosted)
Hybrid Search Focus	Weaviate	Native BM25 + Vector integration, GraphQL API.	Medium
Strict Budget / Edge	FAISS / Local	No external dependencies, runs on edge devices.	Low (Dev time)

Configuration Template

Qdrant Production Configuration (config.yaml) Optimized for high recall and efficient filtering.

storage:
  wal:
    wal_capacity_mb: 32
    segment_flush_interval_sec: 5
  optimizers:
    default_segment_number: 5
    memmap_threshold: 100000
    indexing_threshold: 10000
    flush_interval_sec: 5
  
service:
  host: 0.0.0.0
  http_port: 6333
  grpc_port: 6334

cluster:
  enabled: true
  p2p:
    port: 6335
  consensus:
    tick_interval_ms: 100

# Collection Configuration via API
# {
#   "vectors": {
#     "size": 768,
#     "distance": "Cosine",
#     "on_disk": true
#   },
#   "optimizers_config": {
#     "default_segment_number": 10,
#     "memmap_threshold": 50000
#   },
#   "hnsw_config": {
#     "m": 32,
#     "ef_construct": 128,
#     "full_scan_threshold": 10000
#   },
#   "quantization_config": {
#     "scalar": {
#       "type": "int8",
#       "quantile": 0.99
#     }
#   },
#   "payload_index_schema": {
#     "tenant_id": { "type": "keyword" }
#   }
# }

• Key Settings:
  • on_disk: true: Reduces RAM usage by storing vectors on disk.
  • hnsw_config.m: 32: Balances recall and memory.
  • quantization_config.scalar: Enables INT8 quantization for 4x memory reduction.
  • payload_index_schema: Ensures efficient filtering on tenant_id.

Quick Start Guide

1. Spin Up Instance:

   # Qdrant
   docker run -p 6333:6333 -p 6334:6334 \
     -v $(pwd)/qdrant_storage:/qdrant/storage \
     qdrant/qdrant
   
   # Milvus Standalone
   docker run -d --name milvus-standalone \
     -p 19530:19530 \
     -p 9091:9091 \
     milvusdb/milvus:latest
   

2. Create Collection: Use the API or CLI to create a collection with appropriate vector size and distance metric. Apply the configuration template settings.

3. Load Data: Insert a sample dataset. Ensure embeddings are normalized. Add metadata fields for filtering.

   # Example using curl for Qdrant
   curl -X PUT "http://localhost:6333/collections/my_collection" \
     -H "Content-Type: application/json" \
     -d '{...collection config...}'
   

4. Run Test Query: Execute a search query with and without filters. Measure latency and verify recall.

   const results = await client.search("my_collection", {
     vector: queryEmbedding,
     limit: 10,
     filter: { must: [{ key: "tenant_id", match: { value: "tenant_42" } }] }
   });
   

5. Benchmark & Tune: Run the benchmarking script. Adjust ef_search, m, and quantization settings based on results. Iterate until SLA is met.