// src/knowledge/source-manager.ts
import { promises as fs } from 'fs';
import path from 'path';

export class SourceManager {
  private readonly sourceDir: string;
  private readonly wikiDir: string;

  constructor(projectRoot: string) {
    this.sourceDir = path.join(projectRoot, '.knowledgebase', 'inputs');
    this.wikiDir = path.join(projectRoot, '.knowledgebase', 'output');
  }

  async validateSources(): Promise<string[]> {
    const files = await fs.readdir(this.sourceDir);
    const markdownFiles = files.filter(f => f.endsWith('.md'));
    
    if (markdownFiles.length === 0) {
      throw new Error('No source documents found. Create .knowledgebase/inputs/ with architecture docs.');
    }
    return markdownFiles;
  }

  async readSource(filename: string): Promise<string> {
    const filePath = path.join(this.sourceDir, filename);
    return fs.readFile(filePath, 'utf-8');
  }
}

// src/knowledge/schema-config.ts
export interface WikiSchema {
  domain: 'code' | 'research' | 'business';
  entities: string[];
  relationships: { source: string; target: string; type: string }[];
  pageTemplates: {
    entity: string;
    concept: string;
    synthesis: string;
  };
  extractionRules: {
    pattern: RegExp;
    category: string;
  }[];
}

export const defaultCodeSchema: WikiSchema = {
  domain: 'code',
  entities: ['Module', 'Endpoint', 'DatabaseTable', 'Service', 'Configuration'],
  relationships: [
    { source: 'Module', target: 'Endpoint', type: 'exposes' },
    { source: 'Service', target: 'DatabaseTable', type: 'queries' },
    { source: 'Configuration', target: 'Service', type: 'configures' }
  ],
  pageTemplates: {
    entity: '# {name}\n**Type:** {type}\n**Location:** {path}\n**Description:** {desc}\n**References:** {refs}',
    concept: '# {name}\n**Category:** {category}\n**Context:** {context}\n**Related:** {related}',
    synthesis: '# {name}\n**Question:** {question}\n**Answer:** {answer}\n**Sources:** {sources}'
  },
  extractionRules: [
    { pattern: /export\s+(class|interface|function)\s+(\w+)/g, category: 'Module' },
    { pattern: /(app|router)\.(get|post|put|delete)\(['"]([^'"]+)['"]/g, category: 'Endpoint' },
    { pattern: /CREATE\s+TABLE\s+(\w+)/gi, category: 'DatabaseTable' }
  ]
};

// src/knowledge/ingestion-pipeline.ts
import { SourceManager } from './source-manager';
import { WikiSchema } from './schema-config';

export class IngestionPipeline {
  constructor(
    private sourceManager: SourceManager,
    private schema: WikiSchema
  ) {}

  async execute(

): Promise<void> { const sources = await this.sourceManager.validateSources(); const extractedEntities: Map<string, any> = new Map();

for (const file of sources) {
  const content = await this.sourceManager.readSource(file);
  this.schema.extractionRules.forEach(rule => {
    let match;
    const regex = new RegExp(rule.pattern);
    while ((match = regex.exec(content)) !== null) {
      const name = match[2] || match[1];
      if (!extractedEntities.has(name)) {
        extractedEntities.set(name, {
          name,
          category: rule.category,
          source: file,
          context: content.substring(Math.max(0, match.index - 200), match.index + 200)
        });
      }
    }
  });
}

await this.generateWikiPages(extractedEntities);

}

private async generateWikiPages(entities: Map<string, any>): Promise<void> { // Logic to format entities into schema templates, resolve cross-references, // and write markdown files to the wiki directory. // Health scoring and orphan detection run here. console.log(Wiki sync complete. ${entities.size} entities indexed.); } }

### Step 4: Register the MCP Server

The Model Context Protocol enables standardized tool exposure to AI agents. The server runs as a local subprocess, communicating via stdio. This eliminates network overhead and keeps data within the developer's machine.

```typescript
// src/mcp/knowledge-server.ts
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import { IngestionPipeline } from '../knowledge/ingestion-pipeline';

export class KnowledgeBaseServer {
  private server: Server;

  constructor() {
    this.server = new Server(
      { name: 'persistent-knowledge-server', version: '1.0.0' },
      { capabilities: { tools: {} } }
    );

    this.registerTools();
  }

  private registerTools(): void {
    // Tool: query_wiki
    this.server.setRequestHandler('tools/call', async (request) => {
      const { name, arguments: args } = request.params;
      
      if (name === 'query_wiki') {
        const question = args?.question as string;
        // Direct lookup in generated wiki pages, synthesize answer with citations
        return { content: [{ type: 'text', text: `Synthesized answer for: ${question}` }] };
      }

      if (name === 'sync_knowledge') {
        const pipeline = new IngestionPipeline(new SourceManager(process.cwd()), {} as any);
        await pipeline.execute();
        return { content: [{ type: 'text', text: 'Knowledge base synchronized.' }] };
      }

      throw new Error(`Unknown tool: ${name}`);
    });
  }

  async start(): Promise<void> {
    const transport = new StdioServerTransport();
    await this.server.connect(transport);
    console.log('Knowledge server listening on stdio...');
  }
}

Architecture Rationale

Why Markdown Over Vector Databases? Vector stores require embedding models, sync pipelines, and external dependencies. Markdown files are natively git-trackable, diffable, and instantly readable by both humans and LLMs. The trade-off is slightly higher token usage per query, but this is offset by eliminating embedding latency and vector search overhead.

Why stdio Communication? Network-based MCP servers introduce latency, authentication complexity, and potential data leakage. stdio keeps the agent-tool communication local, deterministic, and secure. It aligns with how Claude, Copilot, and Cursor natively consume MCP tools.

Why Immutable Sources? Separating human-authored inputs from machine-generated outputs creates a clear boundary for audits. If the wiki degrades, you rebuild from sources. If sources change, you trigger a sync. This prevents drift and ensures reproducibility.

Pitfall Guide

1. Treating Generated Wiki as Source

Explanation: Developers occasionally edit the generated markdown pages directly, expecting changes to persist. The next sync cycle overwrites these edits because the pipeline treats the wiki as ephemeral output. Fix: Enforce strict read-only permissions on the wiki directory. Route all modifications through the source layer or use the save_answer_as_page tool, which safely appends to the wiki without breaking the sync contract.

2. Ignoring Health Scores & Orphan Nodes

Explanation: The wiki accumulates pages over time. Without monitoring, orphaned concepts (pages with no incoming references) and stale entities drift into the knowledge base, degrading answer quality. Fix: Run the health diagnostic tool after every sync. Address orphan nodes by adding explicit cross-references in related pages. Archive pages that haven't been referenced in 3+ sync cycles.

3. Overloading the Ingestion Pipeline

Explanation: Dumping entire codebases into the source directory causes the extraction engine to generate thousands of low-signal pages. The agent spends tokens navigating noise instead of high-value architecture. Fix: Curate sources intentionally. Limit inputs to architecture diagrams, API contracts, deployment guides, and critical workflow documentation. Let the agent discover trivial files on-demand.

4. Skipping Domain Schema Customization

Explanation: Using the default schema on specialized projects (e.g., data pipelines, embedded systems, research papers) results in misaligned entity extraction. The wiki generates generic pages that miss domain-specific relationships. Fix: Override the WikiSchema configuration before first sync. Define custom extraction patterns, relationship types, and page templates that match your project's terminology and structure.

5. Neglecting Cross-Reference Validation

Explanation: The LLM generates pages independently. Without explicit cross-reference rules, pages become isolated islands. Agents receive fragmented answers instead of connected knowledge graphs. Fix: Implement a post-sync validation step that scans for broken or missing references. Use the dependency graph tool to visualize coupling and manually link disconnected clusters.

6. Running Without Watch Mode in Active Development

Explanation: Codebases evolve rapidly. Static syncs become stale within hours. Agents query outdated wiki pages, leading to hallucinated implementations or deprecated API references. Fix: Enable background watch mode during active development. Configure it to trigger incremental syncs on file changes, focusing only on modified source documents rather than full rebuilds.

7. Mixing Environment Secrets with Source Docs

Explanation: Developers accidentally include .env files, API keys, or internal credentials in the source directory. The ingestion pipeline extracts and indexes them, exposing sensitive data to agent queries. Fix: Implement a pre-sync scanner that rejects files containing secret patterns. Maintain a .knowledgeignore file similar to .gitignore to explicitly exclude sensitive paths.

Production Bundle

Action Checklist

• Initialize knowledge directory: Create .knowledgebase/inputs/ and populate with curated architecture docs, API contracts, and deployment guides.
• Configure domain schema: Override default extraction rules and page templates to match your project's terminology and relationship structure.
• Run initial sync: Execute the ingestion pipeline to generate the first wiki layer. Verify health score exceeds 90/100.
• Register MCP server: Configure your AI agent (Claude, Copilot, Cursor) to connect via stdio. Validate tool exposure with a test query.
• Enable watch mode: Start the background daemon to track source changes. Verify incremental syncs trigger on file modifications.
• Implement cross-reference rules: Use the dependency graph tool to identify orphan pages. Manually link disconnected concepts.
• Schedule periodic audits: Run health diagnostics weekly. Archive stale pages and update extraction rules as the codebase evolves.

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
Small team, rapid iteration	Watch mode + incremental sync	Keeps wiki current without full rebuilds	Low compute, minimal latency
Large monorepo, strict compliance	Manual sync + health validation	Prevents drift, ensures audit trail	Higher manual overhead, zero risk
Multi-domain project (code + research)	Custom schema per domain	Aligns extraction with domain terminology	Medium setup time, high accuracy
Air-gapped / high-security environment	Local stdio + markdown storage	No external APIs, zero network exposure	Zero cloud cost, full data control
Legacy codebase, sparse docs	AI-assisted doc generation + sync	Bootstraps wiki from code patterns	Higher initial LLM cost, fast ROI

Configuration Template

{
  "knowledgeBase": {
    "version": "1.0",
    "schema": {
      "domain": "code",
      "entities": ["Service", "Endpoint", "DatabaseTable", "Middleware", "Configuration"],
      "relationships": [
        {"source": "Service", "target": "Endpoint", "type": "exposes"},
        {"source": "Middleware", "target": "Endpoint", "type": "protects"},
        {"source": "Service", "target": "DatabaseTable", "type": "queries"}
      ],
      "extractionPatterns": [
        {"regex": "export\\s+(class|interface)\\s+(\\w+)", "category": "Service"},
        {"regex": "(app|router)\\.(get|post|put|delete)\\(['\"]([^'\"]+)['\"]", "category": "Endpoint"},
        {"regex": "process\\.env\\.(\\w+)", "category": "Configuration"}
      ]
    },
    "sync": {
      "mode": "watch",
      "debounceMs": 1500,
      "maxConcurrency": 3,
      "healthThreshold": 85
    },
    "ignore": [
      "**/node_modules/**",
      "**/.git/**",
      "**/*.test.ts",
      "**/.env*",
      "**/secrets/**"
    ]
  }
}

Quick Start Guide

1. Create the knowledge directory: Run mkdir -p .knowledgebase/inputs in your project root. Add 3-5 markdown files covering architecture, authentication, database schema, and deployment.
2. Configure the schema: Copy the configuration template above into .knowledgebase/config.json. Adjust extraction patterns to match your framework's syntax.
3. Initialize the server: Install the MCP SDK, register the stdio transport, and expose the query_wiki, sync_knowledge, and health_check tools. Start the subprocess.
4. Connect your agent: Configure Claude, Copilot, or Cursor to use the local MCP server. Test with a query like query_wiki("How does authentication flow work?").
5. Enable continuous sync: Launch the watch daemon. Verify that modifying a source document triggers an incremental wiki update within 2 seconds.