// app/Services/SearchDataIngestor.php

namespace App\Services;

use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Collection;

class SearchDataIngestor
{
    private const API_ENDPOINT = 'https://api.dataforseo.com/v3';

    public function pullDomainKeywords(string $targetDomain, string $locale = 'en', int $minVolume = 50): Collection
    {
        $payload = [[
            'target' => $targetDomain,
            'location_code' => 2840,
            'language_code' => $locale,
            'include_serp_info' => true,
        ]];

        $response = Http::withBasicAuth(
            config('services.dataforseo.login'),
            config('services.dataforseo.password')
        )->timeout(30)->post(self::API_ENDPOINT . '/keywords_data/google_ads/keywords_for_site/live', $payload);

        if ($response->failed()) {
            Log::error('DataForSEO ingestion failed', ['status' => $response->status()]);
            return collect();
        }

        return collect($response->json('tasks.0.result'))
            ->map(fn(array $row) => [
                'term' => $row['keyword'] ?? '',
                'monthly_volume' => (int) ($row['search_volume'] ?? 0),
                'difficulty_score' => (float) ($row['keyword_difficulty'] ?? 0.0),
                'estimated_cpc' => (float) ($row['cpc'] ?? 0.0),
            ])
            ->filter(fn(array $item) => $item['monthly_volume'] >= $minVolume && strlen($item['term']) > 2);
    }
}

// app/Jobs/RouteSearchIntent.php

namespace App\Jobs;

use App\Models\SearchTerm;
use Illuminate\Bus\Batchable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Support\Facades\Bus;
use OpenAI\Laravel\Facades\OpenAI;

class RouteSearchIntent implements ShouldQueue
{
    use Batchable;

    public function __construct(protected array $termBatch) {}

    public function handle(): void
    {
        $formattedTerms = implode("\n", array_column($this->termBatch, 'term'));

        $completion = OpenAI::chat()->create([
            'model' => 'gpt-4o-mini',
            'messages' => [
                ['role' => 'system', 'content' => 'Classify search terms by intent. Return strictly JSON.'],
                ['role' => 'user', 'content' => "Map each term to one of: informational, navigational, commercial, transactional.\n\n{$formattedTerms}"]
            ],
            'response_format' => ['type' => 'json_object'],
        ]);

        $parsed = json_decode($completion->choices[0]->message->content, true);
        $mappings = $parsed['classifications'] ?? [];

        foreach ($mappings as $entry) {
            SearchTerm::where('term', $entry['term'])->update([
                'intent_category' => $entry['intent'],
                'classified_at' => now(),
            ]);
        }
    }
}

$chunks = $ingestedTerms->chunk(25);

Bus::batch(
    $chunks->map(fn($chunk) => new RouteSearchIntent($chunk->toArray()))->all()
)
->name('intent-routing')
->onQueue('ai-processing')
->dispatch();

le maintaining sufficient reasoning accuracy for intent mapping. The onQueue('ai-processing') isolation prevents AI jobs from starving critical application queues.

3. Semantic Gap Detection

Exact keyword matching fails to capture topical coverage. Embeddings measure semantic proximity between existing pages and target terms. We calculate cosine similarity to identify gaps where no page adequately addresses the query.

// app/Services/ContentGapAnalyzer.php

namespace App\Services;

use Illuminate\Support\Collection;
use OpenAI\Laravel\Facades\OpenAI;

class ContentGapAnalyzer
{
    private const EMBEDDING_MODEL = 'text-embedding-3-small';
    private const SIMILARITY_THRESHOLD = 0.82;

    public function identifyUncoveredTerms(Collection $keywords, Collection $publishedPages): Collection
    {
        $pageVectors = $publishedPages->map(fn($page) => [
            'url' => $page->slug,
            'vector' => $this->generateVector($page->title . ' ' . $page->summary),
        ]);

        return $keywords->filter(function ($keyword) use ($pageVectors) {
            $queryVector = $this->generateVector($keyword['term']);
            
            $highestMatch = $pageVectors->max(fn($page) => 
                $this->calculateCosineSimilarity($queryVector, $page['vector'])
            );

            return $highestMatch < self::SIMILARITY_THRESHOLD;
        });
    }

    private function generateVector(string $input): array
    {
        $response = OpenAI::embeddings()->create([
            'model' => self::EMBEDDING_MODEL,
            'input' => $input,
        ]);

        return $response->embeddings[0]->embedding;
    }

    private function calculateCosineSimilarity(array $vecA, array $vecB): float
    {
        $dotProduct = array_sum(array_map(fn($a, $b) => $a * $b, $vecA, $vecB));
        $magnitudeA = sqrt(array_sum(array_map(fn($x) => $x ** 2, $vecA)));
        $magnitudeB = sqrt(array_sum(array_map(fn($x) => $x ** 2, $vecB)));

        return $magnitudeA && $magnitudeB ? $dotProduct / ($magnitudeA * $magnitudeB) : 0.0;
    }
}

Architecture Rationale: The 0.82 threshold is empirically derived for general-purpose content. Niche technical domains may require lowering it to 0.75 to avoid false gaps. Embedding generation is isolated in a private method to enable future caching or vector database offloading. Cosine similarity is computed natively to avoid external dependencies.

4. Metadata Synthesis & Human Validation

Automated meta generation should never bypass editorial review. The pipeline drafts compliant titles and descriptions, then routes them to a staging queue for approval.

// app/Jobs/SynthesizePageMetadata.php

namespace App\Jobs;

use App\Models\ContentPage;
use Illuminate\Contracts\Queue\ShouldQueue;
use OpenAI\Laravel\Facades\OpenAI;

class SynthesizePageMetadata implements ShouldQueue
{
    public function __construct(public ContentPage $page) {}

    public function handle(): void
    {
        $completion = OpenAI::chat()->create([
            'model' => 'gpt-4o-mini',
            'messages' => [
                ['role' => 'system', 'content' => 'Draft SEO metadata. Max 155 chars for description. Include primary keyword naturally. Avoid clickbait.'],
                ['role' => 'user', 'content' => "Title: {$this->page->title}\nExcerpt: {$this->page->lead_paragraph}\n\nReturn JSON with keys: 'meta_title', 'meta_description'."]
            ],
            'response_format' => ['type' => 'json_object'],
        ]);

        $draft = json_decode($completion->choices[0]->message->content, true);

        $this->page->update([
            'draft_meta_title' => substr($draft['meta_title'] ?? $this->page->title, 0, 60),
            'draft_meta_description' => substr($draft['meta_description'] ?? '', 0, 155),
            'metadata_status' => 'pending_review',
        ]);
    }
}

Architecture Rationale: Drafts are stored in separate columns (draft_meta_*) to prevent accidental production deployment. Length truncation happens at the application layer, not relying solely on the model. The pending_review status integrates cleanly with Livewire approval interfaces, enabling editors to accept, modify, or reject suggestions with full audit trails.

Pitfall Guide

1. Unbatched AI Requests

Explanation: Dispatching individual jobs per keyword or page exhausts OpenAI rate limits and inflates costs. Queue workers also compete for API slots, causing timeouts. Fix: Always chunk payloads (20–50 items) and use Bus::batch(). Implement exponential backoff on 429 responses and route AI jobs to a dedicated queue with concurrency limits.

2. Hardcoded Similarity Thresholds

Explanation: A fixed cosine similarity cutoff (e.g., 0.82) works for broad topics but fails for highly technical or localized content where semantic variance is naturally lower. Fix: Store thresholds in configuration or database. Allow per-category overrides. Log gap detections that fall near the threshold for manual calibration.

3. Ignoring Embedding Model Versioning

Explanation: OpenAI periodically updates embedding models. Vectors generated with text-embedding-3-small v1 differ from v2, causing drift in similarity calculations over time. Fix: Tag stored embeddings with a model_version field. Schedule quarterly re-embedding jobs for high-traffic pages. Maintain a vector cache table to avoid redundant API calls.

4. Skipping Human-in-the-Loop Validation

Explanation: Shipping AI-generated metadata directly to production risks brand voice misalignment, factual inaccuracies, and compliance violations. Fix: Implement a staging workflow. Use a diff-view interface for editors to compare current vs. drafted metadata. Require explicit approval before publishing. Log all changes for auditability.

5. Token Budget Blind Spots

Explanation: Embedding and chat completions accumulate costs rapidly at scale. A pipeline processing 10k keywords monthly can easily exceed $150–$300 without monitoring. Fix: Track token usage per job using middleware or queue events. Implement caching for repeated queries. Use gpt-4o-mini for classification/drafting and reserve gpt-4 for complex reasoning only. Set budget alerts in OpenAI's dashboard.

6. Over-Optimizing Meta Descriptions

Explanation: Forcing exact keyword matches into meta descriptions triggers search engine penalties for stuffing and degrades click-through rates. Fix: Instruct the model to prioritize natural language and value propositions. Enforce character limits strictly. Validate output against a regex pattern that flags excessive keyword repetition before staging.

7. Silent Queue Failures

Explanation: AI jobs can fail due to API changes, malformed responses, or network timeouts. Without monitoring, pipelines degrade silently, leaving content gaps unaddressed. Fix: Enable Laravel Horizon with Slack/email failure notifications. Implement job retry policies with tries(3) and backoff(). Log raw API responses for debugging. Schedule weekly pipeline health checks.

Production Bundle

Action Checklist

• Configure DataForSEO credentials in .env and verify API connectivity with a dry-run request
• Create dedicated queue (ai-processing) with concurrency limits matching your OpenAI tier
• Implement staging columns for all AI-generated fields (draft_*, metadata_status)
• Build a Livewire approval interface with diff viewing and one-click publish
• Set up Laravel Horizon with failure alerts routed to your team's communication channel
• Add token usage tracking via queue events and log to a metrics table
• Schedule weekly pipeline execution with staggered commands to prevent API contention
• Document threshold configurations and review them quarterly with editorial leads

Decision Matrix

Scenario	Recommended Approach	Why	Cost Impact
Startup / Low Volume (<5k keywords)	Single queue, gpt-4o-mini for all tasks	Simplicity outweighs optimization needs	Low ($15–$40/mo)
Mid-Market / High Volume (5k–50k)	Dedicated AI queue, batched jobs, embedding cache	Prevents queue starvation and reduces redundant API calls	Medium ($60–$150/mo)
Enterprise / Compliance-Heavy	Human-in-the-loop mandatory, vector DB offload, model versioning	Ensures auditability, brand safety, and long-term vector consistency	High ($200–$500+/mo)
Budget-Constrained	Skip embeddings, use TF-IDF + exact match for gap detection	Reduces OpenAI dependency while maintaining basic coverage	Minimal ($5–$15/mo)

Configuration Template

# .env
DATAFORSEO_LOGIN=your_login
DATAFORSEO_PASSWORD=your_password

OPENAI_API_KEY=sk-proj-xxxx
OPENAI_ORG=org-xxxx

# Queue & Horizon
QUEUE_CONNECTION=database
HORIZON_PREFIX=horizon:
HORIZON_BALANCE_STRATEGY=auto

# Pipeline Thresholds
CONTENT_GAP_SIMILARITY_THRESHOLD=0.82
META_DESCRIPTION_MAX_LENGTH=155
META_TITLE_MAX_LENGTH=60
AI_BATCH_SIZE=25

// config/services.php
'dataforseo' => [
    'login' => env('DATAFORSEO_LOGIN'),
    'password' => env('DATAFORSEO_PASSWORD'),
],

'openai' => [
    'api_key' => env('OPENAI_API_KEY'),
    'organization' => env('OPENAI_ORG'),
],

// routes/console.php
use Illuminate\Support\Facades\Schedule;

Schedule::command('seo:ingest-keywords')
    ->weekly()
    ->mondays()
    ->at('02:00');

Schedule::command('seo:route-intents')
    ->weekly()
    ->mondays()
    ->at('02:30')
    ->withoutOverlapping();

Schedule::command('seo:analyze-gaps')
    ->weekly()
    ->wednesdays()
    ->at('02:00');

Schedule::command('seo:synthesize-metadata')
    ->daily()
    ->at('03:00');

Quick Start Guide

1. Initialize the environment: Run composer require openai-php/laravel and configure your .env with DataForSEO and OpenAI credentials. Publish Horizon config with php artisan vendor:publish --provider="Laravel\Horizon\HorizonServiceProvider".
2. Seed the database: Create migration tables for search_terms, content_pages, and metadata_drafts. Run php artisan migrate.
3. Test ingestion: Execute php artisan tinker and call SearchDataIngestor::pullDomainKeywords('example.com'). Verify filtered results persist to search_terms.
4. Run a dry batch: Dispatch a single RouteSearchIntent job with 5 test terms. Monitor Horizon for successful completion and verify intent_category updates.
5. Deploy the scheduler: Enable php artisan schedule:work locally or configure your server's cron to run php artisan schedule:run every minute. Verify pipeline execution logs and adjust queue concurrency based on your OpenAI rate limits.

This pipeline transforms SEO from a reactive editorial task into a measurable, scalable operation. By isolating data processing, enforcing human validation, and monitoring token economics, teams can maintain content velocity without sacrificing quality or compliance. Treat the AI as a high-throughput analyst, not an autonomous publisher, and the system will compound in value as your content library grows.