ersist events to the outbox table within the same transaction.
public class OrderDbContext : DbContext
{
public DbSet<Order> Orders { get; set; }
public DbSet<OutboxMessage> OutboxMessages { get; set; }
public override async Task<int> SaveChangesAsync(CancellationToken cancellationToken = default)
{
// Extract domain events from tracked entities
var domainEvents = ChangeTracker.Entries<IEntity>()
.Select(e => e.Entity)
.SelectMany(e =>
{
var events = e.DomainEvents.ToList();
e.DomainEvents.Clear();
return events;
})
.ToList();
// Convert to outbox messages
var outboxMessages = domainEvents.Select(e => new OutboxMessage
{
Id = Guid.NewGuid(),
Type = e.GetType().AssemblyQualifiedName!,
Content = JsonSerializer.Serialize(e),
OccurredOn = DateTime.UtcNow
}).ToList();
await OutboxMessages.AddRangeAsync(outboxMessages, cancellationToken);
var result = await base.SaveChangesAsync(cancellationToken);
// Events are now persisted atomically with business data
return result;
}
}
public class OutboxMessage
{
public Guid Id { get; set; }
public string Type { get; set; } = string.Empty;
public string Content { get; set; } = string.Empty;
public DateTime OccurredOn { get; set; }
public bool Processed { get; set; }
public DateTime? ProcessedOn { get; set; }
}
Background Processor:
A BackgroundService polls for unprocessed messages and publishes them.
public class OutboxProcessor : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<OutboxProcessor> _logger;
public OutboxProcessor(IServiceScopeFactory scopeFactory, ILogger<OutboxProcessor> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
using var scope = _scopeFactory.CreateScope();
var context = scope.ServiceProvider.GetRequiredService<OrderDbContext>();
var publisher = scope.ServiceProvider.GetRequiredService<IEventPublisher>();
var messages = await context.OutboxMessages
.Where(m => !m.Processed)
.OrderBy(m => m.OccurredOn)
.Take(100)
.ToListAsync(stoppingToken);
foreach (var message in messages)
{
try
{
await publisher.PublishAsync(message.Type, message.Content);
message.Processed = true;
message.ProcessedOn = DateTime.UtcNow;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to publish outbox message {Id}", message.Id);
// Leave processed=false for retry
}
}
await context.SaveChangesAsync(stoppingToken);
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
}
}
}
2. Resilience Pipeline with Polly
.NET 8+ integrates Polly via IHttpClientFactory and ResiliencePipeline. This pattern handles transient faults, rate limiting, and circuit breaking.
Configuration:
public static class ResiliencePipelineConfig
{
public static ResiliencePipeline CreateStandardPipeline()
{
return new ResiliencePipelineBuilder()
.AddRetry(new RetryStrategyOptions
{
BackoffType = DelayBackoffType.Exponential,
MaxRetryAttempts = 3,
UseJitter = true
})
.AddCircuitBreaker(new CircuitBreakerStrategyOptions
{
HandlingPredicates = { StandardHandlePredicates.NetworkAndHttp5xxErrors },
MinimumThroughput = 10,
SamplingDuration = TimeSpan.FromSeconds(30),
BreakDuration = TimeSpan.FromSeconds(15)
})
.AddTimeout(TimeSpan.FromSeconds(10))
.Build();
}
}
Injection:
builder.Services.AddResiliencePipeline("order-service-pipeline", builder =>
{
builder.AddRetry(new RetryStrategyOptions { /* config */ });
builder.AddCircuitBreaker(new CircuitBreakerStrategyOptions { /* config */ });
});
Pitfall Guide
1. The Synchronous Communication Trap
Mistake: Relying exclusively on synchronous HTTP/gRPC calls between services for core business flows.
Impact: Creates tight coupling. If Service B is slow or down, Service A's threads are blocked, leading to cascading failures and thread pool starvation.
Best Practice: Use asynchronous event-driven communication for non-critical paths and eventual consistency. Reserve synchronous calls for read operations where immediate consistency is required, and always wrap them in resilience pipelines.
2. Ignoring Idempotency
Mistake: Assuming message delivery is exactly-once or that retries are safe without idempotency checks.
Impact: Duplicate processing of orders, charges, or state changes when retries occur or the Outbox processor re-publishes events.
Best Practice: Implement idempotency keys in all commands and events. Services must check for processed keys before executing state changes. Store processed keys in a deduplication store with a TTL.
3. Shared Database Anti-pattern
Mistake: Multiple microservices accessing the same database schema to avoid duplication.
Impact: Violates bounded context isolation. Schema changes in one service break others. Teams cannot deploy independently.
Best Practice: Enforce database-per-service. If data needs to be shared, replicate it via events or expose it through a dedicated API. Use the Anti-Corruption Layer pattern when integrating with legacy systems.
4. Chatty Interfaces
Mistake: Designing microservice APIs that require multiple round-trips to complete a single user action.
Impact: High latency, increased network overhead, and complex client logic.
Best Practice: Implement the API Composition or Backend-for-Frontend (BFF) pattern. Aggregate data from multiple services into a single response or use GraphQL/BFF to orchestrate calls server-side. Optimize gRPC contracts to minimize payload size.
5. Over-Engineering Resilience
Mistake: Applying complex circuit breakers and bulkheads to every single dependency indiscriminately.
Impact: Increased complexity, difficulty in debugging, and potential masking of critical errors.
Best Practice: Apply resilience patterns based on risk analysis. Critical path dependencies require strict circuit breakers and timeouts. Non-critical dependencies may use simple retries. Configure circuit breakers with appropriate thresholds to prevent premature tripping.
6. Lack of Distributed Tracing
Mistake: Relying on isolated logs per service without correlation IDs.
Impact: Debugging cross-service issues becomes nearly impossible. MTTR increases significantly.
Best Practice: Implement OpenTelemetry from day one. Propagate trace context via HTTP headers or message metadata. Ensure all logs and metrics include trace/span IDs. Use tools like Jaeger or Zipkin for visualization.
7. Neglecting Contract Testing
Mistake: Assuming API contracts remain stable and testing integrations only in staging.
Impact: Breaking changes in one service silently break consumers, leading to runtime failures in production.
Best Practice: Use Consumer-Driven Contract Testing (e.g., Pact). Integrate contract verification into the CI/CD pipeline. Ensure that service interfaces are versioned and backward-compatible.
Production Bundle
Action Checklist
Decision Matrix
| Scenario | Recommended Approach | Why | Cost Impact |
|---|
| High-throughput event processing | Outbox + MassTransit/RabbitMQ | Guarantees delivery, decouples services, scales independently. | Medium (Infra + Dev effort) |
| Low-latency read operations | gRPC with Caching | Binary protocol reduces latency; caching reduces load. | Low |
| Complex business workflow | Saga Pattern (Choreography) | Manages distributed transactions without central coordinator overhead. | Medium (Complexity) |
| External API integration | Resilience Pipeline + Fallback | Handles transient faults gracefully; maintains UX during outages. | Low |
| Team autonomy required | Database-per-service + Event replication | Prevents coupling; allows independent schema evolution. | High (Storage duplication) |
Configuration Template
appsettings.json for Resilience and Messaging:
{
"Resilience": {
"DefaultPipeline": {
"Retry": {
"MaxRetryAttempts": 3,
"Delay": "00:00:01",
"BackoffType": "Exponential"
},
"CircuitBreaker": {
"SamplingDuration": "00:00:30",
"MinimumThroughput": 10,
"BreakDuration": "00:00:15"
},
"Timeout": "00:00:10"
}
},
"Messaging": {
"RabbitMq": {
"Host": "amqp://localhost",
"Username": "guest",
"Password": "guest",
"Outbox": {
"PollingInterval": "00:00:05",
"BatchSize": 50
}
}
}
}
Program.cs Setup Snippet:
var builder = WebApplication.CreateBuilder(args);
// Resilience
builder.Services.AddResiliencePipeline("default", pipeline =>
{
var config = builder.Configuration.GetSection("Resilience:DefaultPipeline");
pipeline.AddRetry(new RetryStrategyOptions
{
MaxRetryAttempts = config.GetValue<int>("Retry:MaxRetryAttempts"),
BackoffType = DelayBackoffType.Exponential
});
pipeline.AddCircuitBreaker(new CircuitBreakerStrategyOptions
{
SamplingDuration = TimeSpan.Parse(config["CircuitBreaker:SamplingDuration"]!),
BreakDuration = TimeSpan.Parse(config["CircuitBreaker:BreakDuration"]!)
});
});
// Messaging & Outbox
builder.Services.AddMassTransit(x =>
{
x.AddEntityFrameworkOutbox<OrderDbContext>(o =>
{
o.QueryDelay = TimeSpan.FromSeconds(5);
o.UseSqlServer();
o.UseBusOutbox();
});
x.UsingRabbitMq((context, cfg) =>
{
cfg.ConfigureEndpoints(context);
});
});
// OpenTelemetry
builder.Services.AddOpenTelemetry()
.WithTracing(tracing => tracing
.AddAspNetCoreInstrumentation()
.AddHttpClientInstrumentation()
.AddEntityFrameworkCoreInstrumentation()
.AddJaegerExporter());
Quick Start Guide
- Initialize Project: Run
dotnet new webapi -n OrderService. Add packages: dotnet add package Polly, dotnet add package MassTransit, dotnet add package MassTransit.EntityFrameworkCore, dotnet add package OpenTelemetry.Exporter.Jaeger.
- Configure DI: In
Program.cs, register OrderDbContext, configure MassTransit with EF Core Outbox, and set up Resilience Pipelines using the template above.
- Implement Outbox Entity: Create
OutboxMessage class and add to DbContext. Override SaveChangesAsync to capture domain events and persist to outbox.
- Add Resilience: Inject
IResiliencePipelineProvider<string> into your service client. Wrap external calls with pipeline.ExecuteAsync(...).
- Run Infrastructure: Start a local RabbitMQ container and SQL Server. Run
dotnet run and verify via logs that outbox messages are processed and traces are exported.
This implementation provides a production-ready foundation for .NET microservices, addressing consistency, resilience, and observability through proven patterns.
