⚙️ Configuration & Logging

Complete configuration example with all available options:

{
  "Logging": {
    "LogLevel": {
      "Default": "Information",
      "Microsoft.AspNetCore": "Warning",
      "WebSpark.HttpClientUtility": "Debug"
    }
  },
  
  // cURL Command Logging Configuration
  "CsvOutputFolder": "C:\\Logs\\CurlCommands",
  "CsvFileName": "http_requests",
  "CurlCommandSaver": {
    "UseBatchProcessing": true,
    "BatchSize": 100,
    "BatchFlushIntervalMs": 5000,
 "MaxFileSize": 10485760,  // 10MB
    "MaxRetries": 3,
    "RetryDelayMs": 200,
    "SanitizeSensitiveInfo": true,
 "SensitiveHeaders": [
      "Authorization",
      "Api-Key",
      "X-Api-Key",
      "Password",
      "Token",
      "Cookie"
    ]
  },

  // HTTP Client Configuration
  "HttpClient": {
    "Timeout": "00:00:30",  // 30 seconds
 "MaxRetries": 3,
    "RetryDelayMs": 1000
  },
  
  // Polly Resilience Configuration
  "PollyOptions": {
    "MaxRetryAttempts": 3,
    "RetryDelaySeconds": 1,
    "RetryStrategy": "Exponential",
    "EnableCircuitBreaker": true,
    "CircuitBreakerThreshold": 5,
    "CircuitBreakerDurationSeconds": 30
  },
  
  // Cache Configuration
  "CacheOptions": {
    "DefaultDurationMinutes": 5,
    "MaxCacheSizeMB": 100
  },
  
  // OpenTelemetry Configuration
  "OpenTelemetry": {
    "ServiceName": "WebSpark.HttpClientUtility.Demo",
    "ServiceVersion": "1.3.0",
    "ConsoleExporter": true,
    "JaegerExporter": false,
    "JaegerEndpoint": "http://localhost:14268",
    "OtlpExporter": false,
    "OtlpEndpoint": "http://localhost:4317"
  }
}

Full Program.cs configuration with all features enabled:

using Microsoft.Extensions.Caching.Memory;
using WebSpark.HttpClientUtility.ClientService;
using WebSpark.HttpClientUtility.RequestResult;
using WebSpark.HttpClientUtility.StringConverter;
using WebSpark.HttpClientUtility.CurlService;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;

var builder = WebApplication.CreateBuilder(args);

// === Core Services ===
builder.Services.AddHttpClient();
builder.Services.AddMemoryCache();
builder.Services.AddControllersWithViews();

// === JSON Serialization ===
builder.Services.AddSingleton<IStringConverter, SystemJsonStringConverter>();
// Or use Newtonsoft.Json:
// builder.Services.AddSingleton<IStringConverter, NewtonsoftJsonStringConverter>();

// === Basic HTTP Client Service ===
builder.Services.AddScoped<IHttpClientService, HttpClientService>();

// === HttpRequestResult Service with Full Decorator Chain ===
builder.Services.AddScoped<HttpRequestResultService>(); // Base service

builder.Services.AddScoped<IHttpRequestResultService>(provider =>
{
    // Start with base service
    IHttpRequestResultService service = 
        provider.GetRequiredService<HttpRequestResultService>();
    
    // Layer 1: Add Caching (optional)
    if (builder.Configuration.GetValue<bool>("Features:EnableCaching", true))
    {
    service = new HttpRequestResultServiceCache(
          provider.GetRequiredService<ILogger<HttpRequestResultServiceCache>>(),
         service,
            provider.GetRequiredService<IMemoryCache>()
        );
    }
    
    // Layer 2: Add Polly Resilience (optional)
    if (builder.Configuration.GetValue<bool>("Features:EnablePolly", true))
    {
        var pollyOptions = new HttpRequestResultPollyOptions
      {
            MaxRetryAttempts = builder.Configuration.GetValue<int>("PollyOptions:MaxRetryAttempts", 3),
            RetryDelay = TimeSpan.FromSeconds(
              builder.Configuration.GetValue<int>("PollyOptions:RetryDelaySeconds", 1)),
            EnableCircuitBreaker = builder.Configuration.GetValue<bool>(
                "PollyOptions:EnableCircuitBreaker", true),
  CircuitBreakerThreshold = builder.Configuration.GetValue<int>(
      "PollyOptions:CircuitBreakerThreshold", 5),
       CircuitBreakerDuration = TimeSpan.FromSeconds(
      builder.Configuration.GetValue<int>("PollyOptions:CircuitBreakerDurationSeconds", 30))
        };
        
     service = new HttpRequestResultServicePolly(
        provider.GetRequiredService<ILogger<HttpRequestResultServicePolly>>(),
       service,
     pollyOptions
        );
}
    
    // Layer 3: Add Telemetry (outermost layer - always enabled)
    service = new HttpRequestResultServiceTelemetry(
        provider.GetRequiredService<ILogger<HttpRequestResultServiceTelemetry>>(),
        service
    );
    
    return service;
});

// === cURL Command Logging (optional) ===
if (!string.IsNullOrWhiteSpace(builder.Configuration["CsvOutputFolder"]))
{
    builder.Services.AddSingleton<CurlCommandSaver>(provider =>
        new CurlCommandSaver(
        provider.GetRequiredService<ILogger<CurlCommandSaver>>(),
            builder.Configuration
 )
    );
}

// === OpenTelemetry (optional) ===
if (builder.Configuration.GetValue<bool>("OpenTelemetry:Enabled", false))
{
    builder.Services.AddOpenTelemetry()
        .WithTracing(tracerBuilder =>
        {
    tracerBuilder
  .SetResourceBuilder(ResourceBuilder.CreateDefault()
        .AddService(
         builder.Configuration["OpenTelemetry:ServiceName"] ?? "MyApp",
     builder.Configuration["OpenTelemetry:ServiceVersion"] ?? "1.0.0"))
     .AddSource("WebSpark.HttpClientUtility")
  .AddHttpClientInstrumentation()
     .AddAspNetCoreInstrumentation();
 
            if (builder.Configuration.GetValue<bool>("OpenTelemetry:ConsoleExporter", false))
  {
        tracerBuilder.AddConsoleExporter();
  }
 
      if (builder.Configuration.GetValue<bool>("OpenTelemetry:JaegerExporter", false))
  {
         tracerBuilder.AddJaegerExporter(options =>
         {
  options.AgentHost = builder.Configuration["OpenTelemetry:JaegerHost"] ?? "localhost";
       options.AgentPort = builder.Configuration.GetValue<int>("OpenTelemetry:JaegerPort", 6831);
    });
            }
  });
}

var app = builder.Build();

// Configure middleware...
app.UseHttpsRedirection();
app.UseStaticFiles();
app.UseRouting();
app.UseAuthorization();
app.MapControllerRoute(name: "default", pattern: "{controller=Home}/{action=Index}/{id?}");

app.Run();

Log Levels Explained

Example Logging Configuration

{
  "Logging": {
    "LogLevel": {
      // Default for all namespaces
      "Default": "Information",
      
      // Reduce noise from Microsoft libraries
      "Microsoft": "Warning",
      "Microsoft.AspNetCore": "Warning",
      "Microsoft.Hosting.Lifetime": "Information",
      
 // Enable detailed logging for HttpClientUtility
    "WebSpark.HttpClientUtility": "Debug",
      "WebSpark.HttpClientUtility.RequestResult": "Debug",
      "WebSpark.HttpClientUtility.ClientService": "Information",
      "WebSpark.HttpClientUtility.CurlService": "Information",
    
      // External services
   "System.Net.Http.HttpClient": "Warning"
  },
    
 // File logging provider (optional - requires Serilog or similar)
    "File": {
 "Path": "Logs/app-.log",
      "RollingInterval": "Day",
      "RetainedFileCountLimit": 30,
    "OutputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {SourceContext} {Message:lj}{NewLine}{Exception}"
    }
  }
}

Structured Logging Example

// All HttpClientUtility logs include structured data:
// - CorrelationId: Unique request identifier
// - RequestPath: URL being requested
// - RequestMethod: HTTP method (GET, POST, etc.)
// - Duration: Request duration in milliseconds
// - StatusCode: HTTP status code
// - CallerMemberName: Method that made the request
// - CallerFilePath: File where request originated
// - CallerLineNumber: Line number of the request

// Example log output:
// [2025-01-08 10:30:45.123 -05:00] [INF] WebSpark.HttpClientUtility.RequestResult.HttpRequestResultService 
// HTTP Request: GET https://v2.jokeapi.dev/joke/Programming?safe-mode
// CorrelationId: a1b2c3d4-e5f6-7890-abcd-ef1234567890
// Duration: 245ms
// StatusCode: 200
// CallerMemberName: GetJokeAsync
// CallerFilePath: C:\Projects\MyApp\Services\JokeService.cs
// CallerLineNumber: 42

Configuration Options

{
// === Required (Feature is disabled if not set) ===
  "CsvOutputFolder": "C:\\Logs\\CurlCommands",    // Where to save CSV files
  
  // === Optional Settings ===
  "CsvFileName": "http_requests",        // Base filename (default: "curl_commands")

  "CurlCommandSaver": {
    // Batch Processing (improves performance for high-volume scenarios)
    "UseBatchProcessing": true,             // Enable batch mode (default: true)
 "BatchSize": 100,       // Records per batch (default: 100)
    "BatchFlushIntervalMs": 5000,                // Flush interval in ms (default: 5000)
    
  // File Management
    "MaxFileSize": 10485760,           // 10MB - rotate when reached (default: 10MB)
    "MaxRetries": 3,        // Retry attempts for file operations (default: 3)
    "RetryDelayMs": 200,     // Delay between retries (default: 200ms)
    
    // Security & Privacy
    "SanitizeSensitiveInfo": true,   // Redact sensitive data (default: true)
    "SensitiveHeaders": [               // Headers to redact
      "Authorization",
      "Api-Key",
      "X-Api-Key",
      "Password",
      "Token",
   "Cookie",
      "X-Auth-Token",
      "Access-Token"
    ]
  }
}

CSV Output Format

Each HTTP request is logged as a row with the following columns:

File Rotation

When a CSV file reaches MaxFileSize, it's automatically rotated:

// Original file
http_requests.csv

// After rotation (timestamp appended)
http_requests_20250108_103045.csv
http_requests.csv  ← New file starts

Performance Impact

Disabling cURL Logging

To disable, simply remove or comment out the CsvOutputFolder setting:

{
  // "CsvOutputFolder": "C:\\Logs\\CurlCommands",  ← Commented out or removed
  
  // The application will log a warning and continue without file logging:
  // [WARN] CsvOutputFolder is not configured. cURL commands will not be saved to file.
}

For comprehensive distributed tracing and observability:

// Install required packages:
// dotnet add package OpenTelemetry.Extensions.Hosting
// dotnet add package OpenTelemetry.Instrumentation.AspNetCore
// dotnet add package OpenTelemetry.Instrumentation.Http
// dotnet add package OpenTelemetry.Exporter.Console     // For console output
// dotnet add package OpenTelemetry.Exporter.Jaeger   // For Jaeger
// dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol  // For OTLP

builder.Services.AddOpenTelemetry()
  .WithTracing(tracerBuilder =>
  {
        tracerBuilder
// Set service information
            .SetResourceBuilder(ResourceBuilder.CreateDefault()
                .AddService("MyApplication", "1.0.0"))
            
            // Add instrumentation
            .AddSource("WebSpark.HttpClientUtility")
        .AddHttpClientInstrumentation(options =>
{
            // Enrich with additional data
           options.EnrichWithHttpRequestMessage = (activity, request) =>
    {
        activity.SetTag("http.request.header.correlation_id", 
         request.Headers.GetValues("X-Correlation-ID").FirstOrDefault());
             };
          options.EnrichWithHttpResponseMessage = (activity, response) =>
                {
            activity.SetTag("http.response.status_code", (int)response.StatusCode);
           };
       })
 .AddAspNetCoreInstrumentation()
            
            // Choose exporters
            .AddConsoleExporter()  // Development
      .AddJaegerExporter(options =>  // Production
   {
options.AgentHost = "localhost";
     options.AgentPort = 6831;
})
            .AddOtlpExporter(options =>  // OTLP-compatible backends (Grafana, etc.)
            {
                options.Endpoint = new Uri("http://localhost:4317");
   });
    });

The decorator pattern allows you to add features without modifying core code:

Order Matters!

The recommended order is: Base → Cache → Polly → Telemetry

Use different settings for Development, Staging, and Production:

File Structure

appsettings.json    ← Base settings (all environments)
appsettings.Development.json          ← Development overrides
appsettings.Staging.json   ← Staging overrides
appsettings.Production.json           ← Production overrides

appsettings.Development.json

{
  "Logging": {
    "LogLevel": {
      "Default": "Debug",
      "WebSpark.HttpClientUtility": "Trace"  // Maximum detail for development
  }
  },
  "CsvOutputFolder": "C:\\Dev\\Logs", // Local dev folder
  "CurlCommandSaver": {
    "UseBatchProcessing": false,              // Immediate writes for debugging
    "SanitizeSensitiveInfo": false            // Full data in development
  },
  "OpenTelemetry": {
    "ConsoleExporter": true,      // Console output for easy viewing
    "JaegerExporter": false
  }
}

appsettings.Production.json

{
  "Logging": {
    "LogLevel": {
"Default": "Information",
      "WebSpark.HttpClientUtility": "Information"  // Less noise in production
    }
  },
  "CsvOutputFolder": "/var/log/myapp/curl",      // Production log path
  "CurlCommandSaver": {
    "UseBatchProcessing": true,       // Better performance
    "BatchSize": 1000,         // Larger batches
    "SanitizeSensitiveInfo": true        // Always sanitize in production!
  },
  "OpenTelemetry": {
    "ConsoleExporter": false,
    "JaegerExporter": true,  // Send to Jaeger/Grafana
    "OtlpExporter": true
  }
}