Kibana & Observability

# Kibana alerting rule (via API)
curl -X POST "http://localhost:5601/api/alerting/rule"   -H "kbn-xsrf: true" -H "Content-Type: application/json" -d'
{
  "name": "High Error Rate",
  "rule_type_id": "observability.logs.alert.document.count",
  "consumer": "alerts",
  "schedule": { "interval": "5m" },
  "params": {
    "criteria": [{
      "field": "level",
      "comparator": "equals",
      "value": "ERROR",
      "timeSize": 5,
      "timeUnit": "m",
      "threshold": [100]
    }]
  },
  "actions": [{
    "group": "logs.threshold.fired",
    "id": "slack-connector-id",
    "params": {
      "message": "High error rate detected: {{context.matchingDocuments}} errors in 5 min"
    }
  }]
}'

// Elastic APM .NET Agent integration
// NuGet: Elastic.Apm.NetCoreAll

// Program.cs
builder.Services.AddAllElasticApm(builder.Configuration);

// appsettings.json
// {
//   "ElasticApm": {
//     "ServerUrl": "http://apm-server:8200",
//     "ServiceName": "product-api",
//     "Environment": "production",
//     "TransactionSampleRate": 0.5,
//     "CaptureBody": "errors"
//   }
// }

// Custom span for ES operations
public class TracedSearchService
{
    private readonly ElasticsearchClient _client;
    private readonly ITracer _tracer;

    public async Task<List<Product>> SearchWithTracingAsync(string query)
    {
        var span = _tracer.CurrentTransaction?.StartSpan(
            "ES Search", ApiConstants.TypeDb, "elasticsearch");
        try
        {
            span?.SetLabel("query", query);
            var response = await _client.SearchAsync<Product>(s => s
                .Index("products")
                .Query(q => q.Match(m => m.Field(f => f.Name).Query(query))));
            span?.SetLabel("hits", response.Total);
            return response.Documents.ToList();
        }
        catch (Exception ex)
        {
            span?.CaptureException(ex);
            throw;
        }
        finally
        {
            span?.End();
        }
    }
}