package otelsetup import ( "context" "errors" "fmt" "net" "net/http" "os" "time" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" "go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/trace" ) // SetupOTelSDK bootstraps the OpenTelemetry pipeline. func SetupOTelSDK(ctx context.Context, enabled bool, serviceName, buildVersion, environment string) (func(context.Context) error, error) { if os.Getenv("OTEL_RESOURCE_ATTRIBUTES") == "" { // service.instance.id makes every pod a distinct telemetry resource. The // OTLP→Prometheus exporter maps it to the `instance` label on metrics and // target_info, which keeps multi-replica services from colliding on a // single series and gives joins a unique (job, instance) key. Hostname is // the pod name under Kubernetes; fall back to the service name if it is // unavailable so the attribute is always present. instanceID, err := os.Hostname() if err != nil || instanceID == "" { instanceID = serviceName } if err := os.Setenv("OTEL_RESOURCE_ATTRIBUTES", fmt.Sprintf("service.name=%s,service.version=%s,service.environment=%s,service.instance.id=%s", serviceName, buildVersion, environment, instanceID)); err != nil { return func(context.Context) error { return nil }, err } } var shutdownFuncs []func(context.Context) error if !enabled { return func(context.Context) error { return nil }, nil } shutdown := func(ctx context.Context) error { var err error for _, fn := range shutdownFuncs { err = errors.Join(err, fn(ctx)) } shutdownFuncs = nil return err } // handleErr calls shutdown for cleanup and makes sure that all errors are returned. handleErr := func(inErr error) (func(context.Context) error, error) { return nil, errors.Join(inErr, shutdown(ctx)) } // Set up the propagator. prop := propagation.NewCompositeTextMapPropagator( propagation.TraceContext{}, propagation.Baggage{}, ) otel.SetTextMapPropagator(prop) traceExporter, err := otlptracehttp.New(ctx, otlptracehttp.WithHTTPClient(otlpHTTPClient())) if err != nil { return handleErr(err) } shutdownFuncs = append(shutdownFuncs, traceExporter.Shutdown) tracerProvider := trace.NewTracerProvider( trace.WithBatcher(traceExporter, trace.WithBatchTimeout(5*time.Second)), ) shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) otel.SetTracerProvider(tracerProvider) logExporter, err := stdoutlog.New() if err != nil { return handleErr(err) } processor := log.NewSimpleProcessor(logExporter) logProvider := log.NewLoggerProvider(log.WithProcessor(processor)) global.SetLoggerProvider(logProvider) shutdownFuncs = append(shutdownFuncs, logProvider.Shutdown) exp, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithHTTPClient(otlpHTTPClient())) if err != nil { return handleErr(err) } meterProvider := metric.NewMeterProvider(metric.WithReader(metric.NewPeriodicReader(exp))) shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) otel.SetMeterProvider(meterProvider) return shutdown, err } // otlpHTTPClient is the OTLP exporters' default client, except that it drops idle // connections after 30s. The collector (Alloy's OTLP receiver, idle_timeout 1m) closes // connections idle for a minute, and metrics are pushed every 60s. With the exporters' // default of 90s, a push could reuse a connection the collector was closing. The push // then failed (EOF, connection reset) and was dropped: net/http doesn't retry a POST and // the exporters don't retry transport errors. Traces hit the same race after a quiet // spell. Closing first means such a push dials fresh. // // The transport mirrors the exporters' own (otlpmetrichttp/otlptracehttp v1.46.0 // ourTransport). A custom client makes the exporters ignore OTEL_EXPORTER_OTLP_*TIMEOUT // and the OTEL_EXPORTER_OTLP_*CERTIFICATE/CLIENT_* TLS variables; nothing sets them. func otlpHTTPClient() *http.Client { return &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, ForceAttemptHTTP2: true, MaxIdleConns: 100, IdleConnTimeout: 30 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, }, // The exporters' default. Timeout: 10 * time.Second, } } func Handler(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header)) spanCtx, s := StartSpan(ctx, "http") defer s.Finish() h.ServeHTTP(w, r.WithContext(spanCtx)) }) }