fix: close idle OTLP connections before the collector does

Metrics are pushed every 60s and Alloy's OTLP receiver closes connections idle for 1m, while the exporters keep them for 90s. A push could reuse a connection the collector was closing and fail with EOF or connection reset; the data was dropped (no retry for a POST or for transport errors). In prod that was ~3 failed metric pushes per pod per hour, plus occasional trace batches.

Both exporters now use an HTTP client that closes idle connections after 30s. A custom client makes the exporters ignore the OTLP timeout and certificate env vars; none are set.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014rfQ5HJ7zwfuYQcWXd3Mm3

CLAUDE.md

@@ -42,3 +42,4 @@ http.Handle("/", otelsetup.Handler(myHandler))
 
 - `OTEL_RESOURCE_ATTRIBUTES` - Auto-set if not provided (service.name, service.version, service.environment)
 - Standard OTLP environment variables for endpoint configuration
+- The trace and metric exporters use our own HTTP client (idle connections closed after 30s, before the collector's 1m idle timeout), so `OTEL_EXPORTER_OTLP_*TIMEOUT` and the `OTEL_EXPORTER_OTLP_*CERTIFICATE` / `*CLIENT_*` TLS variables are ignored

otel.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"time"
@@ -65,7 +66,7 @@ func SetupOTelSDK(ctx context.Context, enabled bool, serviceName, buildVersion,
 	)
 	otel.SetTextMapPropagator(prop)
 
-	traceExporter, err := otlptracehttp.New(ctx)
+	traceExporter, err := otlptracehttp.New(ctx, otlptracehttp.WithHTTPClient(otlpHTTPClient()))
 	if err != nil {
 		return handleErr(err)
 	}
@@ -88,7 +89,7 @@ func SetupOTelSDK(ctx context.Context, enabled bool, serviceName, buildVersion,
 	global.SetLoggerProvider(logProvider)
 	shutdownFuncs = append(shutdownFuncs, logProvider.Shutdown)
 
-	exp, err := otlpmetrichttp.New(ctx)
+	exp, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithHTTPClient(otlpHTTPClient()))
 	if err != nil {
 		return handleErr(err)
 	}
@@ -99,6 +100,36 @@ func SetupOTelSDK(ctx context.Context, enabled bool, serviceName, buildVersion,
 	return shutdown, err
 }
 
+// otlpHTTPClient is the OTLP exporters' default client, except that it drops idle
+// connections after 30s. The collector (Alloy's OTLP receiver, idle_timeout 1m) closes
+// connections idle for a minute, and metrics are pushed every 60s. With the exporters'
+// default of 90s, a push could reuse a connection the collector was closing. The push
+// then failed (EOF, connection reset) and was dropped: net/http doesn't retry a POST and
+// the exporters don't retry transport errors. Traces hit the same race after a quiet
+// spell. Closing first means such a push dials fresh.
+//
+// The transport mirrors the exporters' own (otlpmetrichttp/otlptracehttp v1.46.0
+// ourTransport). A custom client makes the exporters ignore OTEL_EXPORTER_OTLP_*TIMEOUT
+// and the OTEL_EXPORTER_OTLP_*CERTIFICATE/CLIENT_* TLS variables; nothing sets them.
+func otlpHTTPClient() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       30 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+		// The exporters' default.
+		Timeout: 10 * time.Second,
+	}
+}
+
 func Handler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))

otel_test.go

@@ -0,0 +1,22 @@
+package otelsetup
+
+import (
+	"net/http"
+	"testing"
+	"time"
+)
+
+func TestOTLPHTTPClient_ClosesIdleConnectionsBeforeTheCollector(t *testing.T) {
+	c := otlpHTTPClient()
+	tr, ok := c.Transport.(*http.Transport)
+	if !ok {
+		t.Fatalf("transport is %T, want *http.Transport", c.Transport)
+	}
+	// Alloy's OTLP receiver closes connections idle for 1m; the client must close first.
+	if tr.IdleConnTimeout <= 0 || tr.IdleConnTimeout >= time.Minute {
+		t.Errorf("IdleConnTimeout = %v, want in (0, 1m)", tr.IdleConnTimeout)
+	}
+	if c.Timeout != 10*time.Second {
+		t.Errorf("Timeout = %v, want 10s (the exporters' default)", c.Timeout)
+	}
+}