fix: close idle OTLP connections before the collector does (#184)

otel.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"time"
@@ -65,7 +66,7 @@ func SetupOTelSDK(ctx context.Context, enabled bool, serviceName, buildVersion,
 	)
 	otel.SetTextMapPropagator(prop)
 
-	traceExporter, err := otlptracehttp.New(ctx)
+	traceExporter, err := otlptracehttp.New(ctx, otlptracehttp.WithHTTPClient(otlpHTTPClient()))
 	if err != nil {
 		return handleErr(err)
 	}
@@ -88,7 +89,7 @@ func SetupOTelSDK(ctx context.Context, enabled bool, serviceName, buildVersion,
 	global.SetLoggerProvider(logProvider)
 	shutdownFuncs = append(shutdownFuncs, logProvider.Shutdown)
 
-	exp, err := otlpmetrichttp.New(ctx)
+	exp, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithHTTPClient(otlpHTTPClient()))
 	if err != nil {
 		return handleErr(err)
 	}
@@ -99,6 +100,36 @@ func SetupOTelSDK(ctx context.Context, enabled bool, serviceName, buildVersion,
 	return shutdown, err
 }
 
+// otlpHTTPClient is the OTLP exporters' default client, except that it drops idle
+// connections after 30s. The collector (Alloy's OTLP receiver, idle_timeout 1m) closes
+// connections idle for a minute, and metrics are pushed every 60s. With the exporters'
+// default of 90s, a push could reuse a connection the collector was closing. The push
+// then failed (EOF, connection reset) and was dropped: net/http doesn't retry a POST and
+// the exporters don't retry transport errors. Traces hit the same race after a quiet
+// spell. Closing first means such a push dials fresh.
+//
+// The transport mirrors the exporters' own (otlpmetrichttp/otlptracehttp v1.46.0
+// ourTransport). A custom client makes the exporters ignore OTEL_EXPORTER_OTLP_*TIMEOUT
+// and the OTEL_EXPORTER_OTLP_*CERTIFICATE/CLIENT_* TLS variables; nothing sets them.
+func otlpHTTPClient() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       30 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+		// The exporters' default.
+		Timeout: 10 * time.Second,
+	}
+}
+
 func Handler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))