Add telemetry

cmd/gemfast-server/start.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"os"
 	"time"
 
@@ -10,10 +11,14 @@ import (
 	"github.com/gemfast/server/internal/db"
 	"github.com/gemfast/server/internal/filter"
 	"github.com/gemfast/server/internal/indexer"
+	"github.com/gemfast/server/internal/telemetry"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 )
 
+// serviceVersion is overridable at link time (-ldflags "-X .../cmd.serviceVersion=v1.2.3").
+var serviceVersion = "dev"
+
 var startCmd = &cobra.Command{
 	Use:   "start",
 	Short: "Starts the gemfast rubygems server",
@@ -41,6 +46,19 @@ func start() {
 	cfg := config.NewConfig()
 	log.Info().Msg("starting services")
 
+	// Initialize OpenTelemetry tracing (exports to Honeycomb by default).
+	ctx := context.Background()
+	shutdownTracer, err := telemetry.Init(ctx, serviceVersion, cfg)
+	if err != nil {
+		log.Warn().Err(err).Msg("failed to initialize tracing; continuing without it")
+	} else {
+		defer func() {
+			if err := shutdownTracer(context.Background()); err != nil {
+				log.Warn().Err(err).Msg("error shutting down tracer provider")
+			}
+		}()
+	}
+
 	// Connect to the database
 	database, err := db.NewDB(cfg)
 	if err != nil {

deploy/otel-collector/README.md

@@ -0,0 +1,39 @@
+# Local OTel Collector for gemfast
+
+Routes gemfast-server traces through an OpenTelemetry Collector before they
+reach Honeycomb, applying two transforms:
+
+1. **Attribute enrichment** — every span gets `collector.processed=true` and
+   `collector.deployment_env=<DEPLOYMENT_ENV>`.
+2. **Tail sampling** — full traces are buffered for 5s, then:
+   - **Always kept**: traces with any error-status span, any span over 500ms,
+     and all gem upload traces (`gem.action=upload`).
+   - **Probabilistically kept**: 10% of remaining healthy traces.
+
+## Run
+
+```bash
+cd deploy/otel-collector
+HONEYCOMB_API_KEY=... DEPLOYMENT_ENV=local docker compose up -d
+```
+
+Then point gemfast-server at the local collector instead of Honeycomb:
+
+```bash
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
+  ./gemfast-server start -c test/fixtures/gemfast-local.hcl
+```
+
+To verify the transform fired: in Honeycomb, run
+`COUNT WHERE collector.processed = true` — pre-collector traces won't have the
+attribute.
+
+To verify the sampling fired: compare `COUNT` of `GET /up` (cheap, healthy,
+high-volume) against `COUNT` of `gem.action=upload` over the same window. The
+upload traces should be kept 1:1; the `/up` traces should drop ~90%.
+
+## Stop
+
+```bash
+docker compose down
+```

deploy/otel-collector/collector.yaml

@@ -0,0 +1,67 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  # Transform #1: enrich every span with a marker proving traffic went through
+  # the collector and tag the pipeline's deployment env.
+  attributes/enrich:
+    actions:
+      - key: collector.processed
+        value: true
+        action: insert
+      - key: collector.deployment_env
+        value: ${env:DEPLOYMENT_ENV}
+        action: insert
+
+  # Transform #2: tail-based sampling — buffer complete traces, then keep
+  # interesting ones (errors + slow) and sample healthy ones at 10%.
+  tail_sampling:
+    decision_wait: 5s
+    num_traces: 50000
+    expected_new_traces_per_sec: 100
+    policies:
+      - name: keep-errors
+        type: status_code
+        status_code: { status_codes: [ERROR] }
+      - name: keep-slow-traces
+        type: latency
+        latency: { threshold_ms: 500 }
+      - name: keep-uploads
+        type: string_attribute
+        string_attribute:
+          key: gem.action
+          values: ["upload"]
+      - name: sample-healthy
+        type: probabilistic
+        probabilistic: { sampling_percentage: 10 }
+
+  batch:
+    send_batch_size: 512
+    timeout: 1s
+
+exporters:
+  otlphttp/honeycomb:
+    endpoint: https://api.honeycomb.io
+    headers:
+      x-honeycomb-team: ${env:HONEYCOMB_API_KEY}
+
+  # Mirror to stdout for local debugging.
+  debug:
+    verbosity: basic
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [attributes/enrich, tail_sampling, batch]
+      exporters: [otlphttp/honeycomb, debug]

deploy/otel-collector/docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.116.1
+    container_name: gemfast-otel-collector
+    command: ["--config=/etc/otel/collector.yaml"]
+    volumes:
+      - ./collector.yaml:/etc/otel/collector.yaml:ro
+    environment:
+      HONEYCOMB_API_KEY: ${HONEYCOMB_API_KEY:?HONEYCOMB_API_KEY must be set (e.g. in .env)}
+      DEPLOYMENT_ENV: ${DEPLOYMENT_ENV:-local}
+    ports:
+      - "4317:4317" # OTLP/gRPC
+      - "4318:4318" # OTLP/HTTP
+      - "13133:13133" # health_check

go.mod

@@ -1,6 +1,6 @@
 module github.com/gemfast/server
 
-go 1.24.3
+go 1.25.0
 
 require (
 	github.com/akyoto/cache v1.0.6
@@ -9,19 +9,25 @@ require (
 	github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce
 	github.com/casbin/casbin/v2 v2.109.0
 	github.com/gin-contrib/sessions v1.0.4
-	github.com/gin-gonic/gin v1.10.1
+	github.com/gin-gonic/gin v1.12.0
 	github.com/go-git/go-git/v5 v5.16.2
 	github.com/golang-jwt/jwt/v4 v4.5.2
 	github.com/hashicorp/hcl/v2 v2.24.0
 	github.com/juliangruber/go-intersect v1.1.0
 	github.com/rs/zerolog v1.34.0
 	github.com/sethvargo/go-password v0.3.1
 	github.com/spf13/cobra v1.9.1
-	github.com/stretchr/testify v1.10.0
+	github.com/stretchr/testify v1.11.1
 	github.com/tidwall/gjson v1.18.0
 	github.com/zsais/go-gin-prometheus v1.0.0
 	go.etcd.io/bbolt v1.4.2
-	golang.org/x/crypto v0.45.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.69.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0
+	go.opentelemetry.io/otel v1.44.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0
+	go.opentelemetry.io/otel/sdk v1.44.0
+	go.opentelemetry.io/otel/trace v1.44.0
+	golang.org/x/crypto v0.52.0
 	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -35,68 +41,84 @@ require (
 	github.com/aquasecurity/go-version v0.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect
-	github.com/bytedance/sonic v1.13.3 // indirect
-	github.com/bytedance/sonic/loader v0.3.0 // indirect
+	github.com/bytedance/gopkg v0.1.4 // indirect
+	github.com/bytedance/sonic v1.15.1 // indirect
+	github.com/bytedance/sonic/loader v0.5.1 // indirect
 	github.com/casbin/govaluate v1.8.0 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.6.1 // indirect
-	github.com/cloudwego/base64x v0.1.5 // indirect
+	github.com/cloudwego/base64x v0.1.7 // indirect
 	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/fatih/color v1.9.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
-	github.com/gin-contrib/sse v1.1.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
+	github.com/gin-contrib/sse v1.1.1 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.27.0 // indirect
-	github.com/goccy/go-json v0.10.5 // indirect
-	github.com/goccy/go-yaml v1.8.1 // indirect
+	github.com/go-playground/validator/v10 v10.30.2 // indirect
+	github.com/goccy/go-json v0.10.6 // indirect
+	github.com/goccy/go-yaml v1.19.2 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/context v1.1.2 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/sessions v1.4.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-isatty v0.0.22 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/pelletier/go-toml/v2 v2.3.1 // indirect
 	github.com/pjbgf/sha1cd v0.4.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.22.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect
+	github.com/quic-go/qpack v0.6.0 // indirect
+	github.com/quic-go/quic-go v0.59.1 // indirect
 	github.com/sergi/go-diff v1.4.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.3.1 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/ugorji/go/codec v1.3.0 // indirect
+	github.com/ugorji/go/codec v1.3.1 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	github.com/zclconf/go-cty v1.16.3 // indirect
-	golang.org/x/arch v0.19.0 // indirect
-	golang.org/x/mod v0.29.0 // indirect
-	golang.org/x/net v0.47.0 // indirect
-	golang.org/x/sync v0.18.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
-	golang.org/x/tools v0.38.0 // indirect
+	go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
+	golang.org/x/arch v0.27.0 // indirect
+	golang.org/x/mod v0.35.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/text v0.37.0 // indirect
+	golang.org/x/tools v0.44.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/protobuf v1.36.6 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/grpc v1.81.1 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )