Chi | Last9 Documentation

Instrument your Go Chi router application with OpenTelemetry to send comprehensive telemetry data to Last9. This integration provides automatic instrumentation for HTTP requests, middleware, database operations, and external API calls for high-performance Go web applications.

Prerequisites

Go 1.19 or higher
Chi router v5.0 or higher in your application
Last9 account with OTLP endpoint configured

Installation

Install the required OpenTelemetry packages for Chi instrumentation:

# Core OpenTelemetry packages
go get go.opentelemetry.io/otel@v1.30.0
go get go.opentelemetry.io/otel/sdk@v1.30.0
go get go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.30.0
go get go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc@v1.30.0
go get go.opentelemetry.io/otel/sdk/metric@v1.30.0

# Chi instrumentation
go get github.com/go-chi/chi/v5@v5.1.0
go get github.com/riandyrn/otelchi@v0.8.0

# HTTP client instrumentation
go get go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.55.0
go get go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace@v0.55.0

# Database and cache instrumentation
go get go.nhat.io/otelsql@v0.14.0
go get github.com/redis/go-redis/extra/redisotel/v9@v9.11.0

Configuration

Set Environment Variables

Configure the required environment variables for Last9 OTLP integration:

export OTEL_SERVICE_NAME="your-chi-service"
export OTEL_EXPORTER_OTLP_ENDPOINT="$last9_otlp_endpoint"
export OTEL_EXPORTER_OTLP_HEADERS="Authorization=$last9_otlp_auth_header"
export OTEL_TRACES_SAMPLER="always_on"
export OTEL_RESOURCE_ATTRIBUTES="deployment.environment=production,service.version=1.0.0"
export OTEL_LOG_LEVEL="error"

Create Instrumentation Package

Create pkg/instrumentation/otel.go:

package instrumentation

import (
    "context"
    "fmt"
    "log"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
    "go.opentelemetry.io/otel/trace"
)

// Instrumentation holds OpenTelemetry components
type Instrumentation struct {
    TracerProvider *sdktrace.TracerProvider
    MeterProvider  *metric.MeterProvider
    Tracer         trace.Tracer
    Shutdown       func(context.Context) error
}

// NewInstrumentation creates a new instrumentation instance
func NewInstrumentation(serviceName string) (*Instrumentation, error) {
    // Create resource with service information
    res, err := resource.New(context.Background(),
        resource.WithFromEnv(),
        resource.WithTelemetrySDK(),
        resource.WithProcess(),
        resource.WithOS(),
        resource.WithContainer(),
        resource.WithHost(),
        resource.WithAttributes(
            semconv.ServiceNameKey.String(serviceName),
            semconv.ServiceVersionKey.String("1.0.0"),
        ),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create resource: %w", err)
    }

    // Initialize trace provider
    tp, err := initTraceProvider(res)
    if err != nil {
        return nil, fmt.Errorf("failed to initialize trace provider: %w", err)
    }

    // Initialize metric provider
    mp, err := initMetricProvider(res)
    if err != nil {
        return nil, fmt.Errorf("failed to initialize metric provider: %w", err)
    }

    // Set global providers
    otel.SetTracerProvider(tp)
    otel.SetMeterProvider(mp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    return &Instrumentation{
        TracerProvider: tp,
        MeterProvider:  mp,
        Tracer:         tp.Tracer(serviceName),
        Shutdown: func(ctx context.Context) error {
            var err error
            if shutdownErr := tp.Shutdown(ctx); shutdownErr != nil {
                err = shutdownErr
            }
            if shutdownErr := mp.Shutdown(ctx); shutdownErr != nil {
                if err != nil {
                    err = fmt.Errorf("%v; %v", err, shutdownErr)
                } else {
                    err = shutdownErr
                }
            }
            return err
        },
    }, nil
}

func initTraceProvider(res *resource.Resource) (*sdktrace.TracerProvider, error) {
    exporter, err := otlptracehttp.New(context.Background())
    if err != nil {
        return nil, fmt.Errorf("failed to create trace exporter: %w", err)
    }

    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(sdktrace.AlwaysSample()),
    )

    return tp, nil
}

func initMetricProvider(res *resource.Resource) (*metric.MeterProvider, error) {
    exporter, err := otlpmetricgrpc.New(context.Background())
    if err != nil {
        return nil, fmt.Errorf("failed to create metric exporter: %w", err)
    }

    mp := metric.NewMeterProvider(
        metric.WithResource(res),
        metric.WithReader(metric.NewPeriodicReader(
            exporter,
            metric.WithInterval(1*time.Minute),
        )),
    )

    return mp, nil
}

Create Main Application

Update your main.go:

package main

import (
    "context"
    "database/sql"
    "fmt"
    "log"
    "net/http"
    "os"
    "os/signal"
    "syscall"
    "time"

    "github.com/go-chi/chi/v5"
    "github.com/go-chi/chi/v5/middleware"
    "github.com/riandyrn/otelchi"
    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
    "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace"

    // Import your instrumentation package
    "your-app/pkg/instrumentation"
)

func main() {
    serviceName := os.Getenv("OTEL_SERVICE_NAME")
    if serviceName == "" {
        serviceName = "chi-api"
    }

    // Initialize OpenTelemetry
    otel, err := instrumentation.NewInstrumentation(serviceName)
    if err != nil {
        log.Fatalf("Failed to initialize OpenTelemetry: %v", err)
    }

    // Ensure proper shutdown
    defer func() {
        ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
        defer cancel()
        if err := otel.Shutdown(ctx); err != nil {
            log.Printf("Error shutting down OpenTelemetry: %v", err)
        }
    }()

    // Create Chi router
    r := chi.NewRouter()

    // Add Chi middleware
    r.Use(middleware.RequestID)
    r.Use(middleware.RealIP)
    r.Use(middleware.Logger)
    r.Use(middleware.Recoverer)
    r.Use(middleware.Timeout(60 * time.Second))

    // Add OpenTelemetry middleware
    r.Use(otelchi.Middleware(serviceName, otelchi.WithChiRoutes(r)))

    // Setup routes
    setupRoutes(r, otel)

    // Start HTTP server with graceful shutdown
    server := &http.Server{
        Addr:    ":8080",
        Handler: r,
    }

    // Handle graceful shutdown
    go func() {
        sigChan := make(chan os.Signal, 1)
        signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
        <-sigChan

        log.Println("Shutting down server...")
        ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
        defer cancel()

        if err := server.Shutdown(ctx); err != nil {
            log.Printf("Server forced to shutdown: %v", err)
        }
    }()

    log.Printf("Starting Chi server on port 8080")
    if err := server.ListenAndServe(); err != http.ErrServerClosed {
        log.Fatalf("Server failed to start: %v", err)
    }
}

func setupRoutes(r *chi.Mux, otel *instrumentation.Instrumentation) {
    // Health check route
    r.Get("/health", func(w http.ResponseWriter, r *http.Request) {
        w.Header().Set("Content-Type", "application/json")
        w.Write([]byte(`{"status": "healthy", "service": "chi-api"}`))
    })

    // API routes
    r.Route("/api/v1", func(r chi.Router) {
        r.Get("/users", getUsersHandler(otel))
        r.Get("/users/{id}", getUserHandler(otel))
        r.Post("/users", createUserHandler(otel))
        r.Put("/users/{id}", updateUserHandler(otel))
        r.Delete("/users/{id}", deleteUserHandler(otel))
    })
}

Create Route Handlers with Custom Tracing

Create handlers.go:

package main

import (
    "context"
    "encoding/json"
    "net/http"
    "strconv"
    "time"

    "github.com/go-chi/chi/v5"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/codes"
    "go.opentelemetry.io/otel/trace"

    "your-app/pkg/instrumentation"
)

type User struct {
    ID    int    `json:"id"`
    Name  string `json:"name"`
    Email string `json:"email"`
}

func getUsersHandler(otel *instrumentation.Instrumentation) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        ctx := r.Context()
        span := trace.SpanFromContext(ctx)

        span.SetAttributes(
            attribute.String("handler.name", "get-users"),
            attribute.String("operation.type", "read"),
        )

        // Simulate business logic with custom tracing
        users, err := fetchUsers(ctx, otel.Tracer)
        if err != nil {
            span.RecordError(err)
            span.SetStatus(codes.Error, err.Error())
            http.Error(w, err.Error(), http.StatusInternalServerError)
            return
        }

        span.SetAttributes(
            attribute.Int("users.count", len(users)),
            attribute.Bool("operation.success", true),
        )

        w.Header().Set("Content-Type", "application/json")
        json.NewEncoder(w).Encode(map[string]interface{}{
            "users": users,
            "count": len(users),
        })
    }
}

func getUserHandler(otel *instrumentation.Instrumentation) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        ctx := r.Context()
        span := trace.SpanFromContext(ctx)

        userIDStr := chi.URLParam(r, "id")
        userID, err := strconv.Atoi(userIDStr)
        if err != nil {
            span.RecordError(err)
            span.SetStatus(codes.Error, "invalid user ID")
            http.Error(w, "Invalid user ID", http.StatusBadRequest)
            return
        }

        span.SetAttributes(
            attribute.String("handler.name", "get-user"),
            attribute.Int("user.id", userID),
            attribute.String("operation.type", "read"),
        )

        user, err := fetchUserByID(ctx, otel.Tracer, userID)
        if err != nil {
            span.RecordError(err)
            span.SetStatus(codes.Error, err.Error())
            http.Error(w, err.Error(), http.StatusNotFound)
            return
        }

        span.SetAttributes(attribute.Bool("user.found", user != nil))

        w.Header().Set("Content-Type", "application/json")
        json.NewEncoder(w).Encode(user)
    }
}

func createUserHandler(otel *instrumentation.Instrumentation) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        ctx := r.Context()
        span := trace.SpanFromContext(ctx)

        span.SetAttributes(
            attribute.String("handler.name", "create-user"),
            attribute.String("operation.type", "create"),
        )

        var user User
        if err := json.NewDecoder(r.Body).Decode(&user); err != nil {
            span.RecordError(err)
            span.SetStatus(codes.Error, "invalid request body")
            http.Error(w, "Invalid request body", http.StatusBadRequest)
            return
        }

        span.SetAttributes(
            attribute.String("user.name", user.Name),
            attribute.String("user.email", user.Email),
        )

        newUser, err := createUser(ctx, otel.Tracer, user)
        if err != nil {
            span.RecordError(err)
            span.SetStatus(codes.Error, err.Error())
            http.Error(w, err.Error(), http.StatusInternalServerError)
            return
        }

        span.SetAttributes(
            attribute.Int("user.id", newUser.ID),
            attribute.Bool("operation.success", true),
        )

        w.Header().Set("Content-Type", "application/json")
        w.WriteHeader(http.StatusCreated)
        json.NewEncoder(w).Encode(newUser)
    }
}

// Business logic functions with custom tracing
func fetchUsers(ctx context.Context, tracer trace.Tracer) ([]User, error) {
    _, span := tracer.Start(ctx, "fetch-users-from-db")
    defer span.End()

    span.SetAttributes(
        attribute.String("db.operation", "SELECT"),
        attribute.String("db.table", "users"),
    )

    // Simulate database operation
    time.Sleep(50 * time.Millisecond)

    users := []User{
        {ID: 1, Name: "John Doe", Email: "john@example.com"},
        {ID: 2, Name: "Jane Smith", Email: "jane@example.com"},
    }

    span.SetAttributes(
        attribute.Int("db.rows_returned", len(users)),
        attribute.Bool("operation.success", true),
    )

    return users, nil
}

func fetchUserByID(ctx context.Context, tracer trace.Tracer, userID int) (*User, error) {
    _, span := tracer.Start(ctx, "fetch-user-by-id")
    defer span.End()

    span.SetAttributes(
        attribute.String("db.operation", "SELECT"),
        attribute.String("db.table", "users"),
        attribute.Int("user.id", userID),
    )

    // Simulate database operation
    time.Sleep(30 * time.Millisecond)

    if userID == 404 {
        span.SetAttributes(attribute.Bool("user.found", false))
        return nil, fmt.Errorf("user not found")
    }

    user := &User{
        ID:    userID,
        Name:  fmt.Sprintf("User %d", userID),
        Email: fmt.Sprintf("user%d@example.com", userID),
    }

    span.SetAttributes(attribute.Bool("user.found", true))
    return user, nil
}

func createUser(ctx context.Context, tracer trace.Tracer, user User) (*User, error) {
    _, span := tracer.Start(ctx, "create-user-in-db")
    defer span.End()

    span.SetAttributes(
        attribute.String("db.operation", "INSERT"),
        attribute.String("db.table", "users"),
        attribute.String("user.name", user.Name),
        attribute.String("user.email", user.Email),
    )

    // Simulate database operation
    time.Sleep(100 * time.Millisecond)

    user.ID = int(time.Now().Unix()) // Simulate ID generation

    span.SetAttributes(
        attribute.Int("user.id", user.ID),
        attribute.Bool("operation.success", true),
    )

    return &user, nil
}

Database Integration

PostgreSQL with SQL Driver

package database

import (
    "database/sql"
    "fmt"
    "os"

    "go.nhat.io/otelsql"
    semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
    _ "github.com/lib/pq"
)

func InitDatabase() (*sql.DB, error) {
    // Register the instrumented driver
    driverName, err := otelsql.Register("postgres",
        otelsql.AllowRoot(),
        otelsql.TraceQueryWithoutArgs(),
        otelsql.TraceRowsClose(),
        otelsql.TraceRowsAffected(),
        otelsql.WithDatabaseName("your-database"),
        otelsql.WithSystem(semconv.DBSystemPostgreSQL),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to register otelsql driver: %w", err)
    }

    // Connect to database
    db, err := sql.Open(driverName, os.Getenv("DATABASE_URL"))
    if err != nil {
        return nil, fmt.Errorf("failed to open database: %w", err)
    }

    // Configure connection pool
    db.SetMaxOpenConns(25)
    db.SetMaxIdleConns(25)
    db.SetConnMaxLifetime(5 * time.Minute)

    // Record connection pool stats
    if err := otelsql.RecordStats(db); err != nil {
        return nil, fmt.Errorf("failed to record database stats: %w", err)
    }

    return db, nil
}

Redis Integration

package cache

import (
    "fmt"

    "github.com/redis/go-redis/v9"
    "github.com/redis/go-redis/extra/redisotel/v9"
)

func InitRedis() (*redis.Client, error) {
    rdb := redis.NewClient(&redis.Options{
        Addr:     "localhost:6379",
        Password: os.Getenv("REDIS_PASSWORD"),
        DB:       0,
    })

    // Enable tracing
    if err := redisotel.InstrumentTracing(rdb); err != nil {
        return nil, fmt.Errorf("failed to instrument Redis tracing: %w", err)
    }

    // Enable metrics
    if err := redisotel.InstrumentMetrics(rdb); err != nil {
        return nil, fmt.Errorf("failed to instrument Redis metrics: %w", err)
    }

    return rdb, nil
}

HTTP Client Instrumentation

package httpclient

import (
    "context"
    "net/http"
    "net/http/httptrace"

    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
    "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace"
)

func NewInstrumentedClient() *http.Client {
    return &http.Client{
        Transport: otelhttp.NewTransport(
            http.DefaultTransport,
            otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace {
                return otelhttptrace.NewClientTrace(ctx)
            }),
        ),
    }
}

// Usage example
func makeAPICall(ctx context.Context, client *http.Client) error {
    req, err := http.NewRequestWithContext(ctx, "GET", "https://api.example.com/data", nil)
    if err != nil {
        return err
    }

    resp, err := client.Do(req)
    if err != nil {
        return err
    }
    defer resp.Body.Close()

    return nil
}

Production Deployment

Docker Configuration

# Dockerfile
FROM golang:1.21-alpine AS builder

WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download

COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o main .

FROM alpine:latest
RUN apk --no-cache add ca-certificates
WORKDIR /root/

COPY --from=builder /app/main .

# Set OpenTelemetry environment variables
ENV OTEL_SERVICE_NAME=chi-docker-app
ENV OTEL_EXPORTER_OTLP_ENDPOINT=$last9_otlp_endpoint
ENV OTEL_EXPORTER_OTLP_HEADERS="Authorization=$last9_otlp_auth_header"
ENV OTEL_RESOURCE_ATTRIBUTES="deployment.environment=docker,service.version=1.0.0"

EXPOSE 8080
CMD ["./main"]

Kubernetes Deployment

# kubernetes-chi.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: chi-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: chi-app
  template:
    metadata:
      labels:
        app: chi-app
    spec:
      containers:
        - name: chi-app
          image: your-registry/chi-app:latest
          ports:
            - containerPort: 8080
          env:
            - name: OTEL_SERVICE_NAME
              value: "chi-k8s-app"
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              valueFrom:
                secretKeyRef:
                  name: last9-credentials
                  key: endpoint
            - name: OTEL_EXPORTER_OTLP_HEADERS
              valueFrom:
                secretKeyRef:
                  name: last9-credentials
                  key: auth-header
            - name: OTEL_RESOURCE_ATTRIBUTES
              value: "deployment.environment=kubernetes,service.version=1.0.0"
          resources:
            requests:
              memory: "64Mi"
              cpu: "50m"
            limits:
              memory: "256Mi"
              cpu: "200m"
          livenessProbe:
            httpGet:
              path: /health
              port: 8080
          readinessProbe:
            httpGet:
              path: /health
              port: 8080

---
apiVersion: v1
kind: Service
metadata:
  name: chi-service
spec:
  selector:
    app: chi-app
  ports:
    - protocol: TCP
      port: 80
      targetPort: 8080
  type: LoadBalancer

Troubleshooting

Common Issues

No traces appearing:
- Verify environment variables are correctly set
- Check network connectivity to Last9 OTLP endpoint
- Enable debug logging
Missing spans:
- Ensure otelchi middleware is added before routes
- Verify instrumentation packages are imported
Performance impact:
- Use sampling: export OTEL_TRACES_SAMPLER_ARG=0.1
- Monitor memory usage
- Consider batch processing options

Debug Mode

Enable OpenTelemetry debugging:

import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/log/global"
)

// Enable debug logging
otel.SetLogger(global.GetLoggerProvider().Logger("debug"))

Monitoring Capabilities

This integration captures:

HTTP Requests: All Chi router operations and middleware
Route Performance: Individual handler execution times
Database Operations: SQL queries and connection pool metrics
Cache Operations: Redis commands and performance
External API Calls: Outbound HTTP requests with timing
Custom Business Logic: Through manual instrumentation
Error Tracking: Detailed exception information

Your Chi router application will now provide comprehensive telemetry data to Last9, enabling detailed performance monitoring and debugging capabilities for high-performance Go web applications.