Skip to content

OpenTelemetry

OpenTelemetry instrumentation and tracing utilities for DiracX routers.

This module provides integration with OpenTelemetry for distributed tracing, metrics collection, and observability of API requests.

otel

Classes

OTELSettings

Bases: ServiceSettingsBase

Settings for the Open Telemetry Configuration.

Source code in diracx-routers/src/diracx/routers/otel.py 
35
36
37
38
39
40
41
42
43
44
45
46
class OTELSettings(ServiceSettingsBase):
    """Settings for the Open Telemetry Configuration."""

    model_config = SettingsConfigDict(env_prefix="DIRACX_OTEL_")

    enabled: bool = False
    application_name: str = "diracx"
    grpc_endpoint: str = ""
    grpc_insecure: bool = True
    # headers to pass to the OTEL Collector
    # e.g. {"tenant_id": "lhcbdiracx-cert"}
    headers: Optional[dict[str, str]] = None
Attributes
model_config = SettingsConfigDict(env_prefix='DIRACX_OTEL_') class-attribute instance-attribute
enabled = False class-attribute instance-attribute
application_name = 'diracx' class-attribute instance-attribute
grpc_endpoint = '' class-attribute instance-attribute
grpc_insecure = True class-attribute instance-attribute
headers = None class-attribute instance-attribute

Functions

instrument_otel(app)

Instrument the application to send OpenTelemetryData. Metrics, Traces and Logs are sent to an OTEL collector. The Collector can then redirect it to whatever is configured. Typically: Jaeger for traces, Prometheus for metrics, ElasticSearch for logs.

Note: this is highly experimental, and OpenTelemetry is a quickly moving target

Source code in diracx-routers/src/diracx/routers/otel.py 
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def instrument_otel(app: FastAPI) -> None:
    """Instrument the application to send OpenTelemetryData.
    Metrics, Traces and Logs are sent to an OTEL collector.
    The Collector can then redirect it to whatever is configured.
    Typically: Jaeger for traces, Prometheus for metrics, ElasticSearch for logs.

    Note: this is highly experimental, and OpenTelemetry is a quickly moving target

    """
    otel_settings = OTELSettings()
    if not otel_settings.enabled:
        return

    # set the service name to show in traces
    resource = Resource.create(
        attributes={
            "service.name": otel_settings.application_name,
            "service.instance.id": os.uname().nodename,
        }
    )

    # set the tracer provider
    tracer_provider = TracerProvider(resource=resource)

    # elif MODE == "otel-collector-http":
    #     tracer.add_span_processor(
    #         BatchSpanProcessor(OTLPSpanExporterHTTP(endpoint=OTEL_HTTP_ENDPOINT))
    #     )
    # else:
    # default otel-collector-grpc
    tracer_provider.add_span_processor(
        BatchSpanProcessor(
            OTLPSpanExporter(
                endpoint=otel_settings.grpc_endpoint,
                insecure=otel_settings.grpc_insecure,
                headers=otel_settings.headers,
            )
        )
    )
    trace.set_tracer_provider(tracer_provider)
    # http_exporter = httpOTPLMetricExporter()
    # metric_reader = PeriodicExportingMetricReader(ConsoleMetricExporter(),export_interval_millis=1000)
    metric_reader = PeriodicExportingMetricReader(
        OTLPMetricExporter(
            endpoint=otel_settings.grpc_endpoint,
            insecure=otel_settings.grpc_insecure,
            headers=otel_settings.headers,
        ),
        export_interval_millis=3000,
    )
    meter_provider = MeterProvider(metric_readers=[metric_reader], resource=resource)
    metrics.set_meter_provider(meter_provider)

    ###################################

    # # override logger format which with trace id and span id
    # https://github.com/mhausenblas/ref.otel.help/blob/main/how-to/logs-collection/yoda/main.py

    LoggingInstrumentor().instrument(set_logging_format=False)

    logger_provider = LoggerProvider(resource=resource)
    _logs.set_logger_provider(logger_provider)

    otlp_exporter = OTLPLogExporter(
        endpoint=otel_settings.grpc_endpoint,
        insecure=otel_settings.grpc_insecure,
        headers=otel_settings.headers,
    )
    logger_provider.add_log_record_processor(BatchLogRecordProcessor(otlp_exporter))
    handler = LoggingHandler(level=logging.DEBUG, logger_provider=logger_provider)
    handler.setFormatter(logging.Formatter(DEFAULT_LOGGING_FORMAT))
    # Add the handler to diracx and all uvicorn logger
    # Note adding it to just 'uvicorn' or the root logger
    # is not enough because uvicorn sets propagate=False
    for logger_name in logging.root.manager.loggerDict:
        if "diracx" == logger_name or "uvicorn" in logger_name:
            logging.getLogger(logger_name).addHandler(handler)

    ####################

    FastAPIInstrumentor.instrument_app(
        app, tracer_provider=tracer_provider, meter_provider=meter_provider
    )