From 1b3a21e6f82a290ae006dc4ff8740aecc1bef621 Mon Sep 17 00:00:00 2001 From: GareArc Date: Fri, 6 Feb 2026 03:06:06 -0800 Subject: [PATCH 1/4] feat(telemetry): unify token metric label structure with Pydantic enforcement - Add TokenMetricLabels BaseModel to enforce consistent label structure - All dify.token.* metrics now use identical 6-label structure: * tenant_id, app_id, operation_type, model_provider, model_name, node_type - Pydantic validation ensures runtime enforcement (extra='forbid', frozen=True) - Enables filtering by operation_type to avoid double-counting: * workflow: aggregated workflow-level tokens * node_execution: individual node-level tokens * message: direct message tokens * rule_generate/code_generate: prompt generation tokens Previously, inconsistent label cardinality made aggregation impossible: - WORKFLOW: 3 labels - NODE_EXECUTION: 6 labels - MESSAGE: 5 labels - PROMPT_GENERATION: 5 labels Now all use the same 6-label structure for consistent querying. --- api/enterprise/telemetry/enterprise_trace.py | 68 +++++++++++++++---- api/enterprise/telemetry/entities/__init__.py | 66 ++++++++++++++++++ 2 files changed, 121 insertions(+), 13 deletions(-) diff --git a/api/enterprise/telemetry/enterprise_trace.py b/api/enterprise/telemetry/enterprise_trace.py index 8eb97fd52c..38919ae290 100644 --- a/api/enterprise/telemetry/enterprise_trace.py +++ b/api/enterprise/telemetry/enterprise_trace.py @@ -6,6 +6,25 @@ Only requires a matching ``trace(trace_info)`` method signature. Signal strategy: - **Traces (spans)**: workflow run, node execution, draft node execution only. - **Metrics + structured logs**: all other event types. + +Token metric labels (unified structure): +All token metrics (dify.tokens.input, dify.tokens.output, dify.tokens.total) use the +same label set for consistent filtering and aggregation: +- tenant_id: Tenant identifier +- app_id: Application identifier +- operation_type: Source of token usage (workflow | node_execution | message | rule_generate | etc.) +- model_provider: LLM provider name (empty string if not applicable) +- model_name: LLM model name (empty string if not applicable) +- node_type: Workflow node type (empty string if not node_execution) + +This unified structure allows filtering by operation_type to separate: +- Workflow-level aggregates (operation_type=workflow) +- Individual node executions (operation_type=node_execution) +- Direct message calls (operation_type=message) +- Prompt generation operations (operation_type=rule_generate, code_generate, etc.) + +Without this, tokens are double-counted when querying totals (workflow totals include +node totals, since workflow.total_tokens is the sum of all node tokens). """ from __future__ import annotations @@ -35,6 +54,7 @@ from enterprise.telemetry.entities import ( EnterpriseTelemetryEvent, EnterpriseTelemetryHistogram, EnterpriseTelemetrySpan, + TokenMetricLabels, ) from enterprise.telemetry.telemetry_log import emit_metric_only_event, emit_telemetry_log @@ -218,10 +238,14 @@ class EnterpriseOtelTrace: tenant_id=tenant_id or "", app_id=app_id or "", ) - token_labels = self._labels( - **labels, + token_labels = TokenMetricLabels( + tenant_id=tenant_id or "", + app_id=app_id or "", operation_type=OperationType.WORKFLOW, - ) + model_provider="", + model_name="", + node_type="", + ).to_dict() self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, token_labels) if info.prompt_tokens is not None and info.prompt_tokens > 0: self._exporter.increment_counter(EnterpriseTelemetryCounter.INPUT_TOKENS, info.prompt_tokens, token_labels) @@ -370,11 +394,14 @@ class EnterpriseOtelTrace: model_provider=info.model_provider or "", ) if info.total_tokens: - token_labels = self._labels( - **labels, - model_name=info.model_name or "", + token_labels = TokenMetricLabels( + tenant_id=tenant_id or "", + app_id=app_id or "", operation_type=OperationType.NODE_EXECUTION, - ) + model_provider=info.model_provider or "", + model_name=info.model_name or "", + node_type=info.node_type, + ).to_dict() self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, token_labels) if info.prompt_tokens is not None and info.prompt_tokens > 0: self._exporter.increment_counter( @@ -463,10 +490,14 @@ class EnterpriseOtelTrace: model_provider=metadata.get("ls_provider", ""), model_name=metadata.get("ls_model_name", ""), ) - token_labels = self._labels( - **labels, + token_labels = TokenMetricLabels( + tenant_id=tenant_id or "", + app_id=app_id or "", operation_type=OperationType.MESSAGE, - ) + model_provider=metadata.get("ls_provider", ""), + model_name=metadata.get("ls_model_name", ""), + node_type="", + ).to_dict() self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, token_labels) if info.message_tokens > 0: self._exporter.increment_counter(EnterpriseTelemetryCounter.INPUT_TOKENS, info.message_tokens, token_labels) @@ -819,6 +850,15 @@ class EnterpriseOtelTrace: user_id=user_id, ) + token_labels = TokenMetricLabels( + tenant_id=tenant_id or "", + app_id=app_id or "", + operation_type=info.operation_type, + model_provider=info.model_provider, + model_name=info.model_name, + node_type="", + ).to_dict() + labels = self._labels( tenant_id=tenant_id or "", app_id=app_id or "", @@ -827,11 +867,13 @@ class EnterpriseOtelTrace: model_name=info.model_name, ) - self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, labels) + self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, token_labels) if info.prompt_tokens > 0: - self._exporter.increment_counter(EnterpriseTelemetryCounter.INPUT_TOKENS, info.prompt_tokens, labels) + self._exporter.increment_counter(EnterpriseTelemetryCounter.INPUT_TOKENS, info.prompt_tokens, token_labels) if info.completion_tokens > 0: - self._exporter.increment_counter(EnterpriseTelemetryCounter.OUTPUT_TOKENS, info.completion_tokens, labels) + self._exporter.increment_counter( + EnterpriseTelemetryCounter.OUTPUT_TOKENS, info.completion_tokens, token_labels + ) status = "failed" if info.error else "success" self._exporter.increment_counter( diff --git a/api/enterprise/telemetry/entities/__init__.py b/api/enterprise/telemetry/entities/__init__.py index 388bdc8fa2..4a9bd3dbf8 100644 --- a/api/enterprise/telemetry/entities/__init__.py +++ b/api/enterprise/telemetry/entities/__init__.py @@ -1,4 +1,8 @@ from enum import StrEnum +from typing import cast + +from opentelemetry.util.types import AttributeValue +from pydantic import BaseModel, ConfigDict class EnterpriseTelemetrySpan(StrEnum): @@ -47,9 +51,71 @@ class EnterpriseTelemetryHistogram(StrEnum): PROMPT_GENERATION_DURATION = "prompt_generation_duration" +class TokenMetricLabels(BaseModel): + """Unified label structure for all dify.token.* metrics. + + All token counters (dify.tokens.input, dify.tokens.output, dify.tokens.total) MUST + use this exact label set to ensure consistent filtering and aggregation across + different operation types. + + Attributes: + tenant_id: Tenant identifier. + app_id: Application identifier. + operation_type: Source of token usage (workflow | node_execution | message | + rule_generate | code_generate | structured_output | instruction_modify). + model_provider: LLM provider name. Empty string if not applicable (e.g., workflow-level). + model_name: LLM model name. Empty string if not applicable (e.g., workflow-level). + node_type: Workflow node type. Empty string unless operation_type=node_execution. + + Usage: + labels = TokenMetricLabels( + tenant_id="tenant-123", + app_id="app-456", + operation_type=OperationType.WORKFLOW, + model_provider="", + model_name="", + node_type="", + ) + exporter.increment_counter( + EnterpriseTelemetryCounter.INPUT_TOKENS, + 100, + labels.to_dict() + ) + + Design rationale: + Without this unified structure, tokens get double-counted when querying totals + because workflow.total_tokens is already the sum of all node tokens. The + operation_type label allows filtering to separate workflow-level aggregates from + node-level detail, while keeping the same label cardinality for consistent queries. + """ + + tenant_id: str + app_id: str + operation_type: str + model_provider: str + model_name: str + node_type: str + + model_config = ConfigDict(extra="forbid", frozen=True) + + def to_dict(self) -> dict[str, AttributeValue]: + return cast( + dict[str, AttributeValue], + { + "tenant_id": self.tenant_id, + "app_id": self.app_id, + "operation_type": self.operation_type, + "model_provider": self.model_provider, + "model_name": self.model_name, + "node_type": self.node_type, + }, + ) + + __all__ = [ "EnterpriseTelemetryCounter", "EnterpriseTelemetryEvent", "EnterpriseTelemetryHistogram", "EnterpriseTelemetrySpan", + "TokenMetricLabels", ] From f78b0f1f366bbc02c1eafbcee240c67b8b7591e2 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 9 Feb 2026 01:26:26 -0800 Subject: [PATCH 2/4] feat(enterprise-telemetry): add ENTERPRISE_OTLP_API_KEY config field --- api/configs/enterprise/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/configs/enterprise/__init__.py b/api/configs/enterprise/__init__.py index 4920eeba07..2db9de6195 100644 --- a/api/configs/enterprise/__init__.py +++ b/api/configs/enterprise/__init__.py @@ -45,6 +45,12 @@ class EnterpriseTelemetryConfig(BaseSettings): default="http", ) + ENTERPRISE_OTLP_API_KEY: str = Field( + description="Bearer token for enterprise OTLP export authentication. " + "When set, gRPC exporters automatically use TLS (insecure=False).", + default="", + ) + ENTERPRISE_INCLUDE_CONTENT: bool = Field( description="Include input/output content in traces (privacy toggle).", default=True, From ffa8aedc48719d3efc7567fdd28a962b960b5311 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 9 Feb 2026 01:29:40 -0800 Subject: [PATCH 3/4] feat(enterprise-telemetry): wire bearer token auth and configurable insecure flag into OTEL exporter --- api/enterprise/telemetry/exporter.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/api/enterprise/telemetry/exporter.py b/api/enterprise/telemetry/exporter.py index 8b66157c61..247e03691f 100644 --- a/api/enterprise/telemetry/exporter.py +++ b/api/enterprise/telemetry/exporter.py @@ -64,19 +64,20 @@ def _datetime_to_ns(dt: datetime) -> int: class _ExporterFactory: - def __init__(self, protocol: str, endpoint: str, headers: dict[str, str]): + def __init__(self, protocol: str, endpoint: str, headers: dict[str, str], insecure: bool): self._protocol = protocol self._endpoint = endpoint self._headers = headers self._grpc_headers = tuple(headers.items()) if headers else None self._http_headers = headers or None + self._insecure = insecure def create_trace_exporter(self) -> HTTPSpanExporter | GRPCSpanExporter: if self._protocol == "grpc": return GRPCSpanExporter( endpoint=self._endpoint or None, headers=self._grpc_headers, - insecure=True, + insecure=self._insecure, ) trace_endpoint = f"{self._endpoint}/v1/traces" if self._endpoint else "" return HTTPSpanExporter(endpoint=trace_endpoint or None, headers=self._http_headers) @@ -86,7 +87,7 @@ class _ExporterFactory: return GRPCMetricExporter( endpoint=self._endpoint or None, headers=self._grpc_headers, - insecure=True, + insecure=self._insecure, ) metric_endpoint = f"{self._endpoint}/v1/metrics" if self._endpoint else "" return HTTPMetricExporter(endpoint=metric_endpoint or None, headers=self._http_headers) @@ -107,6 +108,9 @@ class EnterpriseExporter: service_name: str = getattr(config, "ENTERPRISE_SERVICE_NAME", "dify") sampling_rate: float = getattr(config, "ENTERPRISE_OTEL_SAMPLING_RATE", 1.0) self.include_content: bool = getattr(config, "ENTERPRISE_INCLUDE_CONTENT", True) + api_key: str = getattr(config, "ENTERPRISE_OTLP_API_KEY", "") + # Auto-detect TLS: when bearer token is configured, use secure channel + insecure: bool = not bool(api_key) resource = Resource( attributes={ @@ -119,7 +123,14 @@ class EnterpriseExporter: self._tracer_provider = TracerProvider(resource=resource, sampler=sampler, id_generator=id_generator) headers = _parse_otlp_headers(headers_raw) - factory = _ExporterFactory(protocol, endpoint, headers) + if api_key: + if "authorization" in headers: + logger.warning( + "ENTERPRISE_OTLP_API_KEY is set but ENTERPRISE_OTLP_HEADERS also contains " + "'authorization'; the API key will take precedence." + ) + headers["authorization"] = f"Bearer {api_key}" + factory = _ExporterFactory(protocol, endpoint, headers, insecure=insecure) trace_exporter = factory.create_trace_exporter() self._tracer_provider.add_span_processor(BatchSpanProcessor(trace_exporter)) From aa34ec0d257eb24c37cb2c16b44446463dccbb78 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 9 Feb 2026 01:35:17 -0800 Subject: [PATCH 4/4] test(enterprise-telemetry): add unit tests for OTEL bearer auth and insecure flag --- api/tests/unit_tests/enterprise/__init__.py | 0 .../enterprise/telemetry/__init__.py | 0 .../enterprise/telemetry/test_exporter.py | 215 ++++++++++++++++++ 3 files changed, 215 insertions(+) create mode 100644 api/tests/unit_tests/enterprise/__init__.py create mode 100644 api/tests/unit_tests/enterprise/telemetry/__init__.py create mode 100644 api/tests/unit_tests/enterprise/telemetry/test_exporter.py diff --git a/api/tests/unit_tests/enterprise/__init__.py b/api/tests/unit_tests/enterprise/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/enterprise/telemetry/__init__.py b/api/tests/unit_tests/enterprise/telemetry/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/unit_tests/enterprise/telemetry/test_exporter.py b/api/tests/unit_tests/enterprise/telemetry/test_exporter.py new file mode 100644 index 0000000000..48fdd308f8 --- /dev/null +++ b/api/tests/unit_tests/enterprise/telemetry/test_exporter.py @@ -0,0 +1,215 @@ +"""Unit tests for EnterpriseExporter and _ExporterFactory.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from configs.enterprise import EnterpriseTelemetryConfig +from enterprise.telemetry.exporter import EnterpriseExporter + + +def test_config_api_key_default_empty(): + """Test that ENTERPRISE_OTLP_API_KEY defaults to empty string.""" + config = EnterpriseTelemetryConfig() + assert config.ENTERPRISE_OTLP_API_KEY == "" + + +@patch("enterprise.telemetry.exporter.GRPCSpanExporter") +@patch("enterprise.telemetry.exporter.GRPCMetricExporter") +def test_api_key_only_injects_bearer_header(mock_metric_exporter: MagicMock, mock_span_exporter: MagicMock) -> None: + """Test that API key alone injects Bearer authorization header.""" + mock_config = SimpleNamespace( + ENTERPRISE_OTLP_ENDPOINT="https://collector.example.com", + ENTERPRISE_OTLP_HEADERS="", + ENTERPRISE_OTLP_PROTOCOL="grpc", + ENTERPRISE_SERVICE_NAME="dify", + ENTERPRISE_OTEL_SAMPLING_RATE=1.0, + ENTERPRISE_INCLUDE_CONTENT=True, + ENTERPRISE_OTLP_API_KEY="test-secret-key", + ) + + EnterpriseExporter(mock_config) + + # Verify span exporter was called with Bearer header + assert mock_span_exporter.call_args is not None + headers = mock_span_exporter.call_args.kwargs.get("headers") + assert headers is not None + assert ("authorization", "Bearer test-secret-key") in headers + + +@patch("enterprise.telemetry.exporter.GRPCSpanExporter") +@patch("enterprise.telemetry.exporter.GRPCMetricExporter") +def test_empty_api_key_no_auth_header(mock_metric_exporter: MagicMock, mock_span_exporter: MagicMock) -> None: + """Test that empty API key does not inject authorization header.""" + mock_config = SimpleNamespace( + ENTERPRISE_OTLP_ENDPOINT="https://collector.example.com", + ENTERPRISE_OTLP_HEADERS="", + ENTERPRISE_OTLP_PROTOCOL="grpc", + ENTERPRISE_SERVICE_NAME="dify", + ENTERPRISE_OTEL_SAMPLING_RATE=1.0, + ENTERPRISE_INCLUDE_CONTENT=True, + ENTERPRISE_OTLP_API_KEY="", + ) + + EnterpriseExporter(mock_config) + + # Verify span exporter was called without authorization header + assert mock_span_exporter.call_args is not None + headers = mock_span_exporter.call_args.kwargs.get("headers") + # Headers should be None or not contain authorization + if headers is not None: + assert not any(key == "authorization" for key, _ in headers) + + +@patch("enterprise.telemetry.exporter.GRPCSpanExporter") +@patch("enterprise.telemetry.exporter.GRPCMetricExporter") +def test_api_key_and_custom_headers_merge(mock_metric_exporter: MagicMock, mock_span_exporter: MagicMock) -> None: + """Test that API key and custom headers are merged correctly.""" + mock_config = SimpleNamespace( + ENTERPRISE_OTLP_ENDPOINT="https://collector.example.com", + ENTERPRISE_OTLP_HEADERS="x-custom=foo", + ENTERPRISE_OTLP_PROTOCOL="grpc", + ENTERPRISE_SERVICE_NAME="dify", + ENTERPRISE_OTEL_SAMPLING_RATE=1.0, + ENTERPRISE_INCLUDE_CONTENT=True, + ENTERPRISE_OTLP_API_KEY="test-key", + ) + + EnterpriseExporter(mock_config) + + # Verify both headers are present + assert mock_span_exporter.call_args is not None + headers = mock_span_exporter.call_args.kwargs.get("headers") + assert headers is not None + assert ("authorization", "Bearer test-key") in headers + assert ("x-custom", "foo") in headers + + +@patch("enterprise.telemetry.exporter.logger") +@patch("enterprise.telemetry.exporter.GRPCSpanExporter") +@patch("enterprise.telemetry.exporter.GRPCMetricExporter") +def test_api_key_overrides_conflicting_header( + mock_metric_exporter: MagicMock, mock_span_exporter: MagicMock, mock_logger: MagicMock +) -> None: + """Test that API key overrides conflicting authorization header and logs warning.""" + mock_config = SimpleNamespace( + ENTERPRISE_OTLP_ENDPOINT="https://collector.example.com", + ENTERPRISE_OTLP_HEADERS="authorization=Basic old", + ENTERPRISE_OTLP_PROTOCOL="grpc", + ENTERPRISE_SERVICE_NAME="dify", + ENTERPRISE_OTEL_SAMPLING_RATE=1.0, + ENTERPRISE_INCLUDE_CONTENT=True, + ENTERPRISE_OTLP_API_KEY="test-key", + ) + + EnterpriseExporter(mock_config) + + # Verify Bearer header takes precedence + assert mock_span_exporter.call_args is not None + headers = mock_span_exporter.call_args.kwargs.get("headers") + assert headers is not None + assert ("authorization", "Bearer test-key") in headers + # Verify old authorization header is not present + assert ("authorization", "Basic old") not in headers + + # Verify warning was logged + mock_logger.warning.assert_called_once() + assert mock_logger.warning.call_args is not None + warning_message = mock_logger.warning.call_args[0][0] + assert "ENTERPRISE_OTLP_API_KEY is set" in warning_message + assert "authorization" in warning_message + + +@patch("enterprise.telemetry.exporter.GRPCSpanExporter") +@patch("enterprise.telemetry.exporter.GRPCMetricExporter") +def test_api_key_set_uses_secure_grpc(mock_metric_exporter: MagicMock, mock_span_exporter: MagicMock) -> None: + """Test that API key presence enables TLS (insecure=False) for gRPC.""" + mock_config = SimpleNamespace( + ENTERPRISE_OTLP_ENDPOINT="https://collector.example.com", + ENTERPRISE_OTLP_HEADERS="", + ENTERPRISE_OTLP_PROTOCOL="grpc", + ENTERPRISE_SERVICE_NAME="dify", + ENTERPRISE_OTEL_SAMPLING_RATE=1.0, + ENTERPRISE_INCLUDE_CONTENT=True, + ENTERPRISE_OTLP_API_KEY="test-key", + ) + + EnterpriseExporter(mock_config) + + # Verify insecure=False for both exporters + assert mock_span_exporter.call_args is not None + assert mock_span_exporter.call_args.kwargs["insecure"] is False + + assert mock_metric_exporter.call_args is not None + assert mock_metric_exporter.call_args.kwargs["insecure"] is False + + +@patch("enterprise.telemetry.exporter.GRPCSpanExporter") +@patch("enterprise.telemetry.exporter.GRPCMetricExporter") +def test_no_api_key_uses_insecure_grpc(mock_metric_exporter: MagicMock, mock_span_exporter: MagicMock) -> None: + """Test that empty API key uses insecure gRPC (backward compat).""" + mock_config = SimpleNamespace( + ENTERPRISE_OTLP_ENDPOINT="http://collector.example.com", + ENTERPRISE_OTLP_HEADERS="", + ENTERPRISE_OTLP_PROTOCOL="grpc", + ENTERPRISE_SERVICE_NAME="dify", + ENTERPRISE_OTEL_SAMPLING_RATE=1.0, + ENTERPRISE_INCLUDE_CONTENT=True, + ENTERPRISE_OTLP_API_KEY="", + ) + + EnterpriseExporter(mock_config) + + # Verify insecure=True for both exporters + assert mock_span_exporter.call_args is not None + assert mock_span_exporter.call_args.kwargs["insecure"] is True + + assert mock_metric_exporter.call_args is not None + assert mock_metric_exporter.call_args.kwargs["insecure"] is True + + +@patch("enterprise.telemetry.exporter.HTTPSpanExporter") +@patch("enterprise.telemetry.exporter.HTTPMetricExporter") +def test_insecure_not_passed_to_http_exporters(mock_metric_exporter: MagicMock, mock_span_exporter: MagicMock) -> None: + """Test that insecure parameter is not passed to HTTP exporters.""" + mock_config = SimpleNamespace( + ENTERPRISE_OTLP_ENDPOINT="http://collector.example.com", + ENTERPRISE_OTLP_HEADERS="", + ENTERPRISE_OTLP_PROTOCOL="http", + ENTERPRISE_SERVICE_NAME="dify", + ENTERPRISE_OTEL_SAMPLING_RATE=1.0, + ENTERPRISE_INCLUDE_CONTENT=True, + ENTERPRISE_OTLP_API_KEY="test-key", + ) + + EnterpriseExporter(mock_config) + + # Verify insecure kwarg is NOT in HTTP exporter calls + assert mock_span_exporter.call_args is not None + assert "insecure" not in mock_span_exporter.call_args.kwargs + + assert mock_metric_exporter.call_args is not None + assert "insecure" not in mock_metric_exporter.call_args.kwargs + + +@patch("enterprise.telemetry.exporter.GRPCSpanExporter") +@patch("enterprise.telemetry.exporter.GRPCMetricExporter") +def test_api_key_with_special_chars_preserved(mock_metric_exporter: MagicMock, mock_span_exporter: MagicMock) -> None: + """Test that API key with special characters is preserved without mangling.""" + special_key = "abc+def/ghi=jkl==" + mock_config = SimpleNamespace( + ENTERPRISE_OTLP_ENDPOINT="https://collector.example.com", + ENTERPRISE_OTLP_HEADERS="", + ENTERPRISE_OTLP_PROTOCOL="grpc", + ENTERPRISE_SERVICE_NAME="dify", + ENTERPRISE_OTEL_SAMPLING_RATE=1.0, + ENTERPRISE_INCLUDE_CONTENT=True, + ENTERPRISE_OTLP_API_KEY=special_key, + ) + + EnterpriseExporter(mock_config) + + # Verify special characters are preserved in Bearer header + assert mock_span_exporter.call_args is not None + headers = mock_span_exporter.call_args.kwargs.get("headers") + assert headers is not None + assert ("authorization", f"Bearer {special_key}") in headers