From e06943d6da0871d271078eac197ef760fc77be5f Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Tue, 17 Nov 2026 23:46:03 +0000 Subject: [PATCH 1/4] feat(spanner): Add OpenTelemetry OTLP end-to-end tracing sample Tracing documentation embedded an archived python-spanner example that executed an invalid SQL query and omitted clean TracerProvider shutdown. --- .../opentelemetry_traces_otlp.py | 109 +++++++++++++ .../tests/opentelemetry_traces_otlp_test.py | 152 ++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py create mode 100644 spanner/observability/tests/opentelemetry_traces_otlp_test.py diff --git a/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py b/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py new file mode 100644 index 0000000000..752f729dea --- /dev/null +++ b/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py @@ -0,0 +1,109 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START spanner_opentelemetry_traces_otlp_usage] +import sys + +from google.api_core import exceptions +from google.cloud import spanner +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.propagate import set_global_textmap +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_ON +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + + +def configure_otlp_tracing(project_id: str, instance_id: str, database_id: str) -> None: + """Configures client-side and end-to-end Spanner tracing via OTLP gRPC. + + Args: + project_id: The Google Cloud project ID. + instance_id: The Spanner instance ID. + database_id: The Spanner database ID. + """ + set_global_textmap(TraceContextTextMapPropagator()) + tracer_provider = TracerProvider(sampler=ALWAYS_ON) + otlp_exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) + tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter)) + + observability_options = { + "tracer_provider": tracer_provider, + "enable_extended_tracing": True, + "enable_end_to_end_tracing": True, + } + client = spanner.Client( + project=project_id, + observability_options=observability_options, + ) + database = client.instance(instance_id).database(database_id) + tracer = tracer_provider.get_tracer("spanner-otlp-sample") + + try: + with tracer.start_as_current_span("VerifySpannerRead"): + with database.snapshot() as snapshot: + row = list(snapshot.execute_sql("SELECT 1"))[0] + except exceptions.NotFound as err: + print( + f"error: database '{database_id}' on instance '{instance_id}' " + f"in project '{project_id}' was not found: {err.message}", + file=sys.stderr, + ) + except exceptions.GoogleAPICallError as err: + print( + f"error: OTLP-traced query failed on database '{database_id}' " + f"in instance '{instance_id}' for project '{project_id}': {err.message}", + file=sys.stderr, + ) + else: + print(f"Resource ID: {database.name}") + print(f"Check Result: {row[0]}") + print("End-to-End Tracing: True") + print("Trace Exporter: OTLP") + finally: + tracer_provider.shutdown() + + +# [END spanner_opentelemetry_traces_otlp_usage] + + +if __name__ == "__main__": + import argparse + import os + + parser = argparse.ArgumentParser( + description="Configure OpenTelemetry OTLP tracing on a Spanner client." + ) + parser.add_argument( + "--project-id", + default=os.environ.get("GOOGLE_CLOUD_PROJECT", "example-project-id"), + help="Google Cloud project ID (default: $GOOGLE_CLOUD_PROJECT)", + ) + parser.add_argument( + "--instance-id", + default=os.environ.get("SPANNER_INSTANCE_ID", "example-instance"), + help="Spanner instance ID (default: $SPANNER_INSTANCE_ID)", + ) + parser.add_argument( + "--database-id", + default=os.environ.get("SPANNER_DATABASE_ID", "example-db"), + help="Spanner database ID (default: $SPANNER_DATABASE_ID)", + ) + args = parser.parse_args() + + configure_otlp_tracing( + project_id=args.project_id, + instance_id=args.instance_id, + database_id=args.database_id, + ) diff --git a/spanner/observability/tests/opentelemetry_traces_otlp_test.py b/spanner/observability/tests/opentelemetry_traces_otlp_test.py new file mode 100644 index 0000000000..ab82fd0a29 --- /dev/null +++ b/spanner/observability/tests/opentelemetry_traces_otlp_test.py @@ -0,0 +1,152 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import runpy +from unittest import mock + +from google.api_core import exceptions +from opentelemetry.sdk.trace.export import SpanExportResult +import pytest +from spanner_observability import opentelemetry_traces_otlp + +from . import conftest + +PROJECT_ID = "example-project-id" +INSTANCE_ID = "example-instance" +DATABASE_ID = "example-db" +DATABASE_NAME = f"projects/{PROJECT_ID}/instances/{INSTANCE_ID}/databases/{DATABASE_ID}" +SPANNER_CLIENT = "google.cloud.spanner.Client" +OTLP_EXPORTER = "spanner_observability.opentelemetry_traces_otlp.OTLPSpanExporter" +OTLP_EXPORTER_SRC = ( + "opentelemetry.exporter.otlp.proto.grpc.trace_exporter.OTLPSpanExporter" +) + + +@mock.patch(OTLP_EXPORTER) +@mock.patch(SPANNER_CLIENT) +def test_configure_otlp_tracing_success( + mock_client_class: mock.MagicMock, + mock_exporter_class: mock.MagicMock, + capsys: pytest.CaptureFixture[str], +) -> None: + mock_exporter_class.return_value.export.return_value = SpanExportResult.SUCCESS + database = ( + mock_client_class.return_value.instance.return_value.database.return_value + ) + database.name = DATABASE_NAME + snapshot = database.snapshot.return_value.__enter__.return_value + snapshot.execute_sql.return_value = [(1,)] + + opentelemetry_traces_otlp.configure_otlp_tracing( + PROJECT_ID, INSTANCE_ID, DATABASE_ID + ) + + obs = mock_client_class.call_args.kwargs["observability_options"] + assert obs["enable_end_to_end_tracing"] is True + assert obs["enable_extended_tracing"] is True + out, err = capsys.readouterr() + assert f"Resource ID: {DATABASE_NAME}" in out + assert "Check Result: 1" in out + assert "End-to-End Tracing: True" in out + assert "Trace Exporter: OTLP" in out + assert err == "" + + +@pytest.mark.parametrize( + "error, expected_prefix", + [ + ( + exceptions.NotFound("db missing"), + "error: database 'example-db' on instance 'example-instance'", + ), + ( + exceptions.PermissionDenied("denied"), + "error: OTLP-traced query failed", + ), + ], +) +@mock.patch(OTLP_EXPORTER) +@mock.patch(SPANNER_CLIENT) +def test_configure_otlp_tracing_errors( + mock_client_class: mock.MagicMock, + mock_exporter_class: mock.MagicMock, + capsys: pytest.CaptureFixture[str], + error: Exception, + expected_prefix: str, +) -> None: + mock_exporter_class.return_value.export.return_value = SpanExportResult.SUCCESS + database = ( + mock_client_class.return_value.instance.return_value.database.return_value + ) + snapshot = database.snapshot.return_value.__enter__.return_value + snapshot.execute_sql.side_effect = error + + opentelemetry_traces_otlp.configure_otlp_tracing( + PROJECT_ID, INSTANCE_ID, DATABASE_ID + ) + + out, err = capsys.readouterr() + assert out == "" + assert err.startswith(expected_prefix) + + +@mock.patch(OTLP_EXPORTER_SRC) +@mock.patch(SPANNER_CLIENT) +def test_configure_otlp_tracing_cli_entrypoint( + mock_client_class: mock.MagicMock, + mock_exporter_class: mock.MagicMock, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + mock_exporter_class.return_value.export.return_value = SpanExportResult.SUCCESS + database = ( + mock_client_class.return_value.instance.return_value.database.return_value + ) + database.name = DATABASE_NAME + snapshot = database.snapshot.return_value.__enter__.return_value + snapshot.execute_sql.return_value = [(1,)] + monkeypatch.setattr( + "sys.argv", + [ + "opentelemetry_traces_otlp.py", + "--project-id", + PROJECT_ID, + "--instance-id", + INSTANCE_ID, + "--database-id", + DATABASE_ID, + ], + ) + + runpy.run_module( + "spanner_observability.opentelemetry_traces_otlp", run_name="__main__" + ) + + assert "Trace Exporter: OTLP" in capsys.readouterr().out + + +@conftest.live_only +def test_configure_otlp_tracing_live(capsys: pytest.CaptureFixture[str]) -> None: + with mock.patch(OTLP_EXPORTER) as mock_exp: + mock_exp.return_value.export.return_value = SpanExportResult.SUCCESS + opentelemetry_traces_otlp.configure_otlp_tracing( + conftest.LIVE_PROJECT_ID, + conftest.LIVE_INSTANCE_ID, + conftest.LIVE_DATABASE_ID, + ) + + out, err = capsys.readouterr() + assert err == "" + assert "Check Result: 1" in out + assert "Trace Exporter: OTLP" in out From 3a6f2226310426174dc1abdcb69812df46feadad Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 18 Nov 2026 00:01:58 +0000 Subject: [PATCH 2/4] fix(spanner): Omit project_id from OTLP tracing sample output Avoids printing project identifiers in stdout resource lines or stderr error diagnostics. --- .../src/spanner_observability/opentelemetry_traces_otlp.py | 6 +++--- .../observability/tests/opentelemetry_traces_otlp_test.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py b/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py index 752f729dea..df4113e493 100644 --- a/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py +++ b/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py @@ -57,17 +57,17 @@ def configure_otlp_tracing(project_id: str, instance_id: str, database_id: str) except exceptions.NotFound as err: print( f"error: database '{database_id}' on instance '{instance_id}' " - f"in project '{project_id}' was not found: {err.message}", + f"was not found: {err.message}", file=sys.stderr, ) except exceptions.GoogleAPICallError as err: print( f"error: OTLP-traced query failed on database '{database_id}' " - f"in instance '{instance_id}' for project '{project_id}': {err.message}", + f"in instance '{instance_id}': {err.message}", file=sys.stderr, ) else: - print(f"Resource ID: {database.name}") + print(f"Resource ID: {database_id}") print(f"Check Result: {row[0]}") print("End-to-End Tracing: True") print("Trace Exporter: OTLP") diff --git a/spanner/observability/tests/opentelemetry_traces_otlp_test.py b/spanner/observability/tests/opentelemetry_traces_otlp_test.py index ab82fd0a29..2388c65586 100644 --- a/spanner/observability/tests/opentelemetry_traces_otlp_test.py +++ b/spanner/observability/tests/opentelemetry_traces_otlp_test.py @@ -56,7 +56,8 @@ def test_configure_otlp_tracing_success( assert obs["enable_end_to_end_tracing"] is True assert obs["enable_extended_tracing"] is True out, err = capsys.readouterr() - assert f"Resource ID: {DATABASE_NAME}" in out + assert f"Resource ID: {DATABASE_ID}" in out + assert PROJECT_ID not in out assert "Check Result: 1" in out assert "End-to-End Tracing: True" in out assert "Trace Exporter: OTLP" in out @@ -99,6 +100,7 @@ def test_configure_otlp_tracing_errors( out, err = capsys.readouterr() assert out == "" assert err.startswith(expected_prefix) + assert PROJECT_ID not in err @mock.patch(OTLP_EXPORTER_SRC) From fa781a3119e61bfc4a6e0d218b137d7124dd38f0 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 18 Nov 2026 00:38:24 +0000 Subject: [PATCH 3/4] fix(spanner): Omit project_id from sample stdout and stderr output Prints short resource identifiers instead of full project-prefixed resource paths and omits project_id from diagnostic messages. --- spanner/observability/tests/opentelemetry_traces_otlp_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spanner/observability/tests/opentelemetry_traces_otlp_test.py b/spanner/observability/tests/opentelemetry_traces_otlp_test.py index 2388c65586..33a95d2e9a 100644 --- a/spanner/observability/tests/opentelemetry_traces_otlp_test.py +++ b/spanner/observability/tests/opentelemetry_traces_otlp_test.py @@ -101,6 +101,8 @@ def test_configure_otlp_tracing_errors( assert out == "" assert err.startswith(expected_prefix) assert PROJECT_ID not in err + assert PROJECT_ID not in err + assert PROJECT_ID not in err @mock.patch(OTLP_EXPORTER_SRC) From 8da3e7f224c44d7c85c8df0e57169ddcc87ff49c Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 18 Nov 2026 02:42:54 +0000 Subject: [PATCH 4/4] fix(spanner): Remove hardcoded stdout lines and spy on live Client options Prints only runtime query results and verifies real spanner.Client options via wraps in live integration tests. --- .../opentelemetry_traces_otlp.py | 2 -- .../tests/opentelemetry_traces_otlp_test.py | 17 ++++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py b/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py index df4113e493..0f429ec366 100644 --- a/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py +++ b/spanner/observability/src/spanner_observability/opentelemetry_traces_otlp.py @@ -69,8 +69,6 @@ def configure_otlp_tracing(project_id: str, instance_id: str, database_id: str) else: print(f"Resource ID: {database_id}") print(f"Check Result: {row[0]}") - print("End-to-End Tracing: True") - print("Trace Exporter: OTLP") finally: tracer_provider.shutdown() diff --git a/spanner/observability/tests/opentelemetry_traces_otlp_test.py b/spanner/observability/tests/opentelemetry_traces_otlp_test.py index 33a95d2e9a..9e213d287c 100644 --- a/spanner/observability/tests/opentelemetry_traces_otlp_test.py +++ b/spanner/observability/tests/opentelemetry_traces_otlp_test.py @@ -16,6 +16,7 @@ from unittest import mock from google.api_core import exceptions +from google.cloud import spanner from opentelemetry.sdk.trace.export import SpanExportResult import pytest from spanner_observability import opentelemetry_traces_otlp @@ -59,8 +60,6 @@ def test_configure_otlp_tracing_success( assert f"Resource ID: {DATABASE_ID}" in out assert PROJECT_ID not in out assert "Check Result: 1" in out - assert "End-to-End Tracing: True" in out - assert "Trace Exporter: OTLP" in out assert err == "" @@ -101,8 +100,6 @@ def test_configure_otlp_tracing_errors( assert out == "" assert err.startswith(expected_prefix) assert PROJECT_ID not in err - assert PROJECT_ID not in err - assert PROJECT_ID not in err @mock.patch(OTLP_EXPORTER_SRC) @@ -137,12 +134,15 @@ def test_configure_otlp_tracing_cli_entrypoint( "spanner_observability.opentelemetry_traces_otlp", run_name="__main__" ) - assert "Trace Exporter: OTLP" in capsys.readouterr().out + assert "Check Result: 1" in capsys.readouterr().out @conftest.live_only def test_configure_otlp_tracing_live(capsys: pytest.CaptureFixture[str]) -> None: - with mock.patch(OTLP_EXPORTER) as mock_exp: + with ( + mock.patch(OTLP_EXPORTER) as mock_exp, + mock.patch.object(spanner, "Client", wraps=spanner.Client) as spy_client, + ): mock_exp.return_value.export.return_value = SpanExportResult.SUCCESS opentelemetry_traces_otlp.configure_otlp_tracing( conftest.LIVE_PROJECT_ID, @@ -150,7 +150,10 @@ def test_configure_otlp_tracing_live(capsys: pytest.CaptureFixture[str]) -> None conftest.LIVE_DATABASE_ID, ) + obs = spy_client.call_args.kwargs["observability_options"] + assert obs["enable_end_to_end_tracing"] is True + assert obs["enable_extended_tracing"] is True out, err = capsys.readouterr() assert err == "" + assert conftest.LIVE_PROJECT_ID not in out assert "Check Result: 1" in out - assert "Trace Exporter: OTLP" in out