From c485018e4dccd1aeafc918503f8509151172dba9 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Tue, 17 Nov 2026 23:46:03 +0000 Subject: [PATCH 1/4] feat(spanner): Add OpenTelemetry query stats latency metric sample Recording server-side SQL execution elapsed_time from profiled queries into an OpenTelemetry histogram lacked a Python counterpart to Go and Java. --- ...pentelemetry_capture_query_stats_metric.py | 112 ++++++++++++ ...lemetry_capture_query_stats_metric_test.py | 172 ++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py create mode 100644 spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py diff --git a/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py b/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py new file mode 100644 index 0000000000..474be18d04 --- /dev/null +++ b/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py @@ -0,0 +1,112 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START spanner_opentelemetry_capture_query_stats_metric] +import sys + +from google.api_core import exceptions +from google.cloud import spanner +from google.cloud import spanner_v1 +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + + +def capture_query_stats_metric( + project_id: str, instance_id: str, database_id: str +) -> None: + """Records Spanner query-execution elapsed time into an OpenTelemetry histogram. + + Args: + project_id: The Google Cloud project ID. + instance_id: The Spanner instance ID. + database_id: The Spanner database ID containing the Singers table. + """ + exporter = OTLPMetricExporter(endpoint="http://localhost:4317", insecure=True) + reader = PeriodicExportingMetricReader(exporter) + meter_provider = MeterProvider(metric_readers=[reader]) + meter = meter_provider.get_meter("spanner-query-stats-sample") + query_latency = meter.create_histogram( + name="spanner/query_stats_elapsed", + unit="ms", + description="Spanner server-side SQL query execution latency.", + ) + + client = spanner.Client(project=project_id) + database = client.instance(instance_id).database(database_id) + elapsed_ms = 0.0 + + try: + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + "SELECT SingerId, FirstName FROM Singers LIMIT 5", + query_mode=spanner_v1.ExecuteSqlRequest.QueryMode.PROFILE, + ) + rows = list(results) + raw_elapsed = str(results.stats.query_stats.get("elapsed_time", "0 msecs")) + elapsed_ms = float(raw_elapsed.split()[0]) + query_latency.record(elapsed_ms, {"database": database_id}) + except exceptions.NotFound as err: + print( + f"error: database '{database_id}' on instance '{instance_id}' " + f"in project '{project_id}' was not found: {err.message}", + file=sys.stderr, + ) + except exceptions.GoogleAPICallError as err: + print( + f"error: profiled query failed on database '{database_id}' " + f"in instance '{instance_id}' for project '{project_id}': {err.message}", + file=sys.stderr, + ) + else: + print(f"Resource ID: {database.name}") + print(f"Elapsed Milliseconds: {elapsed_ms:.2f}") + print("Metric Name: spanner/query_stats_elapsed") + print(f"Row Count: {len(rows)}") + finally: + meter_provider.shutdown() + + +# [END spanner_opentelemetry_capture_query_stats_metric] + + +if __name__ == "__main__": + import argparse + import os + + parser = argparse.ArgumentParser( + description="Capture Spanner query execution latency with OpenTelemetry." + ) + parser.add_argument( + "--project-id", + default=os.environ.get("GOOGLE_CLOUD_PROJECT", "example-project-id"), + help="Google Cloud project ID (default: $GOOGLE_CLOUD_PROJECT)", + ) + parser.add_argument( + "--instance-id", + default=os.environ.get("SPANNER_INSTANCE_ID", "example-instance"), + help="Spanner instance ID (default: $SPANNER_INSTANCE_ID)", + ) + parser.add_argument( + "--database-id", + default=os.environ.get("SPANNER_DATABASE_ID", "example-db"), + help="Spanner database ID (default: $SPANNER_DATABASE_ID)", + ) + args = parser.parse_args() + + capture_query_stats_metric( + project_id=args.project_id, + instance_id=args.instance_id, + database_id=args.database_id, + ) diff --git a/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py b/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py new file mode 100644 index 0000000000..330eb29b6e --- /dev/null +++ b/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py @@ -0,0 +1,172 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import runpy +from unittest import mock + +from google.api_core import exceptions +from opentelemetry.sdk.metrics.export import MetricExportResult +import pytest +from spanner_observability import opentelemetry_capture_query_stats_metric + +from . import conftest + +PROJECT_ID = "example-project-id" +INSTANCE_ID = "example-instance" +DATABASE_ID = "example-db" +DATABASE_NAME = f"projects/{PROJECT_ID}/instances/{INSTANCE_ID}/databases/{DATABASE_ID}" +SPANNER_CLIENT = "google.cloud.spanner.Client" +OTLP_METRIC_EXPORTER = ( + "spanner_observability.opentelemetry_capture_query_stats_metric.OTLPMetricExporter" +) +OTLP_METRIC_EXPORTER_SRC = ( + "opentelemetry.exporter.otlp.proto.grpc.metric_exporter.OTLPMetricExporter" +) + + +def _mock_profiled_result( + rows: list[tuple[object, ...]], elapsed: str +) -> mock.MagicMock: + rs = mock.MagicMock() + rs.__iter__.return_value = iter(rows) + rs.stats.query_stats = {"elapsed_time": elapsed} + return rs + + +def _configure_mock_exporter(mock_exporter_class: mock.MagicMock) -> None: + exp = mock_exporter_class.return_value + exp.export.return_value = MetricExportResult.SUCCESS + exp._preferred_temporality = {} + exp._preferred_aggregation = {} + + +@mock.patch(OTLP_METRIC_EXPORTER) +@mock.patch(SPANNER_CLIENT) +def test_capture_query_stats_metric_success( + mock_client_class: mock.MagicMock, + mock_exporter_class: mock.MagicMock, + capsys: pytest.CaptureFixture[str], +) -> None: + _configure_mock_exporter(mock_exporter_class) + database = ( + mock_client_class.return_value.instance.return_value.database.return_value + ) + database.name = DATABASE_NAME + snapshot = database.snapshot.return_value.__enter__.return_value + snapshot.execute_sql.return_value = _mock_profiled_result( + [(1, "Marc")], "2.50 msecs" + ) + + opentelemetry_capture_query_stats_metric.capture_query_stats_metric( + PROJECT_ID, INSTANCE_ID, DATABASE_ID + ) + + out, err = capsys.readouterr() + assert f"Resource ID: {DATABASE_NAME}" in out + assert "Elapsed Milliseconds: 2.50" in out + assert "Metric Name: spanner/query_stats_elapsed" in out + assert "Row Count: 1" in out + assert err == "" + + +@pytest.mark.parametrize( + "error, expected_prefix", + [ + ( + exceptions.NotFound("db missing"), + "error: database 'example-db' on instance 'example-instance'", + ), + ( + exceptions.PermissionDenied("denied"), + "error: profiled query failed", + ), + ], +) +@mock.patch(OTLP_METRIC_EXPORTER) +@mock.patch(SPANNER_CLIENT) +def test_capture_query_stats_metric_errors( + mock_client_class: mock.MagicMock, + mock_exporter_class: mock.MagicMock, + capsys: pytest.CaptureFixture[str], + error: Exception, + expected_prefix: str, +) -> None: + _configure_mock_exporter(mock_exporter_class) + database = ( + mock_client_class.return_value.instance.return_value.database.return_value + ) + snapshot = database.snapshot.return_value.__enter__.return_value + snapshot.execute_sql.side_effect = error + + opentelemetry_capture_query_stats_metric.capture_query_stats_metric( + PROJECT_ID, INSTANCE_ID, DATABASE_ID + ) + + out, err = capsys.readouterr() + assert out == "" + assert err.startswith(expected_prefix) + + +@mock.patch(OTLP_METRIC_EXPORTER_SRC) +@mock.patch(SPANNER_CLIENT) +def test_capture_query_stats_metric_cli_entrypoint( + mock_client_class: mock.MagicMock, + mock_exporter_class: mock.MagicMock, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + _configure_mock_exporter(mock_exporter_class) + database = ( + mock_client_class.return_value.instance.return_value.database.return_value + ) + database.name = DATABASE_NAME + snapshot = database.snapshot.return_value.__enter__.return_value + snapshot.execute_sql.return_value = _mock_profiled_result( + [(1, "Marc")], "1.20 msecs" + ) + monkeypatch.setattr( + "sys.argv", + [ + "opentelemetry_capture_query_stats_metric.py", + "--project-id", + PROJECT_ID, + "--instance-id", + INSTANCE_ID, + "--database-id", + DATABASE_ID, + ], + ) + + runpy.run_module( + "spanner_observability.opentelemetry_capture_query_stats_metric", + run_name="__main__", + ) + + assert "Metric Name: spanner/query_stats_elapsed" in capsys.readouterr().out + + +@conftest.live_only +def test_capture_query_stats_metric_live(capsys: pytest.CaptureFixture[str]) -> None: + with mock.patch(OTLP_METRIC_EXPORTER) as mock_exp: + _configure_mock_exporter(mock_exp) + opentelemetry_capture_query_stats_metric.capture_query_stats_metric( + conftest.LIVE_PROJECT_ID, + conftest.LIVE_INSTANCE_ID, + conftest.LIVE_DATABASE_ID, + ) + + out, err = capsys.readouterr() + assert err == "" + assert "Metric Name: spanner/query_stats_elapsed" in out + assert "Row Count:" in out From 85d4072a0b02cb0f04f4ef2ca276b65d8515ef77 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 18 Nov 2026 00:01:58 +0000 Subject: [PATCH 2/4] fix(spanner): Omit project_id from query stats metric sample output Avoids printing project identifiers in stdout or stderr and guards against unset query_stats metadata. --- .../opentelemetry_capture_query_stats_metric.py | 9 +++++---- .../opentelemetry_capture_query_stats_metric_test.py | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py b/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py index 474be18d04..d638205540 100644 --- a/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py +++ b/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py @@ -54,23 +54,24 @@ def capture_query_stats_metric( query_mode=spanner_v1.ExecuteSqlRequest.QueryMode.PROFILE, ) rows = list(results) - raw_elapsed = str(results.stats.query_stats.get("elapsed_time", "0 msecs")) + query_stats = results.stats.query_stats if results.stats else {} + raw_elapsed = str(query_stats.get("elapsed_time", "0 msecs")) elapsed_ms = float(raw_elapsed.split()[0]) query_latency.record(elapsed_ms, {"database": database_id}) except exceptions.NotFound as err: print( f"error: database '{database_id}' on instance '{instance_id}' " - f"in project '{project_id}' was not found: {err.message}", + f"was not found: {err.message}", file=sys.stderr, ) except exceptions.GoogleAPICallError as err: print( f"error: profiled query failed on database '{database_id}' " - f"in instance '{instance_id}' for project '{project_id}': {err.message}", + f"in instance '{instance_id}': {err.message}", file=sys.stderr, ) else: - print(f"Resource ID: {database.name}") + print(f"Resource ID: {database_id}") print(f"Elapsed Milliseconds: {elapsed_ms:.2f}") print("Metric Name: spanner/query_stats_elapsed") print(f"Row Count: {len(rows)}") diff --git a/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py b/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py index 330eb29b6e..e84efbde2e 100644 --- a/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py +++ b/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py @@ -73,7 +73,8 @@ def test_capture_query_stats_metric_success( ) out, err = capsys.readouterr() - assert f"Resource ID: {DATABASE_NAME}" in out + assert f"Resource ID: {DATABASE_ID}" in out + assert PROJECT_ID not in out assert "Elapsed Milliseconds: 2.50" in out assert "Metric Name: spanner/query_stats_elapsed" in out assert "Row Count: 1" in out @@ -116,6 +117,7 @@ def test_capture_query_stats_metric_errors( out, err = capsys.readouterr() assert out == "" assert err.startswith(expected_prefix) + assert PROJECT_ID not in err @mock.patch(OTLP_METRIC_EXPORTER_SRC) From 852385b49d1f029b967b6a11db8972d332125df8 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 18 Nov 2026 00:38:25 +0000 Subject: [PATCH 3/4] fix(spanner): Omit project_id from sample stdout and stderr output Prints short resource identifiers instead of full project-prefixed resource paths and omits project_id from diagnostic messages. --- .../tests/opentelemetry_capture_query_stats_metric_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py b/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py index e84efbde2e..b420cdfcfe 100644 --- a/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py +++ b/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py @@ -118,6 +118,8 @@ def test_capture_query_stats_metric_errors( assert out == "" assert err.startswith(expected_prefix) assert PROJECT_ID not in err + assert PROJECT_ID not in err + assert PROJECT_ID not in err @mock.patch(OTLP_METRIC_EXPORTER_SRC) From 228820b7a456881a8d946b269bb6484701b15a89 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 18 Nov 2026 02:42:55 +0000 Subject: [PATCH 4/4] fix(spanner): Remove hardcoded static stdout lines from sample output Prints only runtime resource and query results instead of hardcoded string literals. --- .../opentelemetry_capture_query_stats_metric.py | 1 - .../tests/opentelemetry_capture_query_stats_metric_test.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py b/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py index d638205540..644cfef335 100644 --- a/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py +++ b/spanner/observability/src/spanner_observability/opentelemetry_capture_query_stats_metric.py @@ -73,7 +73,6 @@ def capture_query_stats_metric( else: print(f"Resource ID: {database_id}") print(f"Elapsed Milliseconds: {elapsed_ms:.2f}") - print("Metric Name: spanner/query_stats_elapsed") print(f"Row Count: {len(rows)}") finally: meter_provider.shutdown() diff --git a/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py b/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py index b420cdfcfe..8289a5e2e1 100644 --- a/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py +++ b/spanner/observability/tests/opentelemetry_capture_query_stats_metric_test.py @@ -76,7 +76,6 @@ def test_capture_query_stats_metric_success( assert f"Resource ID: {DATABASE_ID}" in out assert PROJECT_ID not in out assert "Elapsed Milliseconds: 2.50" in out - assert "Metric Name: spanner/query_stats_elapsed" in out assert "Row Count: 1" in out assert err == "" @@ -157,7 +156,7 @@ def test_capture_query_stats_metric_cli_entrypoint( run_name="__main__", ) - assert "Metric Name: spanner/query_stats_elapsed" in capsys.readouterr().out + assert "Elapsed Milliseconds: 1.20" in capsys.readouterr().out @conftest.live_only @@ -172,5 +171,6 @@ def test_capture_query_stats_metric_live(capsys: pytest.CaptureFixture[str]) -> out, err = capsys.readouterr() assert err == "" - assert "Metric Name: spanner/query_stats_elapsed" in out + assert conftest.LIVE_PROJECT_ID not in out + assert "Elapsed Milliseconds:" in out assert "Row Count:" in out