Repository navigation
feat(spanner): Add OpenTelemetry query stats latency metric sample #14872
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/spanner-disable-builtin-metrics
Are you sure you want to change the base?
Changes from all commits
c485018
85d4072
852385b
228820b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,112 @@ | ||||||||||||||||||||||
| # Copyright 2026 Google LLC | ||||||||||||||||||||||
| # | ||||||||||||||||||||||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||||||||||||||||||||||
| # you may not use this file except in compliance with the License. | ||||||||||||||||||||||
| # You may obtain a copy of the License at | ||||||||||||||||||||||
| # | ||||||||||||||||||||||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||||||||||||||||||||||
| # | ||||||||||||||||||||||
| # Unless required by applicable law or agreed to in writing, software | ||||||||||||||||||||||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||||||||||||||||||||||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||||||||||||||||||||||
| # See the License for the specific language governing permissions and | ||||||||||||||||||||||
| # limitations under the License. | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| # [START spanner_opentelemetry_capture_query_stats_metric] | ||||||||||||||||||||||
| import sys | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| from google.api_core import exceptions | ||||||||||||||||||||||
| from google.cloud import spanner | ||||||||||||||||||||||
| from google.cloud import spanner_v1 | ||||||||||||||||||||||
| from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter | ||||||||||||||||||||||
| from opentelemetry.sdk.metrics import MeterProvider | ||||||||||||||||||||||
| from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
|
|
||||||||||||||||||||||
| def capture_query_stats_metric( | ||||||||||||||||||||||
| project_id: str, instance_id: str, database_id: str | ||||||||||||||||||||||
| ) -> None: | ||||||||||||||||||||||
| """Records Spanner query-execution elapsed time into an OpenTelemetry histogram. | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| Args: | ||||||||||||||||||||||
| project_id: The Google Cloud project ID. | ||||||||||||||||||||||
| instance_id: The Spanner instance ID. | ||||||||||||||||||||||
| database_id: The Spanner database ID containing the Singers table. | ||||||||||||||||||||||
| """ | ||||||||||||||||||||||
| exporter = OTLPMetricExporter(endpoint="http://localhost:4317", insecure=True) | ||||||||||||||||||||||
| reader = PeriodicExportingMetricReader(exporter) | ||||||||||||||||||||||
| meter_provider = MeterProvider(metric_readers=[reader]) | ||||||||||||||||||||||
| meter = meter_provider.get_meter("spanner-query-stats-sample") | ||||||||||||||||||||||
| query_latency = meter.create_histogram( | ||||||||||||||||||||||
| name="spanner/query_stats_elapsed", | ||||||||||||||||||||||
| unit="ms", | ||||||||||||||||||||||
| description="Spanner server-side SQL query execution latency.", | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| client = spanner.Client(project=project_id) | ||||||||||||||||||||||
| database = client.instance(instance_id).database(database_id) | ||||||||||||||||||||||
| elapsed_ms = 0.0 | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| try: | ||||||||||||||||||||||
| with database.snapshot() as snapshot: | ||||||||||||||||||||||
| results = snapshot.execute_sql( | ||||||||||||||||||||||
| "SELECT SingerId, FirstName FROM Singers LIMIT 5", | ||||||||||||||||||||||
| query_mode=spanner_v1.ExecuteSqlRequest.QueryMode.PROFILE, | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
| rows = list(results) | ||||||||||||||||||||||
| query_stats = results.stats.query_stats if results.stats else {} | ||||||||||||||||||||||
| raw_elapsed = str(query_stats.get("elapsed_time", "0 msecs")) | ||||||||||||||||||||||
| elapsed_ms = float(raw_elapsed.split()[0]) | ||||||||||||||||||||||
| query_latency.record(elapsed_ms, {"database": database_id}) | ||||||||||||||||||||||
| except exceptions.NotFound as err: | ||||||||||||||||||||||
| print( | ||||||||||||||||||||||
| f"error: database '{database_id}' on instance '{instance_id}' " | ||||||||||||||||||||||
| f"was not found: {err.message}", | ||||||||||||||||||||||
| file=sys.stderr, | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
| except exceptions.GoogleAPICallError as err: | ||||||||||||||||||||||
| print( | ||||||||||||||||||||||
| f"error: profiled query failed on database '{database_id}' " | ||||||||||||||||||||||
| f"in instance '{instance_id}': {err.message}", | ||||||||||||||||||||||
| file=sys.stderr, | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
| else: | ||||||||||||||||||||||
| print(f"Resource ID: {database_id}") | ||||||||||||||||||||||
| print(f"Elapsed Milliseconds: {elapsed_ms:.2f}") | ||||||||||||||||||||||
| print(f"Row Count: {len(rows)}") | ||||||||||||||||||||||
| finally: | ||||||||||||||||||||||
| meter_provider.shutdown() | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
|
|
||||||||||||||||||||||
| # [END spanner_opentelemetry_capture_query_stats_metric] | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
|
|
||||||||||||||||||||||
| if __name__ == "__main__": | ||||||||||||||||||||||
| import argparse | ||||||||||||||||||||||
| import os | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| parser = argparse.ArgumentParser( | ||||||||||||||||||||||
| description="Capture Spanner query execution latency with OpenTelemetry." | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
| parser.add_argument( | ||||||||||||||||||||||
| "--project-id", | ||||||||||||||||||||||
| default=os.environ.get("GOOGLE_CLOUD_PROJECT", "example-project-id"), | ||||||||||||||||||||||
| help="Google Cloud project ID (default: $GOOGLE_CLOUD_PROJECT)", | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
|
Comment on lines
+91
to
+95
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When using
Suggested change
References
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the suggestion! We are intentionally keeping |
||||||||||||||||||||||
| parser.add_argument( | ||||||||||||||||||||||
| "--instance-id", | ||||||||||||||||||||||
| default=os.environ.get("SPANNER_INSTANCE_ID", "example-instance"), | ||||||||||||||||||||||
| help="Spanner instance ID (default: $SPANNER_INSTANCE_ID)", | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
| parser.add_argument( | ||||||||||||||||||||||
| "--database-id", | ||||||||||||||||||||||
| default=os.environ.get("SPANNER_DATABASE_ID", "example-db"), | ||||||||||||||||||||||
| help="Spanner database ID (default: $SPANNER_DATABASE_ID)", | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
| args = parser.parse_args() | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| capture_query_stats_metric( | ||||||||||||||||||||||
| project_id=args.project_id, | ||||||||||||||||||||||
| instance_id=args.instance_id, | ||||||||||||||||||||||
| database_id=args.database_id, | ||||||||||||||||||||||
| ) | ||||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,176 @@ | ||
| # Copyright 2026 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import runpy | ||
| from unittest import mock | ||
|
|
||
| from google.api_core import exceptions | ||
| from opentelemetry.sdk.metrics.export import MetricExportResult | ||
| import pytest | ||
| from spanner_observability import opentelemetry_capture_query_stats_metric | ||
|
|
||
| from . import conftest | ||
|
|
||
| PROJECT_ID = "example-project-id" | ||
| INSTANCE_ID = "example-instance" | ||
| DATABASE_ID = "example-db" | ||
| DATABASE_NAME = f"projects/{PROJECT_ID}/instances/{INSTANCE_ID}/databases/{DATABASE_ID}" | ||
| SPANNER_CLIENT = "google.cloud.spanner.Client" | ||
| OTLP_METRIC_EXPORTER = ( | ||
| "spanner_observability.opentelemetry_capture_query_stats_metric.OTLPMetricExporter" | ||
| ) | ||
| OTLP_METRIC_EXPORTER_SRC = ( | ||
| "opentelemetry.exporter.otlp.proto.grpc.metric_exporter.OTLPMetricExporter" | ||
| ) | ||
|
|
||
|
|
||
| def _mock_profiled_result( | ||
| rows: list[tuple[object, ...]], elapsed: str | ||
| ) -> mock.MagicMock: | ||
| rs = mock.MagicMock() | ||
| rs.__iter__.return_value = iter(rows) | ||
| rs.stats.query_stats = {"elapsed_time": elapsed} | ||
| return rs | ||
|
|
||
|
|
||
| def _configure_mock_exporter(mock_exporter_class: mock.MagicMock) -> None: | ||
| exp = mock_exporter_class.return_value | ||
| exp.export.return_value = MetricExportResult.SUCCESS | ||
| exp._preferred_temporality = {} | ||
| exp._preferred_aggregation = {} | ||
|
|
||
|
|
||
| @mock.patch(OTLP_METRIC_EXPORTER) | ||
| @mock.patch(SPANNER_CLIENT) | ||
| def test_capture_query_stats_metric_success( | ||
| mock_client_class: mock.MagicMock, | ||
| mock_exporter_class: mock.MagicMock, | ||
| capsys: pytest.CaptureFixture[str], | ||
| ) -> None: | ||
| _configure_mock_exporter(mock_exporter_class) | ||
| database = ( | ||
| mock_client_class.return_value.instance.return_value.database.return_value | ||
| ) | ||
| database.name = DATABASE_NAME | ||
| snapshot = database.snapshot.return_value.__enter__.return_value | ||
| snapshot.execute_sql.return_value = _mock_profiled_result( | ||
| [(1, "Marc")], "2.50 msecs" | ||
| ) | ||
|
|
||
| opentelemetry_capture_query_stats_metric.capture_query_stats_metric( | ||
| PROJECT_ID, INSTANCE_ID, DATABASE_ID | ||
| ) | ||
|
|
||
| out, err = capsys.readouterr() | ||
| assert f"Resource ID: {DATABASE_ID}" in out | ||
| assert PROJECT_ID not in out | ||
| assert "Elapsed Milliseconds: 2.50" in out | ||
| assert "Row Count: 1" in out | ||
| assert err == "" | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "error, expected_prefix", | ||
| [ | ||
| ( | ||
| exceptions.NotFound("db missing"), | ||
| "error: database 'example-db' on instance 'example-instance'", | ||
| ), | ||
| ( | ||
| exceptions.PermissionDenied("denied"), | ||
| "error: profiled query failed", | ||
| ), | ||
| ], | ||
| ) | ||
| @mock.patch(OTLP_METRIC_EXPORTER) | ||
| @mock.patch(SPANNER_CLIENT) | ||
| def test_capture_query_stats_metric_errors( | ||
| mock_client_class: mock.MagicMock, | ||
| mock_exporter_class: mock.MagicMock, | ||
| capsys: pytest.CaptureFixture[str], | ||
| error: Exception, | ||
| expected_prefix: str, | ||
| ) -> None: | ||
| _configure_mock_exporter(mock_exporter_class) | ||
| database = ( | ||
| mock_client_class.return_value.instance.return_value.database.return_value | ||
| ) | ||
| snapshot = database.snapshot.return_value.__enter__.return_value | ||
| snapshot.execute_sql.side_effect = error | ||
|
|
||
| opentelemetry_capture_query_stats_metric.capture_query_stats_metric( | ||
| PROJECT_ID, INSTANCE_ID, DATABASE_ID | ||
| ) | ||
|
|
||
| out, err = capsys.readouterr() | ||
| assert out == "" | ||
| assert err.startswith(expected_prefix) | ||
| assert PROJECT_ID not in err | ||
| assert PROJECT_ID not in err | ||
| assert PROJECT_ID not in err | ||
|
|
||
|
|
||
| @mock.patch(OTLP_METRIC_EXPORTER_SRC) | ||
| @mock.patch(SPANNER_CLIENT) | ||
| def test_capture_query_stats_metric_cli_entrypoint( | ||
| mock_client_class: mock.MagicMock, | ||
| mock_exporter_class: mock.MagicMock, | ||
| monkeypatch: pytest.MonkeyPatch, | ||
| capsys: pytest.CaptureFixture[str], | ||
| ) -> None: | ||
| _configure_mock_exporter(mock_exporter_class) | ||
| database = ( | ||
| mock_client_class.return_value.instance.return_value.database.return_value | ||
| ) | ||
| database.name = DATABASE_NAME | ||
| snapshot = database.snapshot.return_value.__enter__.return_value | ||
| snapshot.execute_sql.return_value = _mock_profiled_result( | ||
| [(1, "Marc")], "1.20 msecs" | ||
| ) | ||
| monkeypatch.setattr( | ||
| "sys.argv", | ||
| [ | ||
| "opentelemetry_capture_query_stats_metric.py", | ||
| "--project-id", | ||
| PROJECT_ID, | ||
| "--instance-id", | ||
| INSTANCE_ID, | ||
| "--database-id", | ||
| DATABASE_ID, | ||
| ], | ||
| ) | ||
|
|
||
| runpy.run_module( | ||
| "spanner_observability.opentelemetry_capture_query_stats_metric", | ||
| run_name="__main__", | ||
| ) | ||
|
|
||
| assert "Elapsed Milliseconds: 1.20" in capsys.readouterr().out | ||
|
|
||
|
|
||
| @conftest.live_only | ||
| def test_capture_query_stats_metric_live(capsys: pytest.CaptureFixture[str]) -> None: | ||
| with mock.patch(OTLP_METRIC_EXPORTER) as mock_exp: | ||
| _configure_mock_exporter(mock_exp) | ||
| opentelemetry_capture_query_stats_metric.capture_query_stats_metric( | ||
| conftest.LIVE_PROJECT_ID, | ||
| conftest.LIVE_INSTANCE_ID, | ||
| conftest.LIVE_DATABASE_ID, | ||
| ) | ||
|
|
||
| out, err = capsys.readouterr() | ||
| assert err == "" | ||
| assert conftest.LIVE_PROJECT_ID not in out | ||
| assert "Elapsed Milliseconds:" in out | ||
| assert "Row Count:" in out |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
To ensure robust execution and accurate error reporting, we should implement two improvements:
results.statsandresults.stats.query_statsare populated before accessing them to avoid a potentialAttributeError.client.projectso that if the project ID is inferred from Application Default Credentials (ADC), the correct ID is displayed in error messages instead ofNone.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks!
results.statsis already guarded (results.stats.query_stats if results.stats else {}), and we intentionally omit project identifiers (project_id/client.project) fromstdoutandstderroutput so project metadata is not echoed in logs.