From 5f47f08eccd88d2fe12a3cc3c70b55fba5ea3cf8 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Tue, 17 Nov 2026 23:46:01 +0000 Subject: [PATCH 1/3] feat(spanner): Add change stream TVF streaming read sample Change stream TVF queries return a sentinel read_timestamp that overflows veneer snapshots, requiring single-use ExecuteStreamingSql via GAPIC. --- .../read_change_stream.py | 123 +++++++++++++++++ .../tests/read_change_stream_test.py | 125 ++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 spanner/change_streams/src/spanner_change_streams/read_change_stream.py create mode 100644 spanner/change_streams/tests/read_change_stream_test.py diff --git a/spanner/change_streams/src/spanner_change_streams/read_change_stream.py b/spanner/change_streams/src/spanner_change_streams/read_change_stream.py new file mode 100644 index 0000000000..13b80876e5 --- /dev/null +++ b/spanner/change_streams/src/spanner_change_streams/read_change_stream.py @@ -0,0 +1,123 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START spanner_read_change_stream] +from datetime import datetime, timedelta, timezone +import sys + +from google.api_core import exceptions +from google.cloud import spanner_v1 +from google.protobuf import struct_pb2 + + +def read_change_stream(project_id: str, instance_id: str, database_id: str) -> None: + """Streams initial partition records from a Spanner change stream TVF. + + Change stream TVF queries require a single-use strong read executed via the + ExecuteStreamingSql RPC. + + Args: + project_id: The Google Cloud project ID. + instance_id: The Spanner instance ID. + database_id: The Spanner database ID containing SingersAndAlbumsStream. + """ + client = spanner_v1.SpannerClient() + database_name = ( + f"projects/{project_id}/instances/{instance_id}/databases/{database_id}" + ) + now = datetime.now(timezone.utc) + end_ts = (now - timedelta(seconds=2)).isoformat().replace("+00:00", "Z") + start_ts = (now - timedelta(seconds=7)).isoformat().replace("+00:00", "Z") + + params = struct_pb2.Struct() + params.update({"start_ts": start_ts, "end_ts": end_ts}) + ts_type = spanner_v1.Type(code=spanner_v1.TypeCode.TIMESTAMP) + read_only = spanner_v1.TransactionOptions.ReadOnly(strong=True) + selector = spanner_v1.TransactionSelector( + single_use=spanner_v1.TransactionOptions(read_only=read_only) + ) + sql = """ + SELECT ChangeRecord + FROM READ_SingersAndAlbumsStream( + start_timestamp => @start_ts, + end_timestamp => @end_ts, + partition_token => NULL, + heartbeat_milliseconds => 10000 + ) + """ + + try: + session_req = spanner_v1.CreateSessionRequest( + database=database_name, + session=spanner_v1.Session(multiplexed=True), + ) + session = client.create_session(request=session_req) + sql_req = spanner_v1.ExecuteSqlRequest( + session=session.name, + sql=sql, + params=params, + param_types={"start_ts": ts_type, "end_ts": ts_type}, + transaction=selector, + ) + chunks = list(client.execute_streaming_sql(request=sql_req)) + except exceptions.NotFound as err: + print( + f"error: database '{database_id}' on instance '{instance_id}' " + f"in project '{project_id}' was not found: {err.message}", + file=sys.stderr, + ) + except exceptions.GoogleAPICallError as err: + print( + f"error: change stream read failed on database '{database_id}' " + f"in instance '{instance_id}' for project '{project_id}': {err.message}", + file=sys.stderr, + ) + else: + print(f"Resource ID: {database_name}") + print("Change Stream: SingersAndAlbumsStream") + print(f"Stream Chunks Received: {len(chunks)}") + + +# [END spanner_read_change_stream] + + +if __name__ == "__main__": + import argparse + import os + + parser = argparse.ArgumentParser( + description="Read initial partition records from a Spanner change stream TVF." + ) + parser.add_argument( + "--project-id", + default=os.environ.get("GOOGLE_CLOUD_PROJECT", "example-project-id"), + help="Google Cloud project ID (default: $GOOGLE_CLOUD_PROJECT)", + ) + parser.add_argument( + "--instance-id", + default=os.environ.get("SPANNER_INSTANCE_ID", "example-instance"), + help="Spanner instance ID (default: $SPANNER_INSTANCE_ID)", + ) + parser.add_argument( + "--database-id", + default=os.environ.get("SPANNER_DATABASE_ID", "example-db"), + help="Spanner database ID (default: $SPANNER_DATABASE_ID)", + ) + args = parser.parse_args() + + read_change_stream( + project_id=args.project_id, + instance_id=args.instance_id, + database_id=args.database_id, + ) diff --git a/spanner/change_streams/tests/read_change_stream_test.py b/spanner/change_streams/tests/read_change_stream_test.py new file mode 100644 index 0000000000..b08108c0a8 --- /dev/null +++ b/spanner/change_streams/tests/read_change_stream_test.py @@ -0,0 +1,125 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import runpy +from unittest import mock + +from google.api_core import exceptions +from google.cloud import spanner_v1 +import pytest +from spanner_change_streams import read_change_stream + +from . import conftest + +PROJECT_ID = "example-project-id" +INSTANCE_ID = "example-instance" +DATABASE_ID = "example-db" +DATABASE_NAME = f"projects/{PROJECT_ID}/instances/{INSTANCE_ID}/databases/{DATABASE_ID}" +SESSION_NAME = f"{DATABASE_NAME}/sessions/mux-1" +GAPIC_CLIENT = "google.cloud.spanner_v1.SpannerClient" + + +@mock.patch(GAPIC_CLIENT) +def test_read_change_stream_success( + mock_client_class: mock.MagicMock, capsys: pytest.CaptureFixture[str] +) -> None: + client = mock_client_class.return_value + client.create_session.return_value = spanner_v1.Session( + name=SESSION_NAME, multiplexed=True + ) + client.execute_streaming_sql.return_value = [spanner_v1.PartialResultSet()] + + read_change_stream.read_change_stream(PROJECT_ID, INSTANCE_ID, DATABASE_ID) + + req = client.execute_streaming_sql.call_args.kwargs["request"] + assert "READ_SingersAndAlbumsStream" in req.sql + assert req.transaction.single_use.read_only.strong is True + out, err = capsys.readouterr() + assert f"Resource ID: {DATABASE_NAME}" in out + assert "Change Stream: SingersAndAlbumsStream" in out + assert "Stream Chunks Received: 1" in out + assert err == "" + + +@pytest.mark.parametrize( + "error, expected_prefix", + [ + ( + exceptions.NotFound("db missing"), + "error: database 'example-db' on instance 'example-instance'", + ), + ( + exceptions.InvalidArgument("bad stream"), + "error: change stream read failed", + ), + ], +) +@mock.patch(GAPIC_CLIENT) +def test_read_change_stream_errors( + mock_client_class: mock.MagicMock, + capsys: pytest.CaptureFixture[str], + error: Exception, + expected_prefix: str, +) -> None: + client = mock_client_class.return_value + client.create_session.side_effect = error + + read_change_stream.read_change_stream(PROJECT_ID, INSTANCE_ID, DATABASE_ID) + + out, err = capsys.readouterr() + assert out == "" + assert err.startswith(expected_prefix) + + +@mock.patch(GAPIC_CLIENT) +def test_read_change_stream_cli_entrypoint( + mock_client_class: mock.MagicMock, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + client = mock_client_class.return_value + client.create_session.return_value = spanner_v1.Session( + name=SESSION_NAME, multiplexed=True + ) + client.execute_streaming_sql.return_value = [spanner_v1.PartialResultSet()] + monkeypatch.setattr( + "sys.argv", + [ + "read_change_stream.py", + "--project-id", + PROJECT_ID, + "--instance-id", + INSTANCE_ID, + "--database-id", + DATABASE_ID, + ], + ) + + runpy.run_module("spanner_change_streams.read_change_stream", run_name="__main__") + + assert "Change Stream: SingersAndAlbumsStream" in capsys.readouterr().out + + +@conftest.live_only +def test_read_change_stream_live(capsys: pytest.CaptureFixture[str]) -> None: + read_change_stream.read_change_stream( + conftest.LIVE_PROJECT_ID, + conftest.LIVE_INSTANCE_ID, + conftest.LIVE_DATABASE_ID, + ) + + out, err = capsys.readouterr() + assert err == "" + assert "Change Stream: SingersAndAlbumsStream" in out + assert "Stream Chunks Received:" in out From 215d90518bc52caf9c23905ab296574144fa1ef8 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 18 Nov 2026 00:38:22 +0000 Subject: [PATCH 2/3] fix(spanner): Clarify multiplexed session lifecycle and omit project_id Documents that multiplexed sessions are Spanner-managed and omits project identifiers from stdout and stderr. --- .../src/spanner_change_streams/read_change_stream.py | 7 ++++--- spanner/change_streams/tests/read_change_stream_test.py | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/spanner/change_streams/src/spanner_change_streams/read_change_stream.py b/spanner/change_streams/src/spanner_change_streams/read_change_stream.py index 13b80876e5..d0b416a7c2 100644 --- a/spanner/change_streams/src/spanner_change_streams/read_change_stream.py +++ b/spanner/change_streams/src/spanner_change_streams/read_change_stream.py @@ -58,6 +58,7 @@ def read_change_stream(project_id: str, instance_id: str, database_id: str) -> N """ try: + # Multiplexed sessions are managed by Spanner and do not support DeleteSession. session_req = spanner_v1.CreateSessionRequest( database=database_name, session=spanner_v1.Session(multiplexed=True), @@ -74,17 +75,17 @@ def read_change_stream(project_id: str, instance_id: str, database_id: str) -> N except exceptions.NotFound as err: print( f"error: database '{database_id}' on instance '{instance_id}' " - f"in project '{project_id}' was not found: {err.message}", + f"was not found: {err.message}", file=sys.stderr, ) except exceptions.GoogleAPICallError as err: print( f"error: change stream read failed on database '{database_id}' " - f"in instance '{instance_id}' for project '{project_id}': {err.message}", + f"in instance '{instance_id}': {err.message}", file=sys.stderr, ) else: - print(f"Resource ID: {database_name}") + print(f"Resource ID: {database_id}") print("Change Stream: SingersAndAlbumsStream") print(f"Stream Chunks Received: {len(chunks)}") diff --git a/spanner/change_streams/tests/read_change_stream_test.py b/spanner/change_streams/tests/read_change_stream_test.py index b08108c0a8..79b0dde4d7 100644 --- a/spanner/change_streams/tests/read_change_stream_test.py +++ b/spanner/change_streams/tests/read_change_stream_test.py @@ -46,7 +46,8 @@ def test_read_change_stream_success( assert "READ_SingersAndAlbumsStream" in req.sql assert req.transaction.single_use.read_only.strong is True out, err = capsys.readouterr() - assert f"Resource ID: {DATABASE_NAME}" in out + assert f"Resource ID: {DATABASE_ID}" in out + assert PROJECT_ID not in out assert "Change Stream: SingersAndAlbumsStream" in out assert "Stream Chunks Received: 1" in out assert err == "" @@ -80,6 +81,8 @@ def test_read_change_stream_errors( out, err = capsys.readouterr() assert out == "" assert err.startswith(expected_prefix) + assert PROJECT_ID not in err + assert PROJECT_ID not in err @mock.patch(GAPIC_CLIENT) From e036644b3e65aa5f40af9f277a84cfff27ec0ed6 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 18 Nov 2026 02:42:53 +0000 Subject: [PATCH 3/3] fix(spanner): Remove hardcoded static stdout lines from sample output Prints only runtime resource and query results instead of hardcoded string literals. --- .../src/spanner_change_streams/read_change_stream.py | 1 - spanner/change_streams/tests/read_change_stream_test.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/spanner/change_streams/src/spanner_change_streams/read_change_stream.py b/spanner/change_streams/src/spanner_change_streams/read_change_stream.py index d0b416a7c2..96922a7a4c 100644 --- a/spanner/change_streams/src/spanner_change_streams/read_change_stream.py +++ b/spanner/change_streams/src/spanner_change_streams/read_change_stream.py @@ -86,7 +86,6 @@ def read_change_stream(project_id: str, instance_id: str, database_id: str) -> N ) else: print(f"Resource ID: {database_id}") - print("Change Stream: SingersAndAlbumsStream") print(f"Stream Chunks Received: {len(chunks)}") diff --git a/spanner/change_streams/tests/read_change_stream_test.py b/spanner/change_streams/tests/read_change_stream_test.py index 79b0dde4d7..c806b152ad 100644 --- a/spanner/change_streams/tests/read_change_stream_test.py +++ b/spanner/change_streams/tests/read_change_stream_test.py @@ -48,7 +48,6 @@ def test_read_change_stream_success( out, err = capsys.readouterr() assert f"Resource ID: {DATABASE_ID}" in out assert PROJECT_ID not in out - assert "Change Stream: SingersAndAlbumsStream" in out assert "Stream Chunks Received: 1" in out assert err == "" @@ -111,7 +110,7 @@ def test_read_change_stream_cli_entrypoint( runpy.run_module("spanner_change_streams.read_change_stream", run_name="__main__") - assert "Change Stream: SingersAndAlbumsStream" in capsys.readouterr().out + assert "Stream Chunks Received: 1" in capsys.readouterr().out @conftest.live_only @@ -124,5 +123,5 @@ def test_read_change_stream_live(capsys: pytest.CaptureFixture[str]) -> None: out, err = capsys.readouterr() assert err == "" - assert "Change Stream: SingersAndAlbumsStream" in out + assert conftest.LIVE_PROJECT_ID not in out assert "Stream Chunks Received:" in out