Repository navigation
feat(spanner): Add table and index AddSplitPoints pre-splitting sample #14957
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/spanner-create-database
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| # Copyright 2026 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # [START spanner_database_add_split_points] | ||
| import sys | ||
|
|
||
| from google.api_core import exceptions | ||
| from google.cloud import spanner_admin_database_v1 | ||
|
|
||
|
|
||
| def add_split_points(project_id: str, instance_id: str, database_id: str) -> None: | ||
| """Pre-splits a Spanner table and secondary index via AddSplitPoints. | ||
|
|
||
| Args: | ||
| project_id: The Google Cloud project ID. | ||
| instance_id: The Spanner instance ID. | ||
| database_id: The Spanner database ID containing Singers. | ||
| """ | ||
| client = spanner_admin_database_v1.DatabaseAdminClient() | ||
| database_path = client.database_path(project_id, instance_id, database_id) | ||
| index_name = "SingersByFirstLastName" | ||
| ddl_request = spanner_admin_database_v1.UpdateDatabaseDdlRequest( | ||
| database=database_path, | ||
| statements=[f""" | ||
| CREATE INDEX IF NOT EXISTS {index_name} | ||
| ON Singers(FirstName, LastName) | ||
| """], | ||
| ) | ||
|
|
||
| table_key = spanner_admin_database_v1.SplitPoints.Key(key_parts=["42"]) | ||
| idx_only_key = spanner_admin_database_v1.SplitPoints.Key(key_parts=["John", "Doe"]) | ||
| idx_prefix_key = spanner_admin_database_v1.SplitPoints.Key( | ||
| key_parts=["Jane", "Doe"] | ||
| ) | ||
| idx_table_key = spanner_admin_database_v1.SplitPoints.Key(key_parts=["38"]) | ||
|
|
||
| split_points = [ | ||
| spanner_admin_database_v1.SplitPoints(table="Singers", keys=[table_key]), | ||
| spanner_admin_database_v1.SplitPoints(index=index_name, keys=[idx_only_key]), | ||
| spanner_admin_database_v1.SplitPoints( | ||
| index=index_name, keys=[idx_prefix_key, idx_table_key] | ||
| ), | ||
| ] | ||
| split_request = spanner_admin_database_v1.AddSplitPointsRequest( | ||
| database=database_path, | ||
| split_points=split_points, | ||
| ) | ||
|
|
||
| try: | ||
| ddl_op = client.update_database_ddl(request=ddl_request) | ||
| print(f"Waiting for index '{index_name}' before adding split points...") | ||
| ddl_op.result(timeout=300) | ||
| client.add_split_points(request=split_request) | ||
| except exceptions.ResourceExhausted as err: | ||
| print( | ||
| f"error: split-point quota exceeded on database '{database_id}' " | ||
| f"in instance '{instance_id}': {err.message}", | ||
| file=sys.stderr, | ||
| ) | ||
| except TimeoutError as err: | ||
| print( | ||
| f"error: timed out waiting for the operation on database '{database_id}' " | ||
| f"to complete; the operation may still be running: {err}", | ||
| file=sys.stderr, | ||
| ) | ||
| except exceptions.GoogleAPICallError as err: | ||
| print( | ||
| f"error: failed to add split points on database '{database_id}' " | ||
| f"in instance '{instance_id}': {err.message}", | ||
| file=sys.stderr, | ||
| ) | ||
| else: | ||
| print(f"Resource ID: {database_id}") | ||
| print(f"Split Points Added: {len(split_points)}") | ||
|
|
||
|
|
||
| # [END spanner_database_add_split_points] | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| import argparse | ||
| import os | ||
|
|
||
| parser = argparse.ArgumentParser( | ||
| description="Add table and index split points to a Spanner database." | ||
| ) | ||
| parser.add_argument( | ||
| "--project-id", | ||
| default=os.environ.get("GOOGLE_CLOUD_PROJECT", "example-project-id"), | ||
| help="Google Cloud project ID (default: $GOOGLE_CLOUD_PROJECT)", | ||
| ) | ||
| parser.add_argument( | ||
| "--instance-id", | ||
| default=os.environ.get("SPANNER_INSTANCE_ID", "example-instance"), | ||
| help="Spanner instance ID (default: $SPANNER_INSTANCE_ID)", | ||
| ) | ||
| parser.add_argument( | ||
| "--database-id", | ||
| default=os.environ.get("SPANNER_DATABASE_ID", "example-db"), | ||
| help="Spanner database ID (default: $SPANNER_DATABASE_ID)", | ||
| ) | ||
| args = parser.parse_args() | ||
|
|
||
| add_split_points( | ||
| project_id=args.project_id, | ||
| instance_id=args.instance_id, | ||
| database_id=args.database_id, | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| # Copyright 2026 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import runpy | ||
| from unittest import mock | ||
| import uuid | ||
|
|
||
| from google.api_core import exceptions | ||
| from google.cloud import spanner_admin_database_v1 | ||
| import pytest | ||
| from spanner_database_admin import create_database | ||
| from spanner_database_admin import database_add_split_points | ||
|
|
||
| from . import conftest | ||
|
|
||
| PROJECT_ID = "example-project-id" | ||
| INSTANCE_ID = "example-instance" | ||
| DATABASE_ID = "example-db" | ||
| ADMIN_CLIENT = "google.cloud.spanner_admin_database_v1.DatabaseAdminClient" | ||
|
|
||
|
|
||
| @mock.patch(ADMIN_CLIENT) | ||
| def test_add_split_points_success( | ||
| mock_client_class: mock.MagicMock, capsys: pytest.CaptureFixture[str] | ||
| ) -> None: | ||
| mock_client_class.return_value.database_path.return_value = ( | ||
| f"projects/{PROJECT_ID}/instances/{INSTANCE_ID}/databases/{DATABASE_ID}" | ||
| ) | ||
|
|
||
| database_add_split_points.add_split_points(PROJECT_ID, INSTANCE_ID, DATABASE_ID) | ||
|
|
||
| mock_client_class.return_value.add_split_points.assert_called_once() | ||
| out, err = capsys.readouterr() | ||
| assert f"Resource ID: {DATABASE_ID}" in out | ||
| assert "Split Points Added: 3" in out | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks! The split-point configuration is unchanged (see the reply on the sample), so the expected count stays as is. |
||
| assert PROJECT_ID not in out | ||
| assert err == "" | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "error, expected_prefix", | ||
| [ | ||
| (TimeoutError("deadline"), "error: timed out waiting for the operation"), | ||
| ( | ||
| exceptions.ResourceExhausted("quota"), | ||
| "error: split-point quota exceeded", | ||
| ), | ||
| ( | ||
| exceptions.InvalidArgument("bad key"), | ||
| "error: failed to add split points", | ||
| ), | ||
| ], | ||
| ) | ||
| @mock.patch(ADMIN_CLIENT) | ||
| def test_add_split_points_errors( | ||
| mock_client_class: mock.MagicMock, | ||
| capsys: pytest.CaptureFixture[str], | ||
| error: Exception, | ||
| expected_prefix: str, | ||
| ) -> None: | ||
| mock_client_class.return_value.database_path.return_value = ( | ||
| f"projects/{PROJECT_ID}/instances/{INSTANCE_ID}/databases/{DATABASE_ID}" | ||
| ) | ||
| mock_client_class.return_value.add_split_points.side_effect = error | ||
|
|
||
| database_add_split_points.add_split_points(PROJECT_ID, INSTANCE_ID, DATABASE_ID) | ||
|
|
||
| _out, err = capsys.readouterr() | ||
| assert err.startswith(expected_prefix) | ||
| assert PROJECT_ID not in err | ||
|
|
||
|
|
||
| @mock.patch(ADMIN_CLIENT) | ||
| def test_add_split_points_cli_entrypoint( | ||
| mock_client_class: mock.MagicMock, | ||
| monkeypatch: pytest.MonkeyPatch, | ||
| capsys: pytest.CaptureFixture[str], | ||
| ) -> None: | ||
| mock_client_class.return_value.database_path.return_value = ( | ||
| f"projects/{PROJECT_ID}/instances/{INSTANCE_ID}/databases/{DATABASE_ID}" | ||
| ) | ||
| monkeypatch.setattr( | ||
| "sys.argv", | ||
| [ | ||
| "database_add_split_points.py", | ||
| "--project-id", | ||
| PROJECT_ID, | ||
| "--instance-id", | ||
| INSTANCE_ID, | ||
| "--database-id", | ||
| DATABASE_ID, | ||
| ], | ||
| ) | ||
|
|
||
| runpy.run_module( | ||
| "spanner_database_admin.database_add_split_points", run_name="__main__" | ||
| ) | ||
|
|
||
| assert "Split Points Added: 3" in capsys.readouterr().out | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks! The split-point configuration is unchanged (see the reply on the sample), so the expected count stays as is. |
||
|
|
||
|
|
||
| @conftest.live_only | ||
| def test_add_split_points_live(capsys: pytest.CaptureFixture[str]) -> None: | ||
| admin = spanner_admin_database_v1.DatabaseAdminClient() | ||
| temp_db_id = f"example-spl-{uuid.uuid4().hex[:6]}" | ||
| db_path = admin.database_path( | ||
| conftest.LIVE_PROJECT_ID, conftest.LIVE_INSTANCE_ID, temp_db_id | ||
| ) | ||
| try: | ||
| create_database.create_database( | ||
| conftest.LIVE_PROJECT_ID, conftest.LIVE_INSTANCE_ID, temp_db_id | ||
| ) | ||
| capsys.readouterr() | ||
| database_add_split_points.add_split_points( | ||
| conftest.LIVE_PROJECT_ID, conftest.LIVE_INSTANCE_ID, temp_db_id | ||
| ) | ||
| out, err = capsys.readouterr() | ||
| assert err == "" | ||
| assert conftest.LIVE_PROJECT_ID not in out | ||
| assert "Split Points Added: 3" in out | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks! The split-point configuration is unchanged (see the reply on the sample), so the expected count stays as is. |
||
| finally: | ||
| try: | ||
| admin.drop_database(database=db_path) | ||
| except exceptions.NotFound: | ||
| pass | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The split points configuration contains a logical error and redundancy:
idx_table_key(with value["38"]) represents a primary key for theSingerstable (similar totable_keywith value["42"]). However, it is being passed as a split point for the secondary indexSingersByFirstLastName(which expects[FirstName, LastName]key parts).SplitPointsobject rather than creating multiple separateSplitPointsentries for the same index.Grouping the table keys under the
Singerstable and the index keys under theSingersByFirstLastNameindex makes the sample correct, cleaner, and easier to understand.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks! Keeping this as is: an index split point can be given as an index key followed by a table key, which is what the last entry shows, so the two keys belong together and the count of split points stays at three.