From 8dd7af12e986b73f18ce5d58b309bb77ebd718f3 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Thu, 19 Nov 2026 09:36:42 -0800 Subject: [PATCH 1/3] Load a partition's variables at once, not one by one Each partition read its data variables one at a time, with one Dask compute or one storage read per variable, each waiting for the last. Now a Dask partition loads in one compute, and a lazily indexed one, as xr.open_zarr(chunks=None) gives, reads its variables on threads. A Zarr scan benchmark tracks the read in CI, with and without Dask. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NyN7idACeDMh94qrnFMu6h --- tests/test_benchmarks.py | 51 ++++++++++++++++++++++++++++++++++++++++ xarray_sql/df.py | 22 +++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index cec50d1a..5a1b3fd3 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -57,3 +57,54 @@ def run(): assert result.sizes["lon"] == grid.sizes["lon"] expected = grid["temperature"].mean("time") np.testing.assert_allclose(result["mean_temp"].values, expected.values) + + +@pytest.fixture(scope="module") +def store(tmp_path_factory) -> str: + """A Zarr store of 20 variables, one time step per chunk, NaN on land. + + It's shaped like an ocean model's training data, whose loader reads a + window of time steps of every variable. + """ + rng = np.random.default_rng(0) + time, lat, lon = 8, 90, 180 + land = rng.random((lat, lon)) < 0.3 + data = {} + for v in range(20): + values = rng.normal(size=(time, lat, lon)).astype("float32") + values[:, land] = np.nan + data[f"v{v:02d}"] = (("time", "lat", "lon"), values) + ds = xr.Dataset( + data, + coords={ + "time": np.arange(time), + "lat": np.arange(lat), + "lon": np.arange(lon), + }, + ) + path = str(tmp_path_factory.mktemp("zarr") / "ocean.zarr") + ds.chunk({"time": 1}).to_zarr(path, zarr_format=3, consolidated=False) + return path + + +@pytest.mark.parametrize("dask", [False, True], ids=["lazy", "dask"]) +def test_read_a_window_of_every_variable_from_zarr(benchmark, store, dask): + """A training loader's read: 4 time steps of all 20 variables.""" + ds = xr.open_zarr(store, chunks={} if dask else None, consolidated=False) + ctx = Context() + ctx.from_dataset("ocean", ds, chunks={"time": 1}) + + def run(): + return ctx.sql( + "SELECT * FROM ocean WHERE time BETWEEN 2 AND 5 " + "ORDER BY time, lat, lon" + ).to_arrow_table() + + rows = benchmark(run) + window = ds.isel(time=slice(2, 6)) + assert rows.num_rows == window.sizes["time"] * 90 * 180 + np.testing.assert_allclose( + np.asarray(rows["v07"].to_numpy(zero_copy_only=False), "float64"), + window["v07"].values.ravel(), + rtol=1e-6, + ) diff --git a/xarray_sql/df.py b/xarray_sql/df.py index 4c74db56..6b86b6f3 100644 --- a/xarray_sql/df.py +++ b/xarray_sql/df.py @@ -2,6 +2,7 @@ import sys from collections import defaultdict from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor from typing import Any import numpy as np @@ -461,6 +462,26 @@ def _as_single_array(values, type: pa.DataType, *, from_pandas: bool = False): return arr +_LOADS = ThreadPoolExecutor() if _CAN_START_THREADS else None +"""Threads that read a partition's variables from storage at once.""" + + +def _load(ds: xr.Dataset) -> xr.Dataset: + """Loads all of a partition's variables at once, rather than in turn. + + Reading them one by one waits for each read before starting the next, + which costs a round trip per variable on remote storage. A Dask Dataset + loads in one compute, which schedules every variable's chunks together. + Lazily indexed variables, as ``xr.open_zarr(chunks=None)`` gives, are + read on threads, one each. + """ + variables = list(ds.variables.values()) + if _LOADS is None or any(v.chunks is not None for v in variables): + return ds.load() + list(_LOADS.map(lambda v: v.load(), variables)) + return ds + + def iter_record_batches( ds: xr.Dataset, schema: pa.Schema, @@ -489,6 +510,7 @@ def iter_record_batches( RecordBatches in schema column order, covering all rows of the partition exactly once. """ + ds = _load(ds) if ds.data_vars: first_var = next(iter(ds.data_vars.values())) dim_names = list(first_var.dims) From 487ede942aff5a07647edaa4c7d1675dd7ccb163 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Thu, 19 Nov 2026 09:58:24 -0800 Subject: [PATCH 2/3] Load only the columns a scan emits, and write the benchmark's store in any Zarr format A partition loaded every variable it held, including coordinates the scan never emits, such as a 2-D latitude. And xarray 2024.7, which the package supports, has no zarr_format argument, nor does Zarr 2 write version 3, as CI's Python 3.10 job installs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NyN7idACeDMh94qrnFMu6h --- tests/test_benchmarks.py | 2 +- xarray_sql/df.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 5a1b3fd3..efd89b39 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -83,7 +83,7 @@ def store(tmp_path_factory) -> str: }, ) path = str(tmp_path_factory.mktemp("zarr") / "ocean.zarr") - ds.chunk({"time": 1}).to_zarr(path, zarr_format=3, consolidated=False) + ds.chunk({"time": 1}).to_zarr(path, consolidated=False) return path diff --git a/xarray_sql/df.py b/xarray_sql/df.py index 6b86b6f3..da9a63ef 100644 --- a/xarray_sql/df.py +++ b/xarray_sql/df.py @@ -466,7 +466,7 @@ def _as_single_array(values, type: pa.DataType, *, from_pandas: bool = False): """Threads that read a partition's variables from storage at once.""" -def _load(ds: xr.Dataset) -> xr.Dataset: +def _load(ds: xr.Dataset, names: list[str]) -> xr.Dataset: """Loads all of a partition's variables at once, rather than in turn. Reading them one by one waits for each read before starting the next, @@ -474,7 +474,14 @@ def _load(ds: xr.Dataset) -> xr.Dataset: loads in one compute, which schedules every variable's chunks together. Lazily indexed variables, as ``xr.open_zarr(chunks=None)`` gives, are read on threads, one each. + + Only the variables named in *names*, and the dimension coordinates, are + loaded; a coordinate the scan doesn't emit, such as a 2-D latitude, is + dropped rather than read. """ + ds = ds.drop_vars( + [n for n in ds.variables if n not in names and n not in ds.dims] + ) variables = list(ds.variables.values()) if _LOADS is None or any(v.chunks is not None for v in variables): return ds.load() @@ -510,7 +517,7 @@ def iter_record_batches( RecordBatches in schema column order, covering all rows of the partition exactly once. """ - ds = _load(ds) + ds = _load(ds, schema.names) if ds.data_vars: first_var = next(iter(ds.data_vars.values())) dim_names = list(first_var.dims) From 935f2df4b208529903cb9baed156e7dedc13b0ea Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Thu, 19 Nov 2026 10:20:14 -0800 Subject: [PATCH 3/3] Keep indexed coordinates in a partition, and new load threads after a fork Dropping every variable outside the schema also dropped a MultiIndex's levels, and with them its dimension coordinate, so a query on a MultiIndex dimension raised KeyError. Indexed coordinates are in memory already, so a partition keeps them all. A forked child inherits the thread pool but none of its threads, so it now starts its own. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NyN7idACeDMh94qrnFMu6h --- tests/test_sql.py | 18 ++++++++++++++++++ xarray_sql/df.py | 20 ++++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/tests/test_sql.py b/tests/test_sql.py index b65288d1..1b609d2c 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -846,3 +846,21 @@ def test_dimension_no_data_variable_has_registers(): ctx.from_dataset("t", ds) assert "t" in ctx.catalog().schema().names() + + +def test_a_multiindex_dimension_reads_as_its_levels(): + index = pd.MultiIndex.from_arrays( + [[1, 1, 2], [10, 20, 10]], names=["a", "b"] + ) + ds = xr.Dataset( + {"v": ("point", np.arange(3.0))}, + coords=xr.Coordinates.from_pandas_multiindex(index, "point"), + ) + ctx = XarrayContext() + ctx.from_dataset("m", ds) + + rows = ctx.sql("SELECT point, v FROM m ORDER BY v").to_arrow_table() + assert rows.to_pydict() == { + "point": [[1, 10], [1, 20], [2, 10]], + "v": [0.0, 1.0, 2.0], + } diff --git a/xarray_sql/df.py b/xarray_sql/df.py index da9a63ef..3e97102c 100644 --- a/xarray_sql/df.py +++ b/xarray_sql/df.py @@ -1,4 +1,5 @@ import itertools +import os import sys from collections import defaultdict from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping @@ -466,6 +467,17 @@ def _as_single_array(values, type: pa.DataType, *, from_pandas: bool = False): """Threads that read a partition's variables from storage at once.""" +def _new_loads() -> None: + """Gives a forked child its own threads, since none of the parent's + survive the fork.""" + global _LOADS + _LOADS = ThreadPoolExecutor() + + +if _CAN_START_THREADS: + os.register_at_fork(after_in_child=_new_loads) + + def _load(ds: xr.Dataset, names: list[str]) -> xr.Dataset: """Loads all of a partition's variables at once, rather than in turn. @@ -475,12 +487,12 @@ def _load(ds: xr.Dataset, names: list[str]) -> xr.Dataset: Lazily indexed variables, as ``xr.open_zarr(chunks=None)`` gives, are read on threads, one each. - Only the variables named in *names*, and the dimension coordinates, are - loaded; a coordinate the scan doesn't emit, such as a 2-D latitude, is - dropped rather than read. + Only the variables named in *names*, and indexed coordinates, which are + already in memory, are loaded; a coordinate the scan doesn't emit, such + as a 2-D latitude, is dropped rather than read. """ ds = ds.drop_vars( - [n for n in ds.variables if n not in names and n not in ds.dims] + [n for n in ds.variables if n not in names and n not in ds.xindexes] ) variables = list(ds.variables.values()) if _LOADS is None or any(v.chunks is not None for v in variables):