diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index cec50d1a..efd89b39 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -57,3 +57,54 @@ def run(): assert result.sizes["lon"] == grid.sizes["lon"] expected = grid["temperature"].mean("time") np.testing.assert_allclose(result["mean_temp"].values, expected.values) + + +@pytest.fixture(scope="module") +def store(tmp_path_factory) -> str: + """A Zarr store of 20 variables, one time step per chunk, NaN on land. + + It's shaped like an ocean model's training data, whose loader reads a + window of time steps of every variable. + """ + rng = np.random.default_rng(0) + time, lat, lon = 8, 90, 180 + land = rng.random((lat, lon)) < 0.3 + data = {} + for v in range(20): + values = rng.normal(size=(time, lat, lon)).astype("float32") + values[:, land] = np.nan + data[f"v{v:02d}"] = (("time", "lat", "lon"), values) + ds = xr.Dataset( + data, + coords={ + "time": np.arange(time), + "lat": np.arange(lat), + "lon": np.arange(lon), + }, + ) + path = str(tmp_path_factory.mktemp("zarr") / "ocean.zarr") + ds.chunk({"time": 1}).to_zarr(path, consolidated=False) + return path + + +@pytest.mark.parametrize("dask", [False, True], ids=["lazy", "dask"]) +def test_read_a_window_of_every_variable_from_zarr(benchmark, store, dask): + """A training loader's read: 4 time steps of all 20 variables.""" + ds = xr.open_zarr(store, chunks={} if dask else None, consolidated=False) + ctx = Context() + ctx.from_dataset("ocean", ds, chunks={"time": 1}) + + def run(): + return ctx.sql( + "SELECT * FROM ocean WHERE time BETWEEN 2 AND 5 " + "ORDER BY time, lat, lon" + ).to_arrow_table() + + rows = benchmark(run) + window = ds.isel(time=slice(2, 6)) + assert rows.num_rows == window.sizes["time"] * 90 * 180 + np.testing.assert_allclose( + np.asarray(rows["v07"].to_numpy(zero_copy_only=False), "float64"), + window["v07"].values.ravel(), + rtol=1e-6, + ) diff --git a/tests/test_sql.py b/tests/test_sql.py index b65288d1..1b609d2c 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -846,3 +846,21 @@ def test_dimension_no_data_variable_has_registers(): ctx.from_dataset("t", ds) assert "t" in ctx.catalog().schema().names() + + +def test_a_multiindex_dimension_reads_as_its_levels(): + index = pd.MultiIndex.from_arrays( + [[1, 1, 2], [10, 20, 10]], names=["a", "b"] + ) + ds = xr.Dataset( + {"v": ("point", np.arange(3.0))}, + coords=xr.Coordinates.from_pandas_multiindex(index, "point"), + ) + ctx = XarrayContext() + ctx.from_dataset("m", ds) + + rows = ctx.sql("SELECT point, v FROM m ORDER BY v").to_arrow_table() + assert rows.to_pydict() == { + "point": [[1, 10], [1, 20], [2, 10]], + "v": [0.0, 1.0, 2.0], + } diff --git a/xarray_sql/df.py b/xarray_sql/df.py index 4c74db56..3e97102c 100644 --- a/xarray_sql/df.py +++ b/xarray_sql/df.py @@ -1,7 +1,9 @@ import itertools +import os import sys from collections import defaultdict from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor from typing import Any import numpy as np @@ -461,6 +463,44 @@ def _as_single_array(values, type: pa.DataType, *, from_pandas: bool = False): return arr +_LOADS = ThreadPoolExecutor() if _CAN_START_THREADS else None +"""Threads that read a partition's variables from storage at once.""" + + +def _new_loads() -> None: + """Gives a forked child its own threads, since none of the parent's + survive the fork.""" + global _LOADS + _LOADS = ThreadPoolExecutor() + + +if _CAN_START_THREADS: + os.register_at_fork(after_in_child=_new_loads) + + +def _load(ds: xr.Dataset, names: list[str]) -> xr.Dataset: + """Loads all of a partition's variables at once, rather than in turn. + + Reading them one by one waits for each read before starting the next, + which costs a round trip per variable on remote storage. A Dask Dataset + loads in one compute, which schedules every variable's chunks together. + Lazily indexed variables, as ``xr.open_zarr(chunks=None)`` gives, are + read on threads, one each. + + Only the variables named in *names*, and indexed coordinates, which are + already in memory, are loaded; a coordinate the scan doesn't emit, such + as a 2-D latitude, is dropped rather than read. + """ + ds = ds.drop_vars( + [n for n in ds.variables if n not in names and n not in ds.xindexes] + ) + variables = list(ds.variables.values()) + if _LOADS is None or any(v.chunks is not None for v in variables): + return ds.load() + list(_LOADS.map(lambda v: v.load(), variables)) + return ds + + def iter_record_batches( ds: xr.Dataset, schema: pa.Schema, @@ -489,6 +529,7 @@ def iter_record_batches( RecordBatches in schema column order, covering all rows of the partition exactly once. """ + ds = _load(ds, schema.names) if ds.data_vars: first_var = next(iter(ds.data_vars.values())) dim_names = list(first_var.dims)