Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions tests/test_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,54 @@ def run():
assert result.sizes["lon"] == grid.sizes["lon"]
expected = grid["temperature"].mean("time")
np.testing.assert_allclose(result["mean_temp"].values, expected.values)


@pytest.fixture(scope="module")
def store(tmp_path_factory) -> str:
"""A Zarr store of 20 variables, one time step per chunk, NaN on land.

It's shaped like an ocean model's training data, whose loader reads a
window of time steps of every variable.
"""
rng = np.random.default_rng(0)
time, lat, lon = 8, 90, 180
land = rng.random((lat, lon)) < 0.3
data = {}
for v in range(20):
values = rng.normal(size=(time, lat, lon)).astype("float32")
values[:, land] = np.nan
data[f"v{v:02d}"] = (("time", "lat", "lon"), values)
ds = xr.Dataset(
data,
coords={
"time": np.arange(time),
"lat": np.arange(lat),
"lon": np.arange(lon),
},
)
path = str(tmp_path_factory.mktemp("zarr") / "ocean.zarr")
ds.chunk({"time": 1}).to_zarr(path, consolidated=False)
return path


@pytest.mark.parametrize("dask", [False, True], ids=["lazy", "dask"])
def test_read_a_window_of_every_variable_from_zarr(benchmark, store, dask):
"""A training loader's read: 4 time steps of all 20 variables."""
ds = xr.open_zarr(store, chunks={} if dask else None, consolidated=False)
ctx = Context()
ctx.from_dataset("ocean", ds, chunks={"time": 1})

def run():
return ctx.sql(
"SELECT * FROM ocean WHERE time BETWEEN 2 AND 5 "
"ORDER BY time, lat, lon"
).to_arrow_table()

rows = benchmark(run)
window = ds.isel(time=slice(2, 6))
assert rows.num_rows == window.sizes["time"] * 90 * 180
np.testing.assert_allclose(
np.asarray(rows["v07"].to_numpy(zero_copy_only=False), "float64"),
window["v07"].values.ravel(),
rtol=1e-6,
)
18 changes: 18 additions & 0 deletions tests/test_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -846,3 +846,21 @@ def test_dimension_no_data_variable_has_registers():
ctx.from_dataset("t", ds)

assert "t" in ctx.catalog().schema().names()


def test_a_multiindex_dimension_reads_as_its_levels():
index = pd.MultiIndex.from_arrays(
[[1, 1, 2], [10, 20, 10]], names=["a", "b"]
)
ds = xr.Dataset(
{"v": ("point", np.arange(3.0))},
coords=xr.Coordinates.from_pandas_multiindex(index, "point"),
)
ctx = XarrayContext()
ctx.from_dataset("m", ds)

rows = ctx.sql("SELECT point, v FROM m ORDER BY v").to_arrow_table()
assert rows.to_pydict() == {
"point": [[1, 10], [1, 20], [2, 10]],
"v": [0.0, 1.0, 2.0],
}
41 changes: 41 additions & 0 deletions xarray_sql/df.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import itertools
import os
import sys
from collections import defaultdict
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor
from typing import Any

import numpy as np
Expand Down Expand Up @@ -461,6 +463,44 @@ def _as_single_array(values, type: pa.DataType, *, from_pandas: bool = False):
return arr


_LOADS = ThreadPoolExecutor() if _CAN_START_THREADS else None
"""Threads that read a partition's variables from storage at once."""


def _new_loads() -> None:
"""Gives a forked child its own threads, since none of the parent's
survive the fork."""
global _LOADS
_LOADS = ThreadPoolExecutor()


if _CAN_START_THREADS:
os.register_at_fork(after_in_child=_new_loads)


def _load(ds: xr.Dataset, names: list[str]) -> xr.Dataset:
"""Loads all of a partition's variables at once, rather than in turn.

Reading them one by one waits for each read before starting the next,
which costs a round trip per variable on remote storage. A Dask Dataset
loads in one compute, which schedules every variable's chunks together.
Lazily indexed variables, as ``xr.open_zarr(chunks=None)`` gives, are
read on threads, one each.

Only the variables named in *names*, and indexed coordinates, which are
already in memory, are loaded; a coordinate the scan doesn't emit, such
as a 2-D latitude, is dropped rather than read.
"""
ds = ds.drop_vars(
[n for n in ds.variables if n not in names and n not in ds.xindexes]
)
variables = list(ds.variables.values())
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if _LOADS is None or any(v.chunks is not None for v in variables):
return ds.load()
list(_LOADS.map(lambda v: v.load(), variables))
return ds


def iter_record_batches(
ds: xr.Dataset,
schema: pa.Schema,
Expand Down Expand Up @@ -489,6 +529,7 @@ def iter_record_batches(
RecordBatches in schema column order, covering all rows of the
partition exactly once.
"""
ds = _load(ds, schema.names)
if ds.data_vars:
first_var = next(iter(ds.data_vars.values()))
dim_names = list(first_var.dims)
Expand Down
Loading