diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64206380..0304843d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,8 +81,10 @@ jobs: - name: Run unit tests env: VIRTUAL_ENV: ${{ github.workspace }}/.venv - # The ML demos run in their own job, below. - run: uv run --no-project pytest -v . -m "not integration" --ignore=tests/test_ml_demos.py + # The ML demos run in their own job, below. --benchmark-disable runs + # each benchmark once, as a test, so a PR that breaks one fails here + # rather than in the benchmarks job on main. + run: uv run --no-project pytest -v . -m "not integration" --benchmark-disable --ignore=tests/test_ml_demos.py ml-demos: # Each ML demo in benchmarks/ml, at its smallest size, checked against @@ -121,3 +123,76 @@ jobs: env: VIRTUAL_ENV: ${{ github.workspace }}/.venv run: uv run --no-project pytest -v tests/test_ml_demos.py + + benchmarks: + # tests/test_benchmarks.py, timed with pytest-benchmark (a pytest + # plugin that runs a function repeatedly and reports its timings) and + # stored on gh-pages by github-action-benchmark (an action that keeps + # each run's results as a history, charts it, and comments on a commit + # that looks like a regression). + name: "benchmarks" + runs-on: ubuntu-latest + # Only on main: a PR from a fork has no permission to push to + # gh-pages, and a dispatch from another branch shouldn't record a + # point into history that doesn't belong to main's timeline. + if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + permissions: + contents: write + deployments: write + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + id: rust-toolchain + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 + + - name: Configure sccache + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: ~/.cargo + key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('Cargo.lock') }} + + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + enable-cache: true + + - name: Install xarray_sql + run: uv sync --locked --dev --extra benchmark --no-install-package xarray-sql + - name: build rust + run: uv run --no-project maturin develop --uv + - name: Run benchmarks + env: + VIRTUAL_ENV: ${{ github.workspace }}/.venv + run: uv run --no-project pytest tests/test_benchmarks.py -m benchmark --benchmark-only --benchmark-json=benchmark.json + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: "Python benchmarks (pytest-benchmark)" + tool: "pytest" + output-file-path: benchmark.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # The action retries a push that gh-pages rejects (up to 10 times, + # re-cloning gh-pages each time), so it can share the branch with + # mike's pushes from docs.yml. A new point goes live when docs.yml + # next publishes gh-pages to Pages, which may be one push to main + # later if that publish checked out gh-pages before this push. + auto-push: true + # mike (docs.yml) owns dev/, 0.x/, latest/, index.html and + # versions.json on gh-pages, and replaces dev/ wholesale on + # every docs deploy; the default dev/bench would collide + # with that and get wiped out. Use a root-level path mike + # never touches. + benchmark-data-dir-path: benchmarks + # Hosted runners are noisy; keep this loose until there's + # enough history on gh-pages to pick a tighter threshold. + alert-threshold: "200%" + comment-on-alert: true + fail-on-alert: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e5220821..b82a3c43 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,6 +84,26 @@ MATURIN_PEP517_ARGS="--no-default-features --profile wasm" pyodide build -o dist On NixOS, run these commands in `nix run .#wasm-shell`. This shell is an FHS environment, which lets the prebuilt Emscripten and rustup binaries run. +### Benchmarks + +`tests/test_benchmarks.py` holds benchmarks written with +[pytest-benchmark](https://github.com/ionelmc/pytest-benchmark), a pytest +plugin that runs a function repeatedly and reports its timings. Each one +also checks its result, so it's a test too. To time them locally: + +```shell +uv run pytest tests/test_benchmarks.py --benchmark-only +``` + +CI's unit run runs each benchmark once, untimed, with `--benchmark-disable`. +On every push to main, the `benchmarks` job times them and stores the result +on the `gh-pages` branch with +[github-action-benchmark](https://github.com/benchmark-action/github-action-benchmark), +which keeps the history, charts it at +, and comments on a commit that +looks like a regression. A new point appears on the chart when the docs are +next published. + ## Before submitting a pull request... diff --git a/pyproject.toml b/pyproject.toml index d7b32771..f23f2489 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,11 @@ geo = [ ml = [ "ddxdb>=0.4.6,<0.5; sys_platform != 'emscripten'", ] +# Benchmarks timed with pytest-benchmark and tracked over time in CI +# (see tests/test_benchmarks.py and CONTRIBUTING.md). +benchmark = [ + "pytest-benchmark>=4.0", +] test = [ "adbc-driver-flightsql>=1.12", "adbc-driver-sqlite>=1.12", @@ -133,6 +138,7 @@ ignore_missing_imports = true dev = [ "xarray_sql[test]", "xarray_sql[docs]", + "xarray_sql[benchmark]", "py-spy>=0.4.0", "ruff>=0.15.10", "maturin>=1.9.1", @@ -156,4 +162,5 @@ cache-keys = [{file = "pyproject.toml"}, {file = "rust/Cargo.toml"}, {file = "** testpaths = ["tests"] markers = [ "integration: needs network (anonymous GCS); excluded from the CI unit run", + "benchmark: timed by pytest-benchmark in its own CI job; the CI unit run runs each once, untimed", ] diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py new file mode 100644 index 00000000..cec50d1a --- /dev/null +++ b/tests/test_benchmarks.py @@ -0,0 +1,59 @@ +"""A benchmark tracked over time in CI (see CONTRIBUTING.md). + +It uses the ``benchmark`` fixture of pytest-benchmark +(https://github.com/ionelmc/pytest-benchmark), a pytest plugin that runs +a function repeatedly and reports its timings. On every push to main, CI +stores the timings with github-action-benchmark +(https://github.com/benchmark-action/github-action-benchmark), which +keeps them as a history and flags a commit that looks like a regression. + +Starts small: one benchmark, over an in-memory Dataset that needs no +network, so a regression in xarray-sql, DataFusion or the in-memory +table adapter shows up in that history. +""" + +import numpy as np +import pytest +import xarray as xr + +from xarray_sql import Context + +pytestmark = pytest.mark.benchmark + + +@pytest.fixture(scope="module") +def grid() -> xr.Dataset: + """A synthetic (time, lat, lon) grid: 100 x 50 x 50 cells, no I/O.""" + rng = np.random.default_rng(0) + time, lat, lon = 100, 50, 50 + return xr.Dataset( + { + "temperature": ( + ("time", "lat", "lon"), + rng.normal(size=(time, lat, lon)), + ), + }, + coords={ + "time": np.arange(time), + "lat": np.arange(lat), + "lon": np.arange(lon), + }, + ) + + +def test_groupby_mean_over_time(benchmark, grid): + """A climatology query: mean over time, grouped by lat and lon.""" + ctx = Context() + ctx.from_dataset("grid", grid) + + def run(): + return ctx.sql( + "SELECT lat, lon, AVG(temperature) AS mean_temp " + 'FROM "grid" GROUP BY lat, lon ORDER BY lat, lon' + ).to_dataset() + + result = benchmark(run) + assert result.sizes["lat"] == grid.sizes["lat"] + assert result.sizes["lon"] == grid.sizes["lon"] + expected = grid["temperature"].mean("time") + np.testing.assert_allclose(result["mean_temp"].values, expected.values) diff --git a/uv.lock b/uv.lock index 1c7aa1dc..c9a60b0a 100644 --- a/uv.lock +++ b/uv.lock @@ -2578,6 +2578,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "py-cpuinfo2" +version = "10.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/97/a8b1ddada14c8280a047c0746f95cb05d94a31b1a331cea22bcdc2b2a82d/py_cpuinfo2-10.1.1.tar.gz", hash = "sha256:7861133863663f16e06eca63b12904ef100b5760415e92372dac0162799a4771", size = 100840, upload-time = "2026-03-25T21:49:40.797Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/0a/ba69d2dde1ae12ef1d389ea5a216384c5ff6ef7a1e7a48d1e9b6686f6790/py_cpuinfo2-10.1.1-py3-none-any.whl", hash = "sha256:adc53396bfb206e6498d078ec2ab407f85799ecd819584ac36a8f80a2d4d762d", size = 23791, upload-time = "2026-03-25T21:49:39.574Z" }, +] + [[package]] name = "py-spy" version = "0.4.0" @@ -2930,6 +2939,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, ] +[[package]] +name = "pytest-benchmark" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo2" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/d1/1bc61155de46f25473a06476ad1b3a94f96d40d49cfa98f82c1184117808/pytest_benchmark-5.3.1.tar.gz", hash = "sha256:c1ee70a58f9bfb1ac955cc634247a340b0df7bcb949071cfca396f16c6fc3516", size = 385467, upload-time = "2026-11-01T03:32:10.048Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/75/919ac85ad3f762e45aa369d2f68d40390e316474b6bb8434aeb6e3c8bf8b/pytest_benchmark-5.3.1-py3-none-any.whl", hash = "sha256:d7732a9952b1c4a4a32277adc693739126eab37afc3a711847f53805ccadc020", size = 49142, upload-time = "2026-11-01T03:32:08.359Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -3580,6 +3602,9 @@ dependencies = [ adbc = [ { name = "adbc-driver-manager" }, ] +benchmark = [ + { name = "pytest-benchmark" }, +] dask = [ { name = "dask" }, ] @@ -3634,7 +3659,7 @@ dev = [ { name = "maturin" }, { name = "py-spy" }, { name = "ruff" }, - { name = "xarray-sql", extra = ["docs", "test"] }, + { name = "xarray-sql", extra = ["benchmark", "docs", "test"] }, ] [package.metadata] @@ -3655,6 +3680,7 @@ requires-dist = [ { name = "pyproj", marker = "extra == 'geo'" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest", marker = "extra == 'test'" }, + { name = "pytest-benchmark", marker = "extra == 'benchmark'", specifier = ">=4.0" }, { name = "watchfiles", marker = "extra == 'dev'" }, { name = "xarray", specifier = ">=2024.7.0" }, { name = "xarray", extras = ["io"], marker = "extra == 'test'" }, @@ -3662,7 +3688,7 @@ requires-dist = [ { name = "xarray-sql", extras = ["docs"], marker = "extra == 'dev'" }, { name = "zensical", marker = "extra == 'docs'" }, ] -provides-extras = ["adbc", "dask", "dev", "docs", "duckdb", "geo", "ml", "polars", "test"] +provides-extras = ["adbc", "benchmark", "dask", "dev", "docs", "duckdb", "geo", "ml", "polars", "test"] [package.metadata.requires-dev] dev = [ @@ -3672,6 +3698,7 @@ dev = [ { name = "maturin", specifier = ">=1.9.1" }, { name = "py-spy", specifier = ">=0.4.0" }, { name = "ruff", specifier = ">=0.15.10" }, + { name = "xarray-sql", extras = ["benchmark"] }, { name = "xarray-sql", extras = ["docs"] }, { name = "xarray-sql", extras = ["test"] }, ]