Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .github/scripts/build-nixl-runtime.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
# Build a CPU-only UCX runtime for the native CLI test job.
set -euo pipefail

: "${NIXL_PREFIX:?set the installation prefix}"
build_dir=$(mktemp -d)
trap 'rm -rf "$build_dir"' EXIT
jobs=${CARGO_BUILD_JOBS:-4}
manifest=$(cargo metadata --locked --manifest-path nixl/Cargo.toml --format-version 1 --filter-platform x86_64-unknown-linux-gnu |
jq -er '.packages[] | select(.name == "nixl-sys") | .manifest_path')
source_dir=$(git -C "$(dirname "$manifest")" rev-parse --show-toplevel)

cd "$build_dir"
curl --fail --location --retry 3 --output ucx.tar.gz https://github.com/openucx/ucx/releases/download/v1.22.0/ucx-1.22.0.tar.gz
echo '258941cddd14ca60d38c0d31b9b09ec1052c901086841011a498da8b55a3cb24 ucx.tar.gz' | sha256sum --check
tar -xzf ucx.tar.gz
(
cd ucx-1.22.0
./configure --prefix="$NIXL_PREFIX" --enable-mt --without-cuda --without-rocm --without-java --without-go --disable-numa --disable-examples
make -j"$jobs"
make install
)

# Meson can populate subprojects. Leave Cargo's reviewed dependency checkout intact.
git -C "$source_dir" archive HEAD | tar -x -C "$build_dir"
meson setup "$build_dir/build" "$build_dir" --prefix "$NIXL_PREFIX" --libdir lib \
-Ducx_path="$NIXL_PREFIX" -Denable_plugins=UCX -Dinstall_headers=true \
-Dbuild_tests=false -Dbuild_examples=false -Dbuild_nixl_ep=false -Dwith_trace=false
meson compile -C "$build_dir/build" -j "$jobs"
meson install -C "$build_dir/build"
56 changes: 56 additions & 0 deletions .github/workflows/nixl.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
name: Native NIXL client

on:
workflow_dispatch:
pull_request:
paths:
- 'nixl/**'
- 'crates/**'
- 'Cargo.toml'
- 'Cargo.lock'
- 'scripts/build-rdma.sh'
- '.github/scripts/build-nixl-runtime.sh'
- '.github/workflows/nixl.yml'

permissions:
contents: read

jobs:
native:
runs-on: ubuntu-24.04
timeout-minutes: 75
env:
CARGO_PROFILE_DEV_DEBUG: '0'
CARGO_PROFILE_TEST_DEBUG: '0'
CARGO_INCREMENTAL: '0'
UCX_TLS: tcp,self
NIXL_NO_STUBS_FALLBACK: '1'
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
workspaces: nixl
- name: Install native build tools
run: |
echo "NIXL_PREFIX=$RUNNER_TEMP/nixl" >> "$GITHUB_ENV"
sudo apt-get update
sudo apt-get install -y --no-install-recommends libclang-dev ninja-build python3-dev python3-venv
python3 -m venv "$RUNNER_TEMP/native-tools"
"$RUNNER_TEMP/native-tools/bin/pip" install meson==1.12.1 pybind11==3.1.0
echo "$RUNNER_TEMP/native-tools/bin" >> "$GITHUB_PATH"
- name: Cache the UCX and NIXL runtime
id: native-cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: ${{ runner.temp }}/nixl
key: nixl-ubuntu-24.04-${{ runner.arch }}-${{ hashFiles('nixl/Cargo.lock', '.github/scripts/build-nixl-runtime.sh') }}
- name: Build UCX and the NIXL revision from Cargo.lock
if: steps.native-cache.outputs.cache-hit != 'true'
run: .github/scripts/build-nixl-runtime.sh
- name: Build and test the native client over UCX loopback
run: |
export LD_LIBRARY_PATH="$NIXL_PREFIX/lib"
export NIXL_PLUGIN_DIR="$NIXL_PREFIX/lib/plugins"
./scripts/build-rdma.sh
cargo test --locked --manifest-path nixl/Cargo.toml
4 changes: 1 addition & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,4 @@ coverage/
# Cargo.lock
CLAUDE.md

# The independent opt-in build owns its artifacts and lockfile.
/rdma/target/
/rdma/Cargo.lock
/nixl/target/
4 changes: 0 additions & 4 deletions .gitmodules

This file was deleted.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[workspace]
exclude = ["rdma", "vendor/rustfs-ucx"]
exclude = ["nixl"]
resolver = "3"
members = ["crates/cli", "crates/core", "crates/s3"]

Expand Down
3 changes: 1 addition & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,7 @@ cd cli
cargo build --release
```

Optional object-data interfaces and the explicit private build are documented
in [RDMA builds](docs/rdma-build.md). The ordinary build requires no private source.
To use RustFS native NIXL/UCX for object transfers, build the [Linux RDMA client](docs/rdma-build.md#rustfs-native-nixlucx).

## Quick Start

Expand Down
2 changes: 1 addition & 1 deletion crates/cli/src/commands/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ fn validate_version_selector(version_id: Option<&str>, rewind: Option<&str>) ->
#[command(propagate_version = true)]
pub struct Cli {
/// Object data transport: http, auto, or rdma (requires an installed provider)
#[arg(long, global = true, default_value = "http")]
#[arg(long, global = true, env = "RC_DATA_TRANSPORT", default_value = "http")]
pub data_transport: DataTransportMode,

/// Maximum multipart parts per object for the destination backend
Expand Down
89 changes: 10 additions & 79 deletions crates/cli/src/launcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,6 @@ fn allow_log_metadata(metadata: &tracing::Metadata<'_>) -> bool {

/// Run the shared CLI launcher, preserving its worker-thread and exit behavior.
pub fn run() {
run_with_shutdown(|| async {});
}

/// Runs host cleanup after the command while its async runtime is still available.
/// The host bounds its own cleanup and reports retained resources as appropriate.
pub fn run_with_shutdown<S, F>(shutdown: S)
where
S: FnOnce() -> F + Send + 'static,
F: Future<Output = ()> + 'static,
{
let cli = Cli::parse();
let env_is_set = std::env::var_os(EnvFilter::DEFAULT_ENV).is_some();
let env_filter = build_env_filter(cli.debug, env_is_set);
Expand All @@ -66,43 +56,18 @@ where
.with(env_filter)
.init();

let exit_code = match run_command(cli, shutdown) {
Ok(exit_code) => exit_code,
Err(error) => {
eprintln!("{error}");
ExitCode::GeneralError
}
};
let exit_code =
match run_async_command(move || Box::pin(commands::execute(cli)), command_runtime) {
Ok(exit_code) => exit_code,
Err(error) => {
eprintln!("{error}");
ExitCode::GeneralError
}
};

std::process::exit(exit_code.as_i32());
}

fn run_command<S, F>(cli: Cli, shutdown: S) -> Result<ExitCode, LauncherError>
where
S: FnOnce() -> F + Send + 'static,
F: Future<Output = ()> + 'static,
{
run_async_command(
move || finish_with_shutdown(commands::execute(cli), shutdown),
command_runtime,
)
}

fn finish_with_shutdown<C, S, F>(command: C, shutdown: S) -> impl Future<Output = C::Output>
where
C: Future,
S: FnOnce() -> F,
F: Future<Output = ()>,
{
// Keep the cleanup future small even when an SDK command has a large state.
let command = Box::pin(command);
async move {
let output = command.await;
shutdown().await;
output
}
}

fn command_runtime() -> std::io::Result<tokio::runtime::Runtime> {
tokio::runtime::Builder::new_multi_thread()
.thread_stack_size(COMMAND_THREAD_STACK_SIZE)
Expand Down Expand Up @@ -205,37 +170,6 @@ mod tests {
assert_eq!(exit_code, ExitCode::UnsupportedFeature);
}

#[test]
fn host_cleanup_runs_after_a_failed_command_before_runtime_shutdown() {
let state = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed = Arc::clone(&state);
let exit_code = run_async_command(
move || {
let shutdown_state = Arc::clone(&state);
finish_with_shutdown(
async move {
state.store(1, Ordering::Release);
ExitCode::GeneralError
},
move || async move {
assert_eq!(shutdown_state.load(Ordering::Acquire), 1);
assert_eq!(std::thread::current().name(), Some(COMMAND_THREAD_NAME));
tokio::spawn(async move {
tokio::task::yield_now().await;
shutdown_state.store(2, Ordering::Release);
})
.await
.expect("cleanup task completes in the command runtime");
},
)
},
test_runtime,
)
.expect("host cleanup completes");
assert_eq!(exit_code, ExitCode::GeneralError);
assert_eq!(observed.load(Ordering::Acquire), 2);
}

#[test]
fn command_launcher_uses_the_named_worker_thread() {
let thread_name = run_async_command(
Expand All @@ -260,11 +194,8 @@ mod tests {
usize::from(payload[0]) + usize::from(payload[LARGE_FUTURE_SIZE - 1])
}

let result = run_async_command(
|| finish_with_shutdown(large_future(), || async {}),
test_runtime,
)
.expect("large command future should complete on the sized worker stack");
let result = run_async_command(|| Box::pin(large_future()), test_runtime)
.expect("large command future should complete on the sized worker stack");

assert_eq!(result, 14);
}
Expand Down
2 changes: 1 addition & 1 deletion crates/cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ pub mod commands;
pub(crate) mod confirm;
pub mod exit_code;
mod launcher;
pub use launcher::{run, run_with_shutdown};
pub use launcher::run;
pub mod output;
pub(crate) mod private_file;
pub(crate) mod secret_input;
Loading