diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b86b37c6e..5c414519a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,11 +24,11 @@ on: workflow_call: inputs: build_mode: - description: 'Build mode: debug or release' + description: "Build mode: debug or release" required: true type: string run_wheels: - description: 'Whether to build distribution wheels' + description: "Whether to build distribution wheels" required: false type: boolean default: false @@ -252,11 +252,57 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] - os: [macos-latest, windows-latest] + include: + - os: macos-latest + python-version: "3.10" + python-tag: cp310 + - os: macos-latest + python-version: "3.11" + python-tag: cp311 + - os: macos-latest + python-version: "3.12" + python-tag: cp312 + - os: macos-latest + python-version: "3.13" + python-tag: cp313 + - os: macos-latest + python-version: "3.14" + python-tag: cp314 + - os: macos-latest + python-version: "3.13t" + python-tag: cp313t + - os: macos-latest + python-version: "3.14t" + python-tag: cp314t + - os: windows-latest + python-version: "3.10" + python-tag: cp310 + - os: windows-latest + python-version: "3.11" + python-tag: cp311 + - os: windows-latest + python-version: "3.12" + python-tag: cp312 + - os: windows-latest + python-version: "3.13" + python-tag: cp313 + - os: windows-latest + python-version: "3.14" + python-tag: cp314 + - os: windows-latest + python-version: "3.13t" + python-tag: cp313t + - os: windows-latest + python-version: "3.14t" + python-tag: cp314t steps: - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + id: setup-python + with: + python-version: ${{ matrix.python-version }} + - uses: dtolnay/rust-toolchain@stable - run: rm LICENSE.txt @@ -293,11 +339,15 @@ jobs: - name: Build Python package (release mode) if: inputs.build_mode == 'release' - run: uv run --no-project maturin build --release --strip --features substrait + env: + MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }} + run: uv run --no-project maturin build --release --strip --features "${MATURIN_FEATURES}" --interpreter "${{ steps.setup-python.outputs.python-path }}" - name: Build Python package (debug mode) if: inputs.build_mode != 'release' - run: uv run --no-project maturin build --features substrait + env: + MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }} + run: uv run --no-project maturin build --features "${MATURIN_FEATURES}" --interpreter "${{ steps.setup-python.outputs.python-path }}" - name: List Windows wheels if: matrix.os == 'windows-latest' @@ -314,7 +364,7 @@ jobs: uses: actions/upload-artifact@v6 if: inputs.build_mode == 'release' with: - name: dist-${{ matrix.os }} + name: dist-${{ matrix.os }}-${{ matrix.python-tag }} path: target/wheels/* # ============================================ @@ -327,10 +377,29 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + include: + - python-version: "3.10" + python-tag: cp310 + - python-version: "3.11" + python-tag: cp311 + - python-version: "3.12" + python-tag: cp312 + - python-version: "3.13" + python-tag: cp313 + - python-version: "3.14" + python-tag: cp314 + - python-version: "3.13t" + python-tag: cp313t + - python-version: "3.14t" + python-tag: cp314t steps: - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + id: setup-python + with: + python-version: ${{ matrix.python-version }} + - uses: dtolnay/rust-toolchain@stable - run: rm LICENSE.txt @@ -359,8 +428,10 @@ jobs: run: uv sync --dev --no-install-package datafusion - name: Build (release mode) + env: + MATURIN_FEATURES: ${{ endsWith(matrix.python-version, 't') && 'substrait' || 'substrait,py-limited-api' }} run: | - uv run --no-project maturin build --release --strip --features substrait + uv run --no-project maturin build --release --strip --features "${MATURIN_FEATURES}" --interpreter "${{ steps.setup-python.outputs.python-path }}" - name: List Mac wheels run: find target/wheels/ @@ -368,7 +439,7 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v6 with: - name: dist-macos-aarch64 + name: dist-macos-aarch64-${{ matrix.python-tag }} path: target/wheels/* # ============================================ @@ -407,9 +478,8 @@ jobs: shell: bash # ============================================ - # Build - Source Distribution + # Build - Merge Artifacts # ============================================ - merge-build-artifacts: runs-on: ubuntu-latest name: Merge build artifacts @@ -434,7 +504,7 @@ jobs: build-docs: name: Build docs runs-on: ubuntu-latest - needs: [build-manylinux-x86_64] # Only need the Linux wheel for docs + needs: [build-manylinux-x86_64] # Only need the Linux wheel for docs # Only run docs on main branch pushes, tags, or PRs if: github.event_name == 'push' || github.event_name == 'pull_request' steps: @@ -477,7 +547,7 @@ jobs: - name: Download pre-built Linux wheel uses: actions/download-artifact@v7 with: - name: dist-manylinux-x86_64 + name: dist-manylinux-x86_64-cp310-314 path: wheels/ # Install from the pre-built wheels diff --git a/Cargo.toml b/Cargo.toml index 313640ec2..7ef95b39c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ include = [ default = ["mimalloc"] protoc = ["datafusion-substrait/protoc"] substrait = ["dep:datafusion-substrait"] +py-limited-api = ["pyo3/abi3-py310"] [dependencies] tokio = { version = "1.49", features = [ @@ -48,11 +49,7 @@ tokio = { version = "1.49", features = [ "rt-multi-thread", "sync", ] } -pyo3 = { version = "0.26", features = [ - "extension-module", - "abi3", - "abi3-py310", -] } +pyo3 = { version = "0.26", features = ["extension-module"] } pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] } pyo3-log = "0.13.3" arrow = { version = "57", features = ["pyarrow"] } diff --git a/dev/release/README.md b/dev/release/README.md index ed28f4aa6..7d067c1cc 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -130,6 +130,10 @@ datafusion-22.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl datafusion-22.0.0-cp37-abi3-win_amd64.whl ``` +Note: PyO3's free-threaded CPython builds (3.13t/3.14t) use a distinct ABI and cannot use the limited API (`abi3`). +The release workflow enables Cargo feature `py-limited-api` for GIL-enabled wheels and disables it for free-threaded +builds, producing version-specific `cp313t`/`cp314t` wheels. + Upload the wheels to testpypi. ```bash @@ -237,6 +241,12 @@ cargo publish ### Publishing Python Artifacts to PyPi +GitHub Actions groups wheel artifacts by platform and interpreter tag using the pattern `dist--`. +For example, standard manylinux wheels live under `dist-manylinux-x86_64-cp310-314` while the free-threaded builds +use `dist-manylinux-x86_64-cp313t-314t`. macOS and Windows jobs publish one artifact per CPython version as well +(`dist-macos-latest-cp311`, `dist-windows-latest-cp313t`, etc.). Download the exact tags you intend to push to PyPI, +and remember that the docs workflow currently installs from the `cp310-314` manylinux artifact. + Go to the Test PyPI page of Datafusion, and download [all published artifacts](https://test.pypi.org/project/datafusion/#files) under `dist-release/` directory. Then proceed uploading them using `twine`: diff --git a/src/lib.rs b/src/lib.rs index 081366b20..468243a3d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,9 +16,9 @@ // under the License. // Re-export Apache Arrow DataFusion dependencies -pub use datafusion; pub use datafusion::{ - common as datafusion_common, logical_expr as datafusion_expr, optimizer, sql as datafusion_sql, + self, common as datafusion_common, logical_expr as datafusion_expr, optimizer, + sql as datafusion_sql, }; #[cfg(feature = "substrait")] pub use datafusion_substrait;