diff --git a/ci/docker/pixi.dockerfile b/ci/docker/pixi.dockerfile new file mode 100644 index 000000000000..f7a810669c31 --- /dev/null +++ b/ci/docker/pixi.dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +FROM ${arch}/ubuntu:24.04 + +# install build essentials +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q \ + curl \ + gdb \ + git \ + libc6-dbg \ + tzdata \ + tzdata-legacy \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# install pixi +RUN curl -fsSL https://pixi.sh/install.sh -o /tmp/install-pixi.sh +RUN PIXI_HOME=/opt/pixi PIXI_NO_PATH_UPDATE=1 sh /tmp/install-pixi.sh && rm /tmp/install-pixi.sh +ENV PATH=/opt/pixi/bin:$PATH + +# Verify pixi installation +RUN pixi --version diff --git a/ci/pixi/.gitignore b/ci/pixi/.gitignore new file mode 100644 index 000000000000..8913651c86dc --- /dev/null +++ b/ci/pixi/.gitignore @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +**/.pixi +**/pixi.lock diff --git a/ci/pixi/asan/cpp/pixi.toml b/ci/pixi/asan/cpp/pixi.toml new file mode 100644 index 000000000000..31c83cec46da --- /dev/null +++ b/ci/pixi/asan/cpp/pixi.toml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64"] +preview = ["pixi-build"] + +[environments] +test = ["test"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" +# GH-39973: debug memory pool conflicts with ASAN +ARROW_DEBUG_MEMORY_POOL = "none" +ASAN_OPTIONS = "allocator_may_return_null=1:detect_leaks=0:symbolize=1" +UBSAN_OPTIONS = "print_stacktrace=1" + +[feature.test.dependencies] +binutils = "*" +cmake = "*" + +[feature.test.tasks] +# This path depends on pixi-build's internal work-directory layout. +# Required for running C++ tests on Docker. +test = "cd $(ls -d /arrow/ci/pixi/asan/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" + +[package] +name = "arrow-cpp" +version = "25.0.0a0" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../../cpp" + +[package.build.backend] +name = "pixi-build-cmake" +version = "*" + +[package.build.config] +extra-args = [ + "-DCMAKE_BUILD_TYPE=Debug", + "-DARROW_USE_ASAN=ON", + "-DARROW_USE_UBSAN=ON", + # Avoid creating huge static libraries with sanitizer instrumentation + "-DARROW_BUILD_STATIC=OFF", + "-DARROW_BUILD_TESTS=ON", + "-DARROW_COMPUTE=ON", + "-DARROW_CSV=ON", + "-DARROW_DATASET=ON", + "-DARROW_FILESYSTEM=ON", + "-DARROW_JSON=ON", + "-DARROW_PARQUET=ON", + "-DARROW_WITH_BROTLI=ON", + "-DARROW_WITH_BZ2=ON", + "-DARROW_WITH_LZ4=ON", + "-DARROW_WITH_SNAPPY=ON", + "-DARROW_WITH_ZLIB=ON", + "-DARROW_WITH_ZSTD=ON", + # libutf8proc on package-host-dependencies fails use Bundled. + "-Dutf8proc_SOURCE=BUNDLED", + "-DOPENSSL_ROOT_DIR=$PREFIX", +] + +[package.host-dependencies] +brotli = "*" +bzip2 = "*" +gflags = "*" +gmock = ">=1.10.0" +gtest = ">=1.10.0" +libboost-devel = "*" +lz4-c = "*" +openssl = "*" +rapidjson = "*" +re2 = "*" +snappy = "*" +thrift-cpp = ">=0.11.0" +xsimd = ">=14.0" +zlib = "*" +zstd = "*" diff --git a/ci/pixi/asan/python/pixi.toml b/ci/pixi/asan/python/pixi.toml new file mode 100644 index 000000000000..011e3581e208 --- /dev/null +++ b/ci/pixi/asan/python/pixi.toml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64"] +preview = ["pixi-build"] + +[environments] +test = ["test"] + +[feature.test] +platforms = ["linux-64"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" +# GH-39973: debug memory pool conflicts with ASAN +ARROW_DEBUG_MEMORY_POOL = "none" +ASAN_OPTIONS = "detect_leaks=0:symbolize=1:strict_init_order=true:allocator_may_return_null=1:use_sigaltstack=0" +UBSAN_OPTIONS = "print_stacktrace=1" + +[feature.test.dependencies] +hypothesis = "*" +pyarrow = { path = "." } +pytest = "*" +pytest-timeout = "*" + +[feature.test.tasks] +test = "pytest -r s --pyargs pyarrow --timeout=180" + +[package] +name = "pyarrow" +version = "25.0.0a0" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../../python" + +[package.build.backend] +name = "pixi-build-python" +version = "*" + +[package.build.config] +compilers = ["c", "cxx"] +extra-args = [ + "-Ccmake.build-type=Debug", + "-Ccmake.define.ARROW_USE_ASAN=ON", + "-Ccmake.define.ARROW_USE_UBSAN=ON", +] +env = { + PYARROW_BUNDLE_ARROW_CPP = "0", + CMAKE_GENERATOR = "Ninja", + ARROW_HOME = "$PREFIX", + ASAN_OPTIONS = "detect_leaks=0:symbolize=1:strict_init_order=true:allocator_may_return_null=1:use_sigaltstack=0" +} + +[package.host-dependencies] +# Sanitizer-instrumented Arrow C++ (sibling pixi package). +arrow-cpp = { path = "../cpp" } +cmake = "*" +cython = ">=3.0" +gcc_linux-64 = "*" +git = "*" +gxx_linux-64 = "*" +ninja = "*" +pip = "*" +pkg-config = "*" +scikit-build-core = "*" +setuptools-scm = "*" +sysroot_linux-64 = "*" +zlib = "*" +# Instrumented CPython built from source (v3.15.0a6). +# commit hash must match numpy's pixi-packages/asan pinned commit, +# so pixi-build doesn't build cpython twice with different versions. +python.git = "https://github.com/python/cpython" +python.subdirectory = "Tools/pixi-packages/asan" +python.rev = "15b216f30d0445469ec31bc7509fcc55a216ef7c" +# Instrumented NumPy built from source. +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/asan" +numpy.rev = "0419105da9cd0a15a4e02bc22019c2b65272c68a" + +[package.run-dependencies] +arrow-cpp = { path = "../cpp" } +# The following are necessary to run the Python test suite, +# which otherwise won't be run with the instrumented Python and NumPy. +# Important to use the same revisions as in host-dependencies to avoid +# mismatches between build and test environments. +python.git = "https://github.com/python/cpython" +python.subdirectory = "Tools/pixi-packages/asan" +python.rev = "15b216f30d0445469ec31bc7509fcc55a216ef7c" +numpy.git = "https://github.com/numpy/numpy" +numpy.subdirectory = "pixi-packages/asan" +numpy.rev = "0419105da9cd0a15a4e02bc22019c2b65272c68a" diff --git a/ci/pixi/default/cpp/pixi.toml b/ci/pixi/default/cpp/pixi.toml new file mode 100644 index 000000000000..36851a3ec047 --- /dev/null +++ b/ci/pixi/default/cpp/pixi.toml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64"] +preview = ["pixi-build"] + +[environments] +test = ["test"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" + +[feature.test.dependencies] +binutils = "*" +cmake = "*" + +[feature.test.tasks] +# This path depends on pixi-build's internal work-directory layout. +# Required for running C++ tests on Docker. +test = "cd $(ls -d /arrow/ci/pixi/default/cpp/.pixi/build/work/arrow-cpp-*/work/build | head -n 1) && ctest --output-on-failure" + +[package] +name = "arrow-cpp" +version = "25.0.0a0" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../../cpp" + +[package.build.backend] +name = "pixi-build-cmake" +version = "*" + +[package.build.config] +extra-args = [ + "-DCMAKE_BUILD_TYPE=Release", + "-DARROW_BUILD_TESTS=ON", + "-DARROW_COMPUTE=ON", + "-DARROW_CSV=ON", + "-DARROW_DATASET=ON", + "-DARROW_FILESYSTEM=ON", + "-DARROW_JSON=ON", + "-DARROW_PARQUET=ON", + "-DARROW_WITH_BROTLI=ON", + "-DARROW_WITH_BZ2=ON", + "-DARROW_WITH_LZ4=ON", + "-DARROW_WITH_SNAPPY=ON", + "-DARROW_WITH_ZLIB=ON", + "-DARROW_WITH_ZSTD=ON", + # libutf8proc on package-host-dependencies fails use Bundled. + "-Dutf8proc_SOURCE=BUNDLED", + "-DOPENSSL_ROOT_DIR=$PREFIX", +] + +[package.host-dependencies] +brotli = "*" +bzip2 = "*" +gflags = "*" +gmock = ">=1.10.0" +gtest = ">=1.10.0" +libboost-devel = "*" +lz4-c = "*" +openssl = "*" +rapidjson = "*" +re2 = "*" +snappy = "*" +thrift-cpp = ">=0.11.0" +xsimd = ">=14.0" +zlib = "*" +zstd = "*" diff --git a/ci/pixi/default/python/pixi.toml b/ci/pixi/default/python/pixi.toml new file mode 100644 index 000000000000..268747afbd01 --- /dev/null +++ b/ci/pixi/default/python/pixi.toml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +channels = ["https://prefix.dev/conda-forge"] +platforms = ["linux-64"] +preview = ["pixi-build"] + +[environments] +test = ["test"] + +[feature.test] +platforms = ["linux-64"] + +[feature.test.activation.env] +ARROW_TEST_DATA = "/arrow/testing/data" +PARQUET_TEST_DATA = "/arrow/cpp/submodules/parquet-testing/data" + +[feature.test.dependencies] +hypothesis = "*" +pyarrow = { path = "." } +pytest = "*" +pytest-timeout = "*" + +[feature.test.tasks] +test = "pytest -r s --pyargs pyarrow --timeout=60" + +[package] +name = "pyarrow" +version = "25.0.0a0" +authors = ["Apache Arrow"] + +[package.build] +source.path = "../../../../python" + +[package.build.backend] +name = "pixi-build-python" +version = "*" + +[package.build.config] +compilers = ["c", "cxx"] +env = { + PYARROW_BUNDLE_ARROW_CPP = "0", + CMAKE_GENERATOR = "Ninja", + ARROW_HOME = "$PREFIX", +} + +[package.host-dependencies] +arrow-cpp = { path = "../cpp" } # source-dep on the arrow-cpp manifest +cmake = "*" +cython = ">=3.0" +gcc_linux-64 = "*" +git = "*" +gxx_linux-64 = "*" +ninja = "*" +numpy = "*" +pip = "*" +pkg-config = "*" +python = ">=3.10,<3.14" +scikit-build-core = "*" +setuptools-scm = "*" +sysroot_linux-64 = "*" +zlib = "*" + +[package.run-dependencies] +arrow-cpp = { path = "../cpp" } +python = ">=3.10,<3.14" +numpy = "*" diff --git a/compose.yaml b/compose.yaml index be32a95dd945..53ed3bd4c59b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -138,6 +138,10 @@ x-hierarchy: - fedora-cpp: - fedora-python - fedora-r-clang + - pixi-cpp: + - pixi-cpp-sanitizer + - pixi-python: + - pixi-python-sanitizer - python-sdist - ubuntu-cpp: - ubuntu-cpp-static @@ -197,6 +201,14 @@ volumes: name: ${ARCH}-fedora-${FEDORA}-ccache maven-cache: name: maven-cache + pixi-cpp-build: + name: ${ARCH}-pixi-cpp-build + pixi-cpp-sanitizer-build: + name: ${ARCH}-pixi-cpp-sanitizer-build + pixi-python-build: + name: ${ARCH}-pixi-python-build + pixi-python-sanitizer-build: + name: ${ARCH}-pixi-python-sanitizer-build python-wheel-manylinux-2-28-ccache: name: python-wheel-manylinux-2-28-ccache python-wheel-musllinux-1-2-ccache: @@ -1158,6 +1170,99 @@ services: volumes: *ubuntu-volumes command: *python-command + ############################ Pixi builds ################################## + + pixi-cpp: + # C++ build with pixi + # + # Usage: + # docker compose build pixi-cpp + # docker compose run --rm pixi-cpp + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-cpp-build:/arrow/ci/pixi/default/cpp/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/default/cpp && + pixi run --environment test --manifest-path /arrow/ci/pixi/default/cpp test" + + pixi-cpp-sanitizer: + # C++ build with pixi and Address/Undefined Behavior sanitizers enabled + # + # Usage: + # docker compose build pixi-cpp-sanitizer + # docker compose run --rm pixi-cpp-sanitizer + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + cap_add: + # For LeakSanitizer + - SYS_PTRACE + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-cpp-sanitizer-build:/arrow/ci/pixi/asan/cpp/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/asan/cpp && + pixi run --environment test --manifest-path /arrow/ci/pixi/asan/cpp test" + + pixi-python: + # CPython build with pixi + # + # Usage: + # docker compose build pixi-python + # docker compose run --rm pixi-python + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-python-build:/arrow/ci/pixi/default/python/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/default/python && + pixi run --environment test --manifest-path /arrow/ci/pixi/default/python test" + + pixi-python-sanitizer: + # PyArrow build with pixi against sanitizer-instrumented CPython, NumPy + # and Arrow C++. Builds CPython and NumPy from source via pixi-build. + # + # Usage: + # docker compose build pixi-python-sanitizer + # docker compose run --rm pixi-python-sanitizer + # Parameters: + # ARCH: amd64, arm32v7 + image: ${REPO}:${ARCH}-pixi + cap_add: + # For LeakSanitizer + - SYS_PTRACE + build: + context: . + dockerfile: ci/docker/pixi.dockerfile + args: + arch: ${ARCH} + volumes: + - .:/arrow:delegated + - ${DOCKER_VOLUME_PREFIX}pixi-python-sanitizer-build:/arrow/ci/pixi/asan/python/.pixi:delegated + command: > + /bin/sh -c "pixi build --path /arrow/ci/pixi/asan/python && + pixi run --environment test --manifest-path /arrow/ci/pixi/asan/python test" + ############################ Python wheels ################################## # See available versions at: diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index 22afe6180024..77911ca27fd4 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -228,6 +228,34 @@ def test_version_pre_tag "+set(MLARROW_VERSION \"#{@release_version}\")"], ], }, + { + path: "ci/pixi/asan/cpp/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@release_version}\""], + ], + }, + { + path: "ci/pixi/asan/python/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@release_version}\""], + ], + }, + { + path: "ci/pixi/default/cpp/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@release_version}\""], + ], + }, + { + path: "ci/pixi/default/python/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@release_version}\""], + ], + }, { path: "python/CMakeLists.txt", hunks: [ diff --git a/dev/release/post-10-bump-versions-test.rb b/dev/release/post-10-bump-versions-test.rb index aad4d7c699cb..ec03f0c2b47f 100644 --- a/dev/release/post-10-bump-versions-test.rb +++ b/dev/release/post-10-bump-versions-test.rb @@ -257,6 +257,34 @@ def test_version_post_tag "+set(PYARROW_VERSION \"#{@next_snapshot_version}\")"], ], }, + { + path: "ci/pixi/asan/cpp/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@next_version}a0\""], + ], + }, + { + path: "ci/pixi/asan/python/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@next_version}a0\""], + ], + }, + { + path: "ci/pixi/default/cpp/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@next_version}a0\""], + ], + }, + { + path: "ci/pixi/default/python/pixi.toml", + hunks: [ + ["-version = \"#{@release_version}a0\"", + "+version = \"#{@next_version}a0\""], + ], + }, { path: "python/pyproject.toml", hunks: [ diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index e67439467cc2..953ff4f0b831 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -70,6 +70,14 @@ update_versions() { git add PKGBUILD popd + pushd "${ARROW_DIR}/ci/pixi" + sed -i.bak -E -e \ + "s/^version = \"[0-9].*\"/version = \"${python_version}\"/" \ + */*/pixi.toml + rm -f */*/pixi.toml.bak + git add */*/pixi.toml + popd + pushd "${ARROW_DIR}/cpp" sed -i.bak -E -e \ "s/^set\(ARROW_VERSION \".+\"\)/set(ARROW_VERSION \"${version}\")/" \ diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 2c182887f175..1e3f3d999901 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -366,6 +366,20 @@ tasks: params: image: conda-cpp-valgrind +{% for image in ["pixi-cpp", + "pixi-cpp-sanitizer", + "pixi-python", + "pixi-python-sanitizer"] %} + test-{{ image }}: + ci: github + template: docker-tests/github.linux.yml + params: + image: {{ image }} + {% if image == "pixi-python-sanitizer" %} + timeout: 120 + {% endif %} +{% endfor %} + test-ubuntu-22.04-cpp: ci: github template: docker-tests/github.linux.yml diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index d0ddb9009f89..8fb9afad196f 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -158,7 +158,12 @@ if(NOT DEFINED ARROW_RUNTIME_SIMD_LEVEL) "MAX" CACHE STRING "Max runtime SIMD optimization level") endif() + +option(ARROW_USE_ASAN "Enable Address Sanitizer checks" OFF) +option(ARROW_USE_UBSAN "Enable Undefined Behavior sanitizer checks" OFF) + include(SetupCxxFlags) +include(san-config) if($ENV{PYODIDE}) # These variables are needed for building PyArrow on Emscripten. diff --git a/python/pyarrow/conftest.py b/python/pyarrow/conftest.py index 9fed75dac3ee..480bf9af4fe7 100644 --- a/python/pyarrow/conftest.py +++ b/python/pyarrow/conftest.py @@ -17,6 +17,7 @@ import pytest +import ctypes import os import pyarrow as pa from pyarrow import Codec @@ -25,6 +26,16 @@ import sys +def _is_asan_build(): + if sys.platform == "win32": + return False + try: + ctypes.CDLL(None).__asan_init + return True + except (OSError, AttributeError): + return False + + groups = [ 'acero', 'azure', @@ -43,6 +54,7 @@ 'large_memory', 'lz4', 'memory_leak', + 'noasan', 'nopandas', 'nonumpy', 'numpy', @@ -80,6 +92,7 @@ 'large_memory': False, 'lz4': Codec.is_available('lz4'), 'memory_leak': False, + 'noasan': not _is_asan_build(), 'nopandas': False, 'nonumpy': False, 'numpy': False, diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index 7452c8eb4a13..f9aca70582fa 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -55,6 +55,8 @@ def wrapper(*args, **kwargs): return wrapper +# Asan tests are flacky on check_bytes_allocated assertion. +@pytest.mark.noasan @check_bytes_allocated @pytest.mark.parametrize( ('value_type', 'np_type_str'), diff --git a/python/pyarrow/tests/test_memory.py b/python/pyarrow/tests/test_memory.py index 56196bd9c5b3..7915b39a5be2 100644 --- a/python/pyarrow/tests/test_memory.py +++ b/python/pyarrow/tests/test_memory.py @@ -257,16 +257,22 @@ def check_debug_memory_pool_disabled(pool_factory, env_value, msg): assert msg in res.stderr +# Asan test fails as we are deliberately doing an out-of-bounds write. +@pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_none(pool_factory): check_debug_memory_pool_disabled(pool_factory, "none", "") +# Asan test fails as we are deliberately doing an out-of-bounds write. +@pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_empty(pool_factory): check_debug_memory_pool_disabled(pool_factory, "", "") +# Asan test fails as we are deliberately doing an out-of-bounds write. +@pytest.mark.noasan @pytest.mark.parametrize('pool_factory', supported_factories()) def test_debug_memory_pool_unknown(pool_factory): env_value = "some_arbitrary_value"