diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile deleted file mode 100644 index 1a5b87ef7296..000000000000 --- a/ci/docker/conda-cpp.dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch -FROM ${repo}:${arch}-conda - -# install the required conda packages into the test environment -COPY ci/conda_env_cpp.yml \ - ci/conda_env_gandiva.yml \ - /arrow/ci/ -RUN conda install \ - --file arrow/ci/conda_env_cpp.yml \ - --file arrow/ci/conda_env_gandiva.yml \ - compilers \ - doxygen \ - gdb \ - valgrind && \ - conda clean --all - -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DATASET=ON \ - ARROW_DEPENDENCY_SOURCE=CONDA \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA=ON \ - ARROW_HOME=$CONDA_PREFIX \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_CCACHE=ON \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - PARQUET_BUILD_EXAMPLES=ON \ - PARQUET_BUILD_EXECUTABLES=ON \ - PARQUET_HOME=$CONDA_PREFIX diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile deleted file mode 100644 index 1f2c9ac5da21..000000000000 --- a/ci/docker/conda-integration.dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -FROM ${repo}:${arch}-conda-cpp - -ARG arch=amd64 -ARG maven=3.5 -ARG node=14 -ARG jdk=8 -ARG go=1.15 - -# Install Archery and integration dependencies -COPY ci/conda_env_archery.yml /arrow/ci/ -RUN conda install -q \ - --file arrow/ci/conda_env_cpp.yml \ - --file arrow/ci/conda_env_archery.yml \ - numpy \ - compilers \ - maven=${maven} \ - nodejs=${node} \ - yarn \ - openjdk=${jdk} && \ - conda clean --all --force-pkgs-dirs - -# Install Rust with only the needed components -# (rustfmt is needed for tonic-build to compile the protobuf definitions) -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile=minimal -y && \ - $HOME/.cargo/bin/rustup component add rustfmt - -ENV GOROOT=/opt/go \ - GOBIN=/opt/go/bin \ - GOPATH=/go \ - PATH=/opt/go/bin:$PATH -RUN wget -nv -O - https://dl.google.com/go/go${go}.linux-${arch}.tar.gz | tar -xzf - -C /opt - -ENV ARROW_BUILD_INTEGRATION=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_COMPUTE=OFF \ - ARROW_CSV=OFF \ - ARROW_DATASET=OFF \ - ARROW_FILESYSTEM=OFF \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA=OFF \ - ARROW_HDFS=OFF \ - ARROW_JEMALLOC=OFF \ - ARROW_JSON=OFF \ - ARROW_ORC=OFF \ - ARROW_PARQUET=OFF \ - ARROW_PLASMA=OFF \ - ARROW_S3=OFF \ - ARROW_USE_GLOG=OFF \ - CMAKE_UNITY_BUILD=ON diff --git a/ci/docker/conda-python-dask.dockerfile b/ci/docker/conda-python-dask.dockerfile deleted file mode 100644 index cd59a5538a9a..000000000000 --- a/ci/docker/conda-python-dask.dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG dask=latest -COPY ci/scripts/install_dask.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_dask.sh ${dask} \ No newline at end of file diff --git a/ci/docker/conda-python-hdfs.dockerfile b/ci/docker/conda-python-hdfs.dockerfile deleted file mode 100644 index f6ffc71ce625..000000000000 --- a/ci/docker/conda-python-hdfs.dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG jdk=8 -ARG maven=3.5 -RUN conda install -q \ - maven=${maven} \ - openjdk=${jdk} \ - pandas && \ - conda clean --all - -# installing libhdfs (JNI) -ARG hdfs=3.2.1 -ENV HADOOP_HOME=/opt/hadoop-${hdfs} \ - HADOOP_OPTS=-Djava.library.path=/opt/hadoop-${hdfs}/lib/native \ - PATH=$PATH:/opt/hadoop-${hdfs}/bin:/opt/hadoop-${hdfs}/sbin -COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/util_download_apache.sh \ - "hadoop/common/hadoop-${hdfs}/hadoop-${hdfs}.tar.gz" /opt - -COPY ci/etc/hdfs-site.xml $HADOOP_HOME/etc/hadoop/ - -# build cpp with tests -ENV CC=gcc \ - CXX=g++ \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=OFF \ - ARROW_PLASMA=OFF \ - ARROW_PARQUET=ON \ - ARROW_ORC=OFF \ - ARROW_HDFS=ON \ - ARROW_PYTHON=ON \ - ARROW_BUILD_TESTS=ON diff --git a/ci/docker/conda-python-jpype.dockerfile b/ci/docker/conda-python-jpype.dockerfile deleted file mode 100644 index f77ef9bf66b4..000000000000 --- a/ci/docker/conda-python-jpype.dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG jdk=11 -ARG maven=3.6 -RUN conda install -q \ - maven=${maven} \ - openjdk=${jdk} \ - jpype1 && \ - conda clean --all diff --git a/ci/docker/conda-python-kartothek.dockerfile b/ci/docker/conda-python-kartothek.dockerfile deleted file mode 100644 index d523161822c0..000000000000 --- a/ci/docker/conda-python-kartothek.dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -# install kartothek dependencies from conda-forge -RUN conda install -c conda-forge -q \ - attrs \ - click \ - cloudpickle \ - dask \ - decorator \ - freezegun \ - msgpack-python \ - prompt-toolkit \ - pytest-mock \ - pytest-xdist \ - pyyaml \ - simplejson \ - simplekv \ - storefact \ - toolz \ - urlquote \ - zstandard && \ - conda clean --all - -ARG kartothek=latest -COPY ci/scripts/install_kartothek.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_kartothek.sh ${kartothek} /kartothek diff --git a/ci/docker/conda-python-pandas.dockerfile b/ci/docker/conda-python-pandas.dockerfile deleted file mode 100644 index 303cc80e48a0..000000000000 --- a/ci/docker/conda-python-pandas.dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG pandas=latest -ARG numpy=latest -COPY ci/scripts/install_pandas.sh /arrow/ci/scripts/ -RUN conda uninstall -q -y numpy && \ - /arrow/ci/scripts/install_pandas.sh ${pandas} ${numpy} diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile deleted file mode 100644 index a2af2ac135c9..000000000000 --- a/ci/docker/conda-python-spark.dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -ARG jdk=8 -ARG maven=3.5 - -RUN conda install -q \ - openjdk=${jdk} \ - maven=${maven} \ - pandas && \ - conda clean --all - -# installing specific version of spark -ARG spark=master -COPY ci/scripts/install_spark.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_spark.sh ${spark} /spark - -# build cpp with tests -ENV CC=gcc \ - CXX=g++ \ - ARROW_PYTHON=ON \ - ARROW_HDFS=ON \ - ARROW_BUILD_TESTS=OFF \ - SPARK_VERSION=${spark} diff --git a/ci/docker/conda-python-turbodbc.dockerfile b/ci/docker/conda-python-turbodbc.dockerfile deleted file mode 100644 index ff7fdf6e1d0b..000000000000 --- a/ci/docker/conda-python-turbodbc.dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch=amd64 -ARG python=3.6 -FROM ${repo}:${arch}-conda-python-${python} - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - odbc-postgresql \ - postgresql \ - sudo && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# install turbodbc dependencies from conda-forge -RUN conda install -c conda-forge -q\ - pybind11 \ - pytest-cov \ - mock \ - unixodbc && \ - conda clean --all - -RUN service postgresql start && \ - sudo -u postgres psql -U postgres -c \ - "CREATE DATABASE test_db;" && \ - sudo -u postgres psql -U postgres -c \ - "ALTER USER postgres WITH PASSWORD 'password';" - -ARG turbodbc=latest -COPY ci/scripts/install_turbodbc.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_turbodbc.sh ${turbodbc} /turbodbc - -ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json" diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile deleted file mode 100644 index a7e76974825d..000000000000 --- a/ci/docker/conda-python.dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch -FROM ${repo}:${arch}-conda-cpp - -# install python specific packages -ARG python=3.6 -COPY ci/conda_env_python.yml /arrow/ci/ -RUN conda install -q \ - --file arrow/ci/conda_env_python.yml \ - $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \ - python=${python} \ - nomkl && \ - conda clean --all - -ENV ARROW_PYTHON=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_TENSORFLOW=ON \ - ARROW_USE_GLOG=OFF diff --git a/ci/docker/conda.dockerfile b/ci/docker/conda.dockerfile deleted file mode 100644 index 94de009904a4..000000000000 --- a/ci/docker/conda.dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -FROM ${arch}/ubuntu:18.04 - -# arch is unset after the FROM statement, so need to define it again -ARG arch=amd64 -ARG prefix=/opt/conda - -# install build essentials -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update -y -q && \ - apt-get install -y -q wget tzdata libc6-dbg \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=${prefix}/bin:$PATH -# install conda and minio -COPY ci/scripts/install_conda.sh \ - ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix} -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix} - -# create a conda environment -ADD ci/conda_env_unix.yml /arrow/ci/ -RUN conda create -n arrow --file arrow/ci/conda_env_unix.yml git && \ - conda clean --all - -# activate the created environment by default -RUN echo "conda activate arrow" >> ~/.profile -ENV CONDA_PREFIX=${prefix}/envs/arrow - -# use login shell to activate arrow environment un the RUN commands -SHELL [ "/bin/bash", "-c", "-l" ] - -# use login shell when running the container -ENTRYPOINT [ "/bin/bash", "-c", "-l" ] diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile deleted file mode 100644 index 83f8ce529cb8..000000000000 --- a/ci/docker/debian-10-cpp.dockerfile +++ /dev/null @@ -1,104 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -FROM ${arch}/debian:10 -ARG arch - -ENV DEBIAN_FRONTEND noninteractive - -RUN \ - echo "deb http://deb.debian.org/debian buster-backports main" > \ - /etc/apt/sources.list.d/backports.list - -ARG llvm -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ccache \ - clang-${llvm} \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-all-dev \ - libbrotli-dev \ - libbz2-dev \ - libc-ares-dev \ - libcurl4-openssl-dev \ - libgflags-dev \ - libgmock-dev \ - libgoogle-glog-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - llvm-${llvm}-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA=ON \ - ARROW_HOME=/usr/local \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_CCACHE=ON \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - cares_SOURCE=BUNDLED \ - CC=gcc \ - CXX=g++ \ - gRPC_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PATH=/usr/lib/ccache/:$PATH \ - Protobuf_SOURCE=BUNDLED diff --git a/ci/docker/debian-10-go.dockerfile b/ci/docker/debian-10-go.dockerfile deleted file mode 100644 index 199f09e24fcb..000000000000 --- a/ci/docker/debian-10-go.dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -ARG go=1.15 -FROM ${arch}/golang:${go} - -# TODO(kszucs): -# 1. add the files required to install the dependencies to .dockerignore -# 2. copy these files to their appropriate path -# 3. download and compile the dependencies diff --git a/ci/docker/debian-10-js.dockerfile b/ci/docker/debian-10-js.dockerfile deleted file mode 100644 index 5bb31f2e32ef..000000000000 --- a/ci/docker/debian-10-js.dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -ARG node=14 -FROM ${arch}/node:${node} - -ENV NODE_NO_WARNINGS=1 - -# TODO(kszucs): -# 1. add the files required to install the dependencies to .dockerignore -# 2. copy these files to their appropriate path -# 3. download and compile the dependencies diff --git a/ci/docker/debian-9-java.dockerfile b/ci/docker/debian-9-java.dockerfile deleted file mode 100644 index 2cc36e3eafba..000000000000 --- a/ci/docker/debian-9-java.dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch=amd64 -ARG jdk=8 -ARG maven=3.5.4 -FROM ${arch}/maven:${maven}-jdk-${jdk} - -ENV ARROW_JAVA_SHADE_FLATBUFS=ON - -# TODO(kszucs): -# 1. add the files required to install the dependencies to .dockerignore -# 2. copy these files to their appropriate path -# 3. download and compile the dependencies diff --git a/ci/docker/fedora-33-cpp.dockerfile b/ci/docker/fedora-33-cpp.dockerfile deleted file mode 100644 index 9dde6999510a..000000000000 --- a/ci/docker/fedora-33-cpp.dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch -FROM ${arch}/fedora:33 -ARG arch - -# install dependencies -RUN dnf update -y && \ - dnf install -y \ - autoconf \ - boost-devel \ - brotli-devel \ - bzip2-devel \ - c-ares-devel \ - ccache \ - clang-devel \ - cmake \ - curl-devel \ - flatbuffers-devel \ - gcc \ - gcc-c++ \ - gflags-devel \ - git \ - glog-devel \ - gmock-devel \ - google-benchmark-devel \ - grpc-devel \ - grpc-plugins \ - gtest-devel \ - java-latest-openjdk-devel \ - java-latest-openjdk-headless \ - libzstd-devel \ - llvm-devel \ - llvm-static \ - lz4-devel \ - make \ - ninja-build \ - openssl-devel \ - protobuf-devel \ - python \ - rapidjson-devel \ - re2-devel \ - snappy-devel \ - thrift-devel \ - utf8proc-devel \ - wget \ - which \ - zlib-devel - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=ON \ - ARROW_GANDIVA_JAVA=ON \ - ARROW_GANDIVA=ON \ - ARROW_HOME=/usr/local \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_S3=ON \ - ARROW_USE_CCACHE=ON \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - CC=gcc \ - CXX=g++ \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXECUTABLES=ON \ - PARQUET_BUILD_EXAMPLES=ON \ - PATH=/usr/lib/ccache/:$PATH diff --git a/ci/docker/linux-apt-c-glib.dockerfile b/ci/docker/linux-apt-c-glib.dockerfile deleted file mode 100644 index 12c6e23a00d8..000000000000 --- a/ci/docker/linux-apt-c-glib.dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -RUN apt-get update -y -q && \ - apt-get install -y -q \ - python3 \ - python3-pip \ - gtk-doc-tools \ - libgirepository1.0-dev \ - libglib2.0-doc \ - lsb-release \ - luarocks \ - pkg-config \ - ruby-dev && \ - if [ "$(lsb_release --codename --short)" = "xenial" ]; then \ - apt-get install -y -q --no-install-recommends -t xenial-backports \ - ninja-build; \ - fi && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN luarocks install lgi - -# pip on Ubuntu 20.04 may be buggy: -# -# Collecting meson -# Downloading meson-0.53.2.tar.gz (1.6 MB) -# Installing build dependencies: started -# Installing build dependencies: finished with status 'done' -# Getting requirements to build wheel: started -# Getting requirements to build wheel: finished with status 'error' -# ERROR: Command errored out with exit status 1: -# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay -# cwd: /tmp/pip-install-jn79a_kh/meson -# Complete output (1 lines): -# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py' -# ---------------------------------------- -# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output. -RUN (python3 -m pip install meson || \ - python3 -m pip install --no-use-pep517 meson) && \ - gem install --no-document bundler - -COPY c_glib/Gemfile /arrow/c_glib/ -RUN bundle install --gemfile /arrow/c_glib/Gemfile - -ENV ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_INSTALL_NAME_RPATH=OFF diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile deleted file mode 100644 index 20cb889f28d8..000000000000 --- a/ci/docker/linux-apt-docs.dockerfile +++ /dev/null @@ -1,107 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -ARG r=4.0 -ARG jdk=8 - -# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu/ -RUN apt-get update -y && \ - apt-get install -y \ - dirmngr \ - apt-transport-https \ - software-properties-common && \ - apt-key adv \ - --keyserver keyserver.ubuntu.com \ - --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \ - apt-get install -y --no-install-recommends \ - autoconf-archive \ - automake \ - curl \ - doxygen \ - gobject-introspection \ - gtk-doc-tools \ - libcurl4-openssl-dev \ - libfontconfig1-dev \ - libfribidi-dev \ - libgirepository1.0-dev \ - libglib2.0-doc \ - libharfbuzz-dev \ - libtiff-dev \ - libtool \ - libxml2-dev \ - ninja-build \ - nvidia-cuda-toolkit \ - openjdk-${jdk}-jdk-headless \ - pandoc \ - r-base=${r}* \ - rsync \ - ruby-dev \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64 - -ARG maven=3.5.4 -COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/util_download_apache.sh \ - "maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" /opt -ENV PATH=/opt/apache-maven-${maven}/bin:$PATH -RUN mvn -version - -ARG node=14 -RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ - apt-get install -y nodejs && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - npm install -g yarn - -RUN pip install \ - meson \ - breathe \ - ipython \ - sphinx \ - pydata-sphinx-theme - -COPY c_glib/Gemfile /arrow/c_glib/ -RUN gem install --no-document bundler && \ - bundle install --gemfile /arrow/c_glib/Gemfile - -# Ensure parallel R package installation, set CRAN repo mirror, -# and use pre-built binaries where possible -COPY ci/etc/rprofile /arrow/ci/etc/ -RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site -# Also ensure parallel compilation of C/C++ code -RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Makeconf - -COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ -COPY r/DESCRIPTION /arrow/r/ -RUN /arrow/ci/scripts/r_deps.sh /arrow && \ - R -e "install.packages('pkgdown')" - -ENV ARROW_FLIGHT=ON \ - ARROW_PYTHON=ON \ - ARROW_S3=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_USE_GLOG=OFF \ - CMAKE_UNITY_BUILD=ON \ diff --git a/ci/docker/linux-apt-jni.dockerfile b/ci/docker/linux-apt-jni.dockerfile deleted file mode 100644 index 1abbf05af3bc..000000000000 --- a/ci/docker/linux-apt-jni.dockerfile +++ /dev/null @@ -1,86 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -# pipefail is enabled for proper error detection in the `wget | apt-key add` -# step -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -ENV DEBIAN_FRONTEND noninteractive - -ARG llvm -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - lsb-release \ - software-properties-common \ - wget && \ - code_name=$(lsb_release --codename --short) && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - apt-add-repository -y \ - "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${llvm} main" && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - ca-certificates \ - ccache \ - clang-${llvm} \ - cmake \ - git \ - g++ \ - gcc \ - libboost-all-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - liblz4-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - llvm-${llvm}-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - rapidjson-dev \ - tzdata \ - zlib1g-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ARG cmake=3.11.4 -RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz | tar -xzf - -C /opt -ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH - -ENV ARROW_BUILD_TESTS=OFF \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA_JAVA=ON \ - ARROW_GANDIVA=ON \ - ARROW_HOME=/usr/local \ - ARROW_JNI=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA_JAVA_CLIENT=ON \ - ARROW_PLASMA=ON \ - ARROW_USE_CCACHE=ON \ - CC=gcc \ - CXX=g++ \ - ORC_SOURCE=BUNDLED \ - PATH=/usr/lib/ccache/:$PATH \ - Protobuf_SOURCE=BUNDLED diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile deleted file mode 100644 index a5c063c74c5d..000000000000 --- a/ci/docker/linux-apt-lint.dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM hadolint/hadolint:v1.17.2 AS hadolint -FROM ${base} - -ARG clang_tools -RUN apt-get update && \ - apt-get install -y -q \ - clang-${clang_tools} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - clang-tools-${clang_tools} \ - cmake \ - curl \ - libclang-${clang_tools}-dev \ - llvm-${clang_tools}-dev \ - openjdk-11-jdk-headless \ - python3 \ - python3-dev \ - python3-pip \ - ruby \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Docker linter -COPY --from=hadolint /bin/hadolint /usr/bin/hadolint - -# IWYU -COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ -RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} - -# Rust linter -ARG rust=nightly-2022-01-17 -RUN curl https://sh.rustup.rs -sSf | \ - sh -s -- --default-toolchain stable -y -ENV PATH /root/.cargo/bin:$PATH -RUN rustup install ${rust} && \ - rustup default ${rust} && \ - rustup component add rustfmt - -# Use python3 by default in scripts -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -COPY dev/archery/requirements.txt \ - dev/archery/requirements-lint.txt \ - /arrow/dev/archery/ -RUN pip install \ - -r arrow/dev/archery/requirements.txt \ - -r arrow/dev/archery/requirements-lint.txt - -ENV LC_ALL=C.UTF-8 \ - LANG=C.UTF-8 diff --git a/ci/docker/linux-apt-python-3.dockerfile b/ci/docker/linux-apt-python-3.dockerfile deleted file mode 100644 index 753ba0d3aea4..000000000000 --- a/ci/docker/linux-apt-python-3.dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -RUN apt-get update -y -q && \ - apt-get install -y -q \ - python3 \ - python3-pip \ - python3-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install -U pip setuptools - -COPY python/requirements-build.txt \ - python/requirements-test.txt \ - /arrow/python/ - -RUN pip install \ - -r arrow/python/requirements-build.txt \ - -r arrow/python/requirements-test.txt - -ENV ARROW_PYTHON=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_USE_GLOG=OFF \ diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile deleted file mode 100644 index f47044e334b9..000000000000 --- a/ci/docker/linux-apt-r.dockerfile +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} -ARG arch - -# Build R -# [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04 -# [2] https://linuxize.com/post/how-to-install-r-on-ubuntu-18-04/#installing-r-packages-from-cran -ARG r=3.6 -RUN apt-get update -y && \ - apt-get install -y \ - dirmngr \ - apt-transport-https \ - software-properties-common && \ - apt-key adv \ - --keyserver keyserver.ubuntu.com \ - --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - # NOTE: R 3.5 and 3.6 are available in the repos with -cran35 suffix - # for trusty, xenial, bionic, and eoan (as of May 2020) - # -cran40 has 4.0 versions for bionic and focal - # R 3.2, 3.3, 3.4 are available without the suffix but only for trusty and xenial - # TODO: make sure OS version and R version are valid together and conditionally set repo suffix - # This is a hack to turn 3.6 into 35 and 4.0 into 40: - add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran'$(echo "${r}" | tr -d . | tr 6 5)'/' && \ - apt-get install -y \ - r-base=${r}* \ - # system libs needed by core R packages - libxml2-dev \ - libgit2-dev \ - libssl-dev \ - # install clang to mirror what was done on Travis - clang \ - clang-format \ - clang-tidy \ - # R CMD CHECK --as-cran needs pdflatex to build the package manual - texlive-latex-base \ - # Need locales so we can set UTF-8 - locales \ - # Need Python to check py-to-r bridge - python3 \ - python3-pip \ - python3-dev && \ - locale-gen en_US.UTF-8 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Ensure parallel R package installation, set CRAN repo mirror, -# and use pre-built binaries where possible -COPY ci/etc/rprofile /arrow/ci/etc/ -RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site -# Also ensure parallel compilation of C/C++ code -RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Makeconf - -COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ -COPY r/DESCRIPTION /arrow/r/ -RUN /arrow/ci/scripts/r_deps.sh /arrow - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -# Set up Python 3 and its dependencies -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -COPY python/requirements-build.txt /arrow/python/ -RUN pip install -r arrow/python/requirements-build.txt - -ENV \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=OFF \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=OFF \ - ARROW_PYTHON=ON \ - ARROW_S3=ON \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_GLOG=OFF \ - LC_ALL=en_US.UTF-8 diff --git a/ci/docker/linux-apt-ruby.dockerfile b/ci/docker/linux-apt-ruby.dockerfile deleted file mode 100644 index 58fd65bd57a7..000000000000 --- a/ci/docker/linux-apt-ruby.dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# depends on a C GLib image -ARG base -FROM ${base} - -COPY ruby/ /arrow/ruby/ -RUN bundle install --gemfile /arrow/ruby/Gemfile -RUN \ - for package in /arrow/ruby/*; do \ - bundle install --gemfile ${package}/Gemfile; \ - done diff --git a/ci/docker/linux-dnf-python-3.dockerfile b/ci/docker/linux-dnf-python-3.dockerfile deleted file mode 100644 index 8c3c5c701339..000000000000 --- a/ci/docker/linux-dnf-python-3.dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -RUN dnf install -y \ - python3 \ - python3-pip \ - python3-devel - -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -COPY python/requirements-build.txt \ - python/requirements-test.txt \ - /arrow/python/ - -RUN pip install \ - -r arrow/python/requirements-build.txt \ - -r arrow/python/requirements-test.txt - -ENV ARROW_PYTHON=ON \ - ARROW_BUILD_STATIC=OFF \ - ARROW_BUILD_TESTS=OFF \ - ARROW_BUILD_UTILITIES=OFF \ - ARROW_USE_GLOG=OFF \ diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile deleted file mode 100644 index ac414829d42b..000000000000 --- a/ci/docker/linux-r.dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# General purpose Dockerfile to take a Docker image containing R -# and install Arrow R package dependencies - -ARG base -FROM ${base} - -ARG r_bin=R -ENV R_BIN=${r_bin} - -ARG r_dev=FALSE -ENV ARROW_R_DEV=${r_dev} - -ARG devtoolset_version=-1 -ENV DEVTOOLSET_VERSION=${devtoolset_version} - -# Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) -ENV PATH "${RPREFIX}/bin:${PATH}" - -# Patch up some of the docker images -COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/ -COPY ci/etc/rprofile /arrow/ci/etc/ -COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/r_docker_configure.sh - -COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ -COPY r/DESCRIPTION /arrow/r/ -RUN /arrow/ci/scripts/r_deps.sh /arrow diff --git a/ci/docker/python-sdist.dockerfile b/ci/docker/python-sdist.dockerfile deleted file mode 100644 index 853b532ab5e9..000000000000 --- a/ci/docker/python-sdist.dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM amd64/ubuntu:20.04 - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - git \ - python3-pip && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -COPY python/requirements-build.txt \ - /arrow/python/requirements-build.txt -RUN pip3 install --requirement /arrow/python/requirements-build.txt - -ENV PYTHON=/usr/bin/python3 diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile deleted file mode 100644 index 19246a46764b..000000000000 --- a/ci/docker/python-wheel-manylinux-201x.dockerfile +++ /dev/null @@ -1,110 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base -FROM ${base} - -ARG arch_alias -ARG arch_short_alias - -RUN yum install -y git flex curl autoconf zip wget - -# Install CMake -ARG cmake=3.19.3 -RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \ - tar -xzf - --directory /usr/local --strip-components=1 - -# Install Ninja -ARG ninja=1.10.2 -RUN mkdir /tmp/ninja && \ - wget -q https://github.com/ninja-build/ninja/archive/v${ninja}.tar.gz -O - | \ - tar -xzf - --directory /tmp/ninja --strip-components=1 && \ - cd /tmp/ninja && \ - ./configure.py --bootstrap && \ - mv ninja /usr/local/bin && \ - rm -rf /tmp/ninja - -# Install ccache -ARG ccache=4.1 -RUN mkdir /tmp/ccache && \ - wget -q https://github.com/ccache/ccache/archive/v${ccache}.tar.gz -O - | \ - tar -xzf - --directory /tmp/ccache --strip-components=1 && \ - cd /tmp/ccache && \ - mkdir build && \ - cd build && \ - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON .. && \ - ninja install && \ - rm -rf /tmp/ccache - -# Install vcpkg -ARG vcpkg -RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \ - git -C /opt/vcpkg checkout ${vcpkg} && \ - /opt/vcpkg/bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics && \ - ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg - -# Patch ports files as needed -COPY ci/vcpkg arrow/ci/vcpkg -RUN cd /opt/vcpkg && git apply --ignore-whitespace /arrow/ci/vcpkg/ports.patch - -ARG build_type=release -ENV CMAKE_BUILD_TYPE=${build_type} \ - VCPKG_FORCE_SYSTEM_BINARIES=1 \ - VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \ - VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \ - VCPKG_FEATURE_FLAGS=-manifests - -# Need to install the boost-build prior installing the boost packages, otherwise -# vcpkg will raise an error. -# TODO(kszucs): factor out the package enumeration to a text file and reuse it -# from the windows image and potentially in a future macos wheel build -RUN vcpkg install --clean-after-build \ - boost-build:${arch_short_alias}-linux && \ - vcpkg install --clean-after-build \ - abseil \ - aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ - boost-filesystem \ - brotli \ - bzip2 \ - c-ares \ - curl \ - flatbuffers \ - gflags \ - glog \ - grpc \ - lz4 \ - openssl \ - orc \ - protobuf \ - rapidjson \ - re2 \ - snappy \ - thrift \ - utf8proc \ - zlib \ - zstd - -ARG python=3.6 -ENV PYTHON_VERSION=${python} -RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \ - echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh - -SHELL ["/bin/bash", "-i", "-c"] -ENTRYPOINT ["/bin/bash", "-i", "-c"] - -COPY python/requirements-wheel-build.txt /arrow/python/ -RUN pip install -r /arrow/python/requirements-wheel-build.txt diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile b/ci/docker/python-wheel-manylinux-test.dockerfile deleted file mode 100644 index 55c27d1d7bbd..000000000000 --- a/ci/docker/python-wheel-manylinux-test.dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG arch -ARG python -FROM ${arch}/python:${python} - -# RUN pip install --upgrade pip - -# pandas doesn't provide wheel for aarch64 yet, so cache the compiled -# test dependencies in a docker image -COPY python/requirements-wheel-test.txt /arrow/python/ -RUN pip install -r /arrow/python/requirements-wheel-test.txt diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile deleted file mode 100644 index 0f66a20396eb..000000000000 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# based on mcr.microsoft.com/windows/servercore:ltsc2019 -# contains choco and vs2017 preinstalled -FROM abrarov/msvc-2017:2.10.0 - -# Install CMake and Ninja -RUN choco install --no-progress -r -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' && \ - choco install --no-progress -r -y gzip wget ninja - -# Add unix tools to path -RUN setx path "%path%;C:\Program Files\Git\usr\bin" - -# Install vcpkg -ARG vcpkg -RUN git clone https://github.com/Microsoft/vcpkg && \ - git -C vcpkg checkout %vcpkg% && \ - vcpkg\bootstrap-vcpkg.bat -disableMetrics -win64 && \ - setx PATH "%PATH%;C:\vcpkg" - -# Patch ports files as needed -COPY ci/vcpkg arrow/ci/vcpkg -RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch - -# Configure vcpkg and install dependencies -# NOTE: use windows batch environment notation for build arguments in RUN -# statements but bash notation in ENV statements -# VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system -# cmake's and ninja's versions are recent enough -COPY ci/vcpkg arrow/ci/vcpkg -ARG build_type=release -ENV CMAKE_BUILD_TYPE=${build_type} \ - VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \ - VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \ - VCPKG_FEATURE_FLAGS=-manifests -RUN vcpkg install --clean-after-build \ - abseil \ - aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ - boost-filesystem \ - boost-multiprecision \ - boost-system \ - brotli \ - bzip2 \ - c-ares \ - curl \ - flatbuffers \ - gflags \ - glog \ - grpc \ - lz4 \ - openssl \ - orc \ - protobuf \ - rapidjson \ - re2 \ - snappy \ - thrift \ - utf8proc \ - zlib \ - zstd - -# Remove previous installations of python from the base image -RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ - rm -rf Python* - -# Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0) -ARG python=3.6 -RUN (if "%python%"=="3.6" setx PYTHON_VERSION 3.6.8) & \ - (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.4) & \ - (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.6) & \ - (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.1) -RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% -RUN python -m pip install -U pip - -COPY python/requirements-wheel-build.txt arrow/python/ -RUN pip install -r arrow/python/requirements-wheel-build.txt - -# TODO(kszucs): set clcache as the compiler -ENV CLCACHE_DIR="C:\clcache" -RUN pip install clcache - -# For debugging purposes -# RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip -# RUN unzip Dependencies_x64_Release.zip -d Dependencies && setx path "%path%;C:\Depencencies" diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile deleted file mode 100644 index 4b855b526108..000000000000 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ /dev/null @@ -1,128 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base=amd64/ubuntu:18.04 -FROM ${base} - -# pipefail is enabled for proper error detection in the `wget | apt-key add` -# step -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -ENV DEBIAN_FRONTEND=noninteractive - -# Installs LLVM toolchain, for Gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build. -ARG clang_tools -ARG llvm -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -ge 10 ]; then \ - echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ - fi && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - clang-${clang_tools} \ - clang-${llvm} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - llvm-${llvm}-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Installs C++ toolchain and dependencies -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libcurl4-openssl-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - liblz4-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libutf8proc-dev \ - libzstd-dev \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - rapidjson-dev \ - tzdata && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Prioritize system packages and local installation -# The following dependencies will be downloaded due to missing/invalid packages -# provided by the distribution: -# - libc-ares-dev does not install CMake config files -# - flatbuffer is not packaged -# - libgtest-dev only provide sources -# - libprotobuf-dev only provide sources -# - thrift is too old -# - s3 tests would require boost-asio that is included since Boost 1.66.0 -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=ON \ - ARROW_HDFS=ON \ - ARROW_HOME=/usr/local \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_TSAN=OFF \ - ARROW_USE_UBSAN=OFF \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXECUTABLES=ON \ - PARQUET_BUILD_EXAMPLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - Thrift_SOURCE=BUNDLED diff --git a/ci/docker/ubuntu-18.04-csharp.dockerfile b/ci/docker/ubuntu-18.04-csharp.dockerfile deleted file mode 100644 index 624ce259d5ae..000000000000 --- a/ci/docker/ubuntu-18.04-csharp.dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG platform=bionic -ARG dotnet=3.1 -FROM mcr.microsoft.com/dotnet/core/sdk:${dotnet}-${platform} - -RUN dotnet tool install --tool-path /usr/local/bin sourcelink diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile deleted file mode 100644 index 3a37ace13811..000000000000 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ /dev/null @@ -1,135 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base=amd64/ubuntu:20.04 -FROM ${base} -ARG arch - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -# Installs LLVM toolchain, for Gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build. -ARG clang_tools -ARG llvm -RUN if [ "${llvm}" -gt "10" ]; then \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ - echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ - fi \ - fi && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - clang-${clang_tools} \ - clang-${llvm} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - llvm-${llvm}-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Installs C++ toolchain and dependencies -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libgflags-dev \ - libcurl4-openssl-dev \ - libgoogle-glog-dev \ - liblz4-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - rapidjson-dev \ - tzdata \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -# Prioritize system packages and local installation -# The following dependencies will be downloaded due to missing/invalid packages -# provided by the distribution: -# - libc-ares-dev does not install CMake config files -# - flatbuffer is not packaged -# - libgtest-dev only provide sources -# - libprotobuf-dev only provide sources -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=ON \ - ARROW_HDFS=ON \ - ARROW_HOME=/usr/local \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_UBSAN=OFF \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXAMPLES=ON \ - PARQUET_BUILD_EXECUTABLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - PYTHON=python3 diff --git a/ci/docker/ubuntu-20.10-cpp.dockerfile b/ci/docker/ubuntu-20.10-cpp.dockerfile deleted file mode 100644 index 80eb072e7ed4..000000000000 --- a/ci/docker/ubuntu-20.10-cpp.dockerfile +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base=amd64/ubuntu:20.10 -FROM ${base} -ARG arch - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -# Installs LLVM toolchain, for Gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build. -ARG clang_tools -ARG llvm -RUN if [ "${llvm}" -gt "10" ]; then \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ - echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ - fi \ - fi && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - clang-${clang_tools} \ - clang-${llvm} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - llvm-${llvm}-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Installs C++ toolchain and dependencies -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libgflags-dev \ - libcurl4-openssl-dev \ - libgoogle-glog-dev \ - libgrpc++-dev \ - liblz4-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - protobuf-compiler-grpc \ - rapidjson-dev \ - tzdata \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local - -# Prioritize system packages and local installation -# The following dependencies will be downloaded due to missing/invalid packages -# provided by the distribution: -# - libc-ares-dev does not install CMake config files -# - flatbuffer is not packaged -# - libgtest-dev only provide sources -# - libprotobuf-dev only provide sources -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=ON \ - ARROW_HDFS=ON \ - ARROW_HOME=/usr/local \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_UBSAN=OFF \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXAMPLES=ON \ - PARQUET_BUILD_EXECUTABLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - PYTHON=python3 diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD deleted file mode 100644 index c5b55eef42ae..000000000000 --- a/ci/scripts/PKGBUILD +++ /dev/null @@ -1,134 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -_realname=arrow -pkgbase=mingw-w64-${_realname} -pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}" -pkgver=3.0.0.9000 -pkgrel=8000 -pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)" -arch=("any") -url="https://arrow.apache.org/" -license=("Apache-2.0") -depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" - "${MINGW_PACKAGE_PREFIX}-libutf8proc" - "${MINGW_PACKAGE_PREFIX}-re2" - "${MINGW_PACKAGE_PREFIX}-thrift" - "${MINGW_PACKAGE_PREFIX}-snappy" - "${MINGW_PACKAGE_PREFIX}-zlib" - "${MINGW_PACKAGE_PREFIX}-lz4" - "${MINGW_PACKAGE_PREFIX}-zstd") -makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" - "${MINGW_PACKAGE_PREFIX}-cmake" - "${MINGW_PACKAGE_PREFIX}-gcc") -options=("staticlibs" "strip" "!buildflags") - -# For installing from a local checkout, set source_dir to . and don't include -# a "source" param below -source_dir="$ARROW_HOME" -# else -# source_dir=apache-${_realname}-${pkgver} - -# For released version: -#source=("https://archive.apache.org/dist/arrow/arrow-${pkgver}/apache-arrow-${pkgver}.tar.gz") -#sha256sums=("ac2a77dd9168e9892e432c474611e86ded0be6dfe15f689c948751d37f81391a") -# For github dev version: -# Append `#commit=54b1b2f688e5e84b4c664b1e12a95f93b94ab2f3` to the URL to select a revision -# source=("${source_dir}"::"git+https://github.com/apache/arrow") -# sha256sums=("SKIP") -# source_dir="${APPVEYOR_BUILD_FOLDER}/${source_dir}" - -cpp_build_dir=build-${CARCH}-cpp - -pkgver() { - # The only purpose of this here is to cause the job to error if the - # version in pkgver is different from what is in r/DESCRIPTION - grep Version "${source_dir}/r/DESCRIPTION" | cut -d " " -f 2 -} - -build() { - ARROW_CPP_DIR="${source_dir}/cpp" - [[ -d ${cpp_build_dir} ]] && rm -rf ${cpp_build_dir} - mkdir -p ${cpp_build_dir} - pushd ${cpp_build_dir} - - # The Rtools libutf8proc is a static lib, but Findutf8proc.cmake doesn't - # set the appropriate compiler definition. - export CPPFLAGS="-DUTF8PROC_STATIC" - - # This is the difference between rtools-packages and rtools-backports - # Remove this when submitting to rtools-packages - if [ "$RTOOLS_VERSION" = "35" ]; then - export CC="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin/gcc" - export CXX="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin/g++" - export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH" - export CPPFLAGS="${CPPFLAGS} -I${MINGW_PREFIX}/include" - export LIBS="-L${MINGW_PREFIX}/libs" - export ARROW_S3=OFF - export ARROW_WITH_RE2=OFF - else - export ARROW_S3=ON - export ARROW_WITH_RE2=ON - fi - - MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - ${MINGW_PREFIX}/bin/cmake.exe \ - ${ARROW_CPP_DIR} \ - -G "MSYS Makefiles" \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_BUILD_STATIC=ON \ - -DARROW_BUILD_UTILITIES=OFF \ - -DARROW_COMPUTE=ON \ - -DARROW_CSV=ON \ - -DARROW_DATASET=ON \ - -DARROW_FILESYSTEM=ON \ - -DARROW_HDFS=OFF \ - -DARROW_JEMALLOC=OFF \ - -DARROW_JSON=ON \ - -DARROW_LZ4_USE_SHARED=OFF \ - -DARROW_MIMALLOC=ON \ - -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ - -DARROW_PARQUET=ON \ - -DARROW_S3="${ARROW_S3}" \ - -DARROW_SNAPPY_USE_SHARED=OFF \ - -DARROW_USE_GLOG=OFF \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_RE2="${ARROW_WITH_RE2}" \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_ZSTD_USE_SHARED=OFF \ - -DARROW_CXXFLAGS="${CPPFLAGS}" \ - -DCMAKE_BUILD_TYPE="release" \ - -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ - -DCMAKE_UNITY_BUILD=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON - - make -j3 - popd -} - -package() { - make -C ${cpp_build_dir} DESTDIR="${pkgdir}" install - - local PREFIX_DEPS=$(cygpath -am ${MINGW_PREFIX}) - pushd "${pkgdir}${MINGW_PREFIX}/lib/pkgconfig" - for pc in *.pc; do - sed -s "s|${PREFIX_DEPS}|${MINGW_PREFIX}|g" -i $pc - done - popd -} diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh deleted file mode 100755 index ce3cea18e715..000000000000 --- a/ci/scripts/c_glib_build.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/c_glib -build_dir=${2}/c_glib -: ${ARROW_GLIB_GTK_DOC:=false} -: ${ARROW_GLIB_DEVELOPMENT_MODE:=false} - -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig - -export CFLAGS="-DARROW_NO_DEPRECATED_API" -export CXXFLAGS="-DARROW_NO_DEPRECATED_API" - -mkdir -p ${build_dir} - -# Build with Meson -meson --prefix=$ARROW_HOME \ - --libdir=lib \ - -Ddevelopment_mode=${ARROW_GLIB_DEVELOPMENT_MODE} \ - -Dgtk_doc=${ARROW_GLIB_GTK_DOC} \ - ${build_dir} \ - ${source_dir} - -pushd ${build_dir} -ninja -ninja install -popd diff --git a/ci/scripts/c_glib_test.sh b/ci/scripts/c_glib_test.sh deleted file mode 100755 index 25c54138ed65..000000000000 --- a/ci/scripts/c_glib_test.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/c_glib -build_dir=${2}/c_glib - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig -export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 - -pushd ${source_dir} - -ruby test/run-test.rb - -if [[ "$(uname -s)" == "Linux" ]]; then - # TODO(kszucs): on osx it fails to load 'lgi.corelgilua51' despite that lgi - # was installed by luarocks - pushd example/lua - lua write-batch.lua - lua read-batch.lua - lua write-stream.lua - lua read-stream.lua - popd -fi - -popd - -pushd ${build_dir} -example/extension-type -popd diff --git a/ci/scripts/ccache_setup.sh b/ci/scripts/ccache_setup.sh deleted file mode 100755 index f77fbb373647..000000000000 --- a/ci/scripts/ccache_setup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -echo "ARROW_USE_CCACHE=ON" >> $GITHUB_ENV -echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV -echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV -echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV -echo "CCACHE_MAXSIZE=500M" >> $GITHUB_ENV diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh deleted file mode 100755 index 8a1e4f32f3a9..000000000000 --- a/ci/scripts/cpp_build.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/cpp -build_dir=${2}/cpp -with_docs=${3:-false} - -: ${ARROW_USE_CCACHE:=OFF} - -# TODO(kszucs): consider to move these to CMake -if [ ! -z "${CONDA_PREFIX}" ]; then - echo -e "===\n=== Conda environment for build\n===" - conda list - - export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_AR=${AR} -DCMAKE_RANLIB=${RANLIB}" - export ARROW_GANDIVA_PC_CXX_FLAGS=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') -elif [ -x "$(command -v xcrun)" ]; then - export ARROW_GANDIVA_PC_CXX_FLAGS="-isysroot;$(xcrun --show-sdk-path)" -fi - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo -e "===\n=== ccache statistics before build\n===" - ccache -s -fi - -mkdir -p ${build_dir} -pushd ${build_dir} - -cmake -G "${CMAKE_GENERATOR:-Ninja}" \ - -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \ - -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \ - -DARROW_BUILD_BENCHMARKS_REFERENCE=${ARROW_BUILD_BENCHMARKS:-OFF} \ - -DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-OFF} \ - -DARROW_BUILD_INTEGRATION=${ARROW_BUILD_INTEGRATION:-OFF} \ - -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \ - -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \ - -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ - -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ - -DARROW_COMPUTE=${ARROW_COMPUTE:-ON} \ - -DARROW_CSV=${ARROW_CSV:-ON} \ - -DARROW_CUDA=${ARROW_CUDA:-OFF} \ - -DARROW_CXXFLAGS=${ARROW_CXXFLAGS:-} \ - -DARROW_DATASET=${ARROW_DATASET:-ON} \ - -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ - -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \ - -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \ - -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \ - -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ - -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ - -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ - -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \ - -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ - -DARROW_HDFS=${ARROW_HDFS:-ON} \ - -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \ - -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ - -DARROW_JNI=${ARROW_JNI:-OFF} \ - -DARROW_JSON=${ARROW_JSON:-ON} \ - -DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \ - -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \ - -DARROW_ORC=${ARROW_ORC:-OFF} \ - -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \ - -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \ - -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \ - -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ - -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ - -DARROW_S3=${ARROW_S3:-OFF} \ - -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ - -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ - -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ - -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-ON} \ - -DARROW_USE_GLOG=${ARROW_USE_GLOG:-OFF} \ - -DARROW_USE_LD_GOLD=${ARROW_USE_LD_GOLD:-OFF} \ - -DARROW_USE_PRECOMPILED_HEADERS=${ARROW_USE_PRECOMPILED_HEADERS:-OFF} \ - -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \ - -DARROW_USE_TSAN=${ARROW_USE_TSAN:-OFF} \ - -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \ - -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-OFF} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-OFF} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \ - -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ - -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ - -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ - -DBOOST_SOURCE=${BOOST_SOURCE:-} \ - -DBrotli_SOURCE=${Brotli_SOURCE:-} \ - -DBUILD_WARNING_LEVEL=${BUILD_WARNING_LEVEL:-CHECKIN} \ - -Dc-ares_SOURCE=${cares_SOURCE:-} \ - -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ - -DCMAKE_C_FLAGS="${CFLAGS:-}" \ - -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \ - -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ - -Dgflags_SOURCE=${gflags_SOURCE:-} \ - -DgRPC_SOURCE=${gRPC_SOURCE:-} \ - -DGTest_SOURCE=${GTest_SOURCE:-} \ - -DLz4_SOURCE=${Lz4_SOURCE:-} \ - -DORC_SOURCE=${ORC_SOURCE:-} \ - -DPARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-OFF} \ - -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \ - -DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON} \ - -DProtobuf_SOURCE=${Protobuf_SOURCE:-} \ - -DRapidJSON_SOURCE=${RapidJSON_SOURCE:-} \ - -Dre2_SOURCE=${re2_SOURCE:-} \ - -DSnappy_SOURCE=${Snappy_SOURCE:-} \ - -DThrift_SOURCE=${Thrift_SOURCE:-} \ - -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ - -Dzstd_SOURCE=${zstd_SOURCE:-} \ - ${CMAKE_ARGS} \ - ${source_dir} - -if [ ! -z "${CPP_MAKE_PARALLELISM}" ]; then - time cmake --build . --target install -- -j${CPP_MAKE_PARALLELISM} -else - time cmake --build . --target install -fi - -popd - -if [ -x "$(command -v ldconfig)" ]; then - ldconfig -fi - -if [ "${ARROW_USE_CCACHE}" == "ON" ]; then - echo -e "===\n=== ccache statistics after build\n===" - ccache -s -fi - -if [ "${with_docs}" == "true" ]; then - pushd ${source_dir}/apidoc - doxygen - popd -fi diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh deleted file mode 100755 index 1bf0a3b88940..000000000000 --- a/ci/scripts/cpp_test.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/cpp -build_dir=${2}/cpp -binary_output_dir=${build_dir}/${ARROW_BUILD_TYPE:-debug} - -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data -export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_PATH} - -# By default, aws-sdk tries to contact a non-existing local ip host -# to retrieve metadata. Disable this so that S3FileSystem tests run faster. -export AWS_EC2_METADATA_DISABLED=TRUE - -ctest_options=() -case "$(uname)" in - Linux) - n_jobs=$(nproc) - ;; - Darwin) - n_jobs=$(sysctl -n hw.ncpu) - ;; - MINGW*) - n_jobs=${NUMBER_OF_PROCESSORS:-1} - # TODO: Enable these crashed tests. - # https://issues.apache.org/jira/browse/ARROW-9072 - exclude_tests="gandiva-internals-test" - exclude_tests="${exclude_tests}|gandiva-projector-test" - exclude_tests="${exclude_tests}|gandiva-utf8-test" - if [ "${MSYSTEM}" = "MINGW32" ]; then - exclude_tests="${exclude_tests}|gandiva-projector-test" - exclude_tests="${exclude_tests}|gandiva-binary-test" - exclude_tests="${exclude_tests}|gandiva-boolean-expr-test" - exclude_tests="${exclude_tests}|gandiva-date-time-test" - exclude_tests="${exclude_tests}|gandiva-decimal-single-test" - exclude_tests="${exclude_tests}|gandiva-decimal-test" - exclude_tests="${exclude_tests}|gandiva-filter-project-test" - exclude_tests="${exclude_tests}|gandiva-filter-test" - exclude_tests="${exclude_tests}|gandiva-hash-test" - exclude_tests="${exclude_tests}|gandiva-if-expr-test" - exclude_tests="${exclude_tests}|gandiva-in-expr-test" - exclude_tests="${exclude_tests}|gandiva-literal-test" - exclude_tests="${exclude_tests}|gandiva-null-validity-test" - fi - ctest_options+=(--exclude-regex "${exclude_tests}") - ;; - *) - n_jobs=${NPROC:-1} - ;; -esac - -pushd ${build_dir} - -if ! which python > /dev/null 2>&1; then - export PYTHON=python3 -fi -ctest \ - --label-regex unittest \ - --output-on-failure \ - --parallel ${n_jobs} \ - --timeout 300 \ - "${ctest_options[@]}" - -if [ "${ARROW_FUZZING}" == "ON" ]; then - # Fuzzing regression tests - ${binary_output_dir}/arrow-ipc-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-stream/crash-* - ${binary_output_dir}/arrow-ipc-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-stream/*-testcase-* - ${binary_output_dir}/arrow-ipc-file-fuzz ${ARROW_TEST_DATA}/arrow-ipc-file/*-testcase-* - ${binary_output_dir}/arrow-ipc-tensor-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-tensor-stream/*-testcase-* - if [ "${ARROW_PARQUET}" == "ON" ]; then - ${binary_output_dir}/parquet-arrow-fuzz ${ARROW_TEST_DATA}/parquet/fuzzing/*-testcase-* - fi -fi - -popd diff --git a/ci/scripts/csharp_build.sh b/ci/scripts/csharp_build.sh deleted file mode 100755 index 5a3976794874..000000000000 --- a/ci/scripts/csharp_build.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/csharp - -pushd ${source_dir} -dotnet build -popd diff --git a/ci/scripts/csharp_pack.sh b/ci/scripts/csharp_pack.sh deleted file mode 100755 index e9dfc664ec55..000000000000 --- a/ci/scripts/csharp_pack.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -source_dir=${1}/csharp - -pushd ${source_dir} -dotnet pack -c Release -popd diff --git a/ci/scripts/csharp_test.sh b/ci/scripts/csharp_test.sh deleted file mode 100755 index 9e4e35dd40d1..000000000000 --- a/ci/scripts/csharp_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/csharp - -pushd ${source_dir} -dotnet test -for pdb in artifacts/Apache.Arrow/*/*/Apache.Arrow.pdb; do - sourcelink test ${pdb} -done -popd diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh deleted file mode 100755 index e6ee768ee876..000000000000 --- a/ci/scripts/docs_build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -ex - -arrow_dir=${1} -build_dir=${2}/docs - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH} -export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 -export CFLAGS="-DARROW_NO_DEPRECATED_API" -export CXXFLAGS="-DARROW_NO_DEPRECATED_API" - -ncpus=$(python3 -c "import os; print(os.cpu_count())") - -# Sphinx docs -sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir} - -# C++ - original doxygen -# rsync -a ${arrow_dir}/cpp/apidoc/ ${build_dir}/cpp - -# R -rsync -a ${arrow_dir}/r/docs/ ${build_dir}/r - -# C GLib -rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_dir}/c_glib - -# Java -rsync -a ${arrow_dir}/java/target/site/apidocs/ ${build_dir}/java/reference - -# Javascript -rsync -a ${arrow_dir}/js/doc/ ${build_dir}/js diff --git a/ci/scripts/go_build.sh b/ci/scripts/go_build.sh deleted file mode 100755 index 7093be4d2386..000000000000 --- a/ci/scripts/go_build.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/go - -pushd ${source_dir}/arrow - -go get -d -t -v ./... -go install -v ./... - -popd - -pushd ${source_dir}/parquet - -go get -d -t -v ./... -go install -v ./... - -popd diff --git a/ci/scripts/go_test.sh b/ci/scripts/go_test.sh deleted file mode 100755 index 7dd873df3e1b..000000000000 --- a/ci/scripts/go_test.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/go - -pushd ${source_dir}/arrow - -for d in $(go list ./... | grep -v vendor); do - go test $d -done - -popd - -pushd ${source_dir}/parquet - -for d in $(go list ./... | grep -v vendor); do - go test $d -done - -popd diff --git a/ci/scripts/install_conda.sh b/ci/scripts/install_conda.sh deleted file mode 100755 index f4d313b63dfb..000000000000 --- a/ci/scripts/install_conda.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -declare -A archs -archs=([amd64]=x86_64 - [arm32v7]=armv7l - [ppc64le]=ppc64le - [i386]=x86) - -declare -A platforms -platforms=([windows]=Windows - [macos]=MacOSX - [linux]=Linux) - -if [ "$#" -ne 4 ]; then - echo "Usage: $0 " - exit 1 -elif [[ -z ${archs[$1]} ]]; then - echo "Unexpected architecture: ${1}" - exit 1 -elif [[ -z ${platforms[$2]} ]]; then - echo "Unexpected platform: ${2}" - exit 1 -fi - -arch=${archs[$1]} -platform=${platforms[$2]} -version=$3 -prefix=$4 - -echo "Downloading Miniconda installer..." -wget -nv https://repo.continuum.io/miniconda/Miniconda3-${version}-${platform}-${arch}.sh -O /tmp/miniconda.sh -bash /tmp/miniconda.sh -b -p ${prefix} -rm /tmp/miniconda.sh - -# Like "conda init", but for POSIX sh rather than bash -ln -s ${prefix}/etc/profile.d/conda.sh /etc/profile.d/conda.sh - -# Configure -source /etc/profile.d/conda.sh -conda config --add channels conda-forge -conda config --set channel_priority strict -conda config --set show_channel_urls True -conda config --set remote_connect_timeout_secs 12 - -# Update and clean -conda update --all -y -conda clean --all -y diff --git a/ci/scripts/install_dask.sh b/ci/scripts/install_dask.sh deleted file mode 100755 index 954ce3249d9d..000000000000 --- a/ci/scripts/install_dask.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -dask=$1 - -if [ "${dask}" = "master" ]; then - pip install https://github.com/dask/dask/archive/main.tar.gz#egg=dask[dataframe] -elif [ "${dask}" = "latest" ]; then - conda install -q dask -else - conda install -q dask=${dask} -fi -conda clean --all diff --git a/ci/scripts/install_iwyu.sh b/ci/scripts/install_iwyu.sh deleted file mode 100755 index 3cd2cbc95fe6..000000000000 --- a/ci/scripts/install_iwyu.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -eu - -source_dir=${1:-/tmp/iwyu} -install_prefix=${2:-/usr/local} -clang_tools_version=${3:-8} - -iwyu_branch_name="clang_${clang_tools_version}" -if [ ${clang_tools_version} -lt 10 ]; then - iwyu_branch_name="${iwyu_branch_name}.0" -fi - -git clone --single-branch --branch ${iwyu_branch_name} \ - https://github.com/include-what-you-use/include-what-you-use.git ${source_dir} - -mkdir -p ${source_dir}/build -pushd ${source_dir}/build - -# Build IWYU for current Clang -export CC=clang-${clang_tools_version} -export CXX=clang++-${clang_tools_version} - -cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${clang_tools_version} \ - -DCMAKE_INSTALL_PREFIX=${install_prefix} \ - ${source_dir} -make -j4 -make install - -popd - -rm -rf ${source_dir} diff --git a/ci/scripts/install_kartothek.sh b/ci/scripts/install_kartothek.sh deleted file mode 100755 index 4d88943b6a9e..000000000000 --- a/ci/scripts/install_kartothek.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -karthothek=$1 -target=$2 - -git clone --recurse-submodules https://github.com/JDASoftwareGroup/kartothek "${target}" -if [ "${kartothek}" = "master" ]; then - git -C "${target}" checkout master; -elif [ "${kartothek}" = "latest" ]; then - git -C "${target}" checkout $(git describe --tags); -else - git -C "${target}" checkout ${kartothek}; -fi - -pushd "${target}" -pip install --no-deps . -popd diff --git a/ci/scripts/install_minio.sh b/ci/scripts/install_minio.sh deleted file mode 100755 index 42f7ce040e04..000000000000 --- a/ci/scripts/install_minio.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -declare -A archs -archs=([amd64]=amd64 - [arm64v8]=arm64 - [arm32v7]=arm - [s390x]=s390x) - -declare -A platforms -platforms=([linux]=linux - [macos]=darwin) - -arch=${archs[$1]} -platform=${platforms[$2]} -version=$3 -prefix=$4 - -if [ "$#" -ne 4 ]; then - echo "Usage: $0 " - exit 1 -elif [[ -z ${arch} ]]; then - echo "Unexpected architecture: ${1}" - exit 1 -elif [[ -z ${platform} ]]; then - echo "Unexpected platform: ${2}" - exit 1 -elif [[ ${version} != "latest" ]]; then - echo "Cannot fetch specific versions of minio, only latest is supported." - exit 1 -fi - -wget -nv -P ${prefix}/bin https://dl.min.io/server/minio/release/${platform}-${arch}/minio -chmod +x ${prefix}/bin/minio diff --git a/ci/scripts/install_osx_sdk.sh b/ci/scripts/install_osx_sdk.sh deleted file mode 100755 index 896d084e0b9d..000000000000 --- a/ci/scripts/install_osx_sdk.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -if [ ${using_homebrew} != "yes" ]; then - export MACOSX_DEPLOYMENT_TARGET="10.9" - export CONDA_BUILD_SYSROOT="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk" - - if [[ ! -d ${CONDA_BUILD_SYSROOT} || "$OSX_FORCE_SDK_DOWNLOAD" == "1" ]]; then - echo "downloading ${macosx_deployment_target} sdk" - curl -L -O https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz - tar -xf MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz -C "$(dirname "$CONDA_BUILD_SYSROOT")" - # set minimum sdk version to our target - plutil -replace MinimumSDKVersion -string ${MACOSX_DEPLOYMENT_TARGET} $(xcode-select -p)/Platforms/MacOSX.platform/Info.plist - plutil -replace DTSDKName -string macosx${MACOSX_DEPLOYMENT_TARGET}internal $(xcode-select -p)/Platforms/MacOSX.platform/Info.plist - fi - - if [ -d "${CONDA_BUILD_SYSROOT}" ]; then - echo "Found CONDA_BUILD_SYSROOT: ${CONDA_BUILD_SYSROOT}" - else - echo "Missing CONDA_BUILD_SYSROOT: ${CONDA_BUILD_SYSROOT}" - exit 1 - fi -fi diff --git a/ci/scripts/install_pandas.sh b/ci/scripts/install_pandas.sh deleted file mode 100755 index 5aca65f825a5..000000000000 --- a/ci/scripts/install_pandas.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -pandas=$1 -numpy=${2:-"latest"} - -if [ "${numpy}" = "nightly" ]; then - pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple --pre numpy -elif [ "${numpy}" = "latest" ]; then - pip install numpy -else - pip install numpy==${numpy} -fi - -if [ "${pandas}" = "master" ]; then - pip install git+https://github.com/pandas-dev/pandas.git --no-build-isolation -elif [ "${pandas}" = "nightly" ]; then - pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple --pre pandas -elif [ "${pandas}" = "latest" ]; then - pip install pandas -else - pip install pandas==${pandas} -fi diff --git a/ci/scripts/install_spark.sh b/ci/scripts/install_spark.sh deleted file mode 100755 index 936313fd809c..000000000000 --- a/ci/scripts/install_spark.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -spark=$1 -target=$2 - -git clone https://github.com/apache/spark "${target}" -git -C "${target}" checkout "${spark}" diff --git a/ci/scripts/install_turbodbc.sh b/ci/scripts/install_turbodbc.sh deleted file mode 100755 index a71520bebf48..000000000000 --- a/ci/scripts/install_turbodbc.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -turbodbc=$1 -target=$2 - -git clone --recurse-submodules https://github.com/blue-yonder/turbodbc "${target}" -if [ "${turbodbc}" = "master" ]; then - git -C "${target}" checkout master; -elif [ "${turbodbc}" = "latest" ]; then - git -C "${target}" checkout $(git describe --tags); -else - git -C "${target}" checkout ${turbodbc}; -fi diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh deleted file mode 100755 index 5d2e71916ed3..000000000000 --- a/ci/scripts/integration_arrow.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/cpp -build_dir=${2}/cpp - -gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration - -pip install -e $arrow_dir/dev/archery - -archery integration --with-all --run-flight \ - --gold-dirs=$gold_dir/0.14.1 \ - --gold-dirs=$gold_dir/0.17.1 \ - --gold-dirs=$gold_dir/1.0.0-bigendian \ - --gold-dirs=$gold_dir/1.0.0-littleendian \ - --gold-dirs=$gold_dir/2.0.0-compression \ - --gold-dirs=$gold_dir/4.0.0-shareddict \ diff --git a/ci/scripts/integration_dask.sh b/ci/scripts/integration_dask.sh deleted file mode 100755 index a73592877588..000000000000 --- a/ci/scripts/integration_dask.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -# check that optional pyarrow modules are available -# because pytest would just skip the dask tests -python -c "import pyarrow.orc" -python -c "import pyarrow.parquet" - -# check that dask.dataframe is correctly installed -python -c "import dask.dataframe" - -# TODO(kszucs): the following tests are also uses pyarrow -# pytest -sv --pyargs dask.bytes.tests.test_s3 -# pytest -sv --pyargs dask.bytes.tests.test_hdfs -# pytest -sv --pyargs dask.bytes.tests.test_local - -# skip failing pickle test, see https://github.com/dask/dask/issues/6374 -pytest -v --pyargs dask.dataframe.tests.test_dataframe -k "not test_dataframe_picklable" -pytest -v --pyargs dask.dataframe.io.tests.test_orc -# skip failing parquet tests, see https://github.com/dask/dask/issues/6243 -# test_illegal_column_name can be removed once next dask release is out -# (https://github.com/dask/dask/pull/6378) -pytest -v --pyargs dask.dataframe.io.tests.test_parquet \ - -k "not test_to_parquet_pyarrow_w_inconsistent_schema_by_partition_fails_by_default and not test_timeseries_nulls_in_schema and not test_illegal_column_name" diff --git a/ci/scripts/integration_hdfs.sh b/ci/scripts/integration_hdfs.sh deleted file mode 100755 index c95449379c32..000000000000 --- a/ci/scripts/integration_hdfs.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source_dir=${1}/cpp -build_dir=${2}/cpp - -export CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath --glob) -export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop -export LIBHDFS3_CONF=$HADOOP_CONF_DIR/hdfs-site.xml -export ARROW_LIBHDFS3_DIR=$CONDA_PREFIX/lib - -libhdfs_dir=$HADOOP_HOME/lib/native -hadoop_home=$HADOOP_HOME - -function use_hadoop_home() { - unset ARROW_LIBHDFS_DIR - export HADOOP_HOME=$hadoop_home -} - -function use_libhdfs_dir() { - unset HADOOP_HOME - export ARROW_LIBHDFS_DIR=$libhdfs_dir -} - -# execute cpp tests -export ARROW_HDFS_TEST_LIBHDFS_REQUIRE=ON -pushd ${build_dir} - -debug/arrow-io-hdfs-test -debug/arrow-hdfs-test - -use_libhdfs_dir -debug/arrow-io-hdfs-test -debug/arrow-hdfs-test -use_hadoop_home - -popd - -# cannot use --pyargs with custom arguments like --hdfs or --only-hdfs, because -# pytest ignores them, see https://github.com/pytest-dev/pytest/issues/3517 -export PYARROW_TEST_HDFS=ON - -export PYARROW_HDFS_TEST_LIBHDFS_REQUIRE=ON - -pytest -vs --pyargs pyarrow.tests.test_fs -pytest -vs --pyargs pyarrow.tests.test_hdfs - -use_libhdfs_dir -pytest -vs --pyargs pyarrow.tests.test_fs -pytest -vs --pyargs pyarrow.tests.test_hdfs -use_hadoop_home diff --git a/ci/scripts/integration_hiveserver2.sh b/ci/scripts/integration_hiveserver2.sh deleted file mode 100755 index 36fba5ca8d8a..000000000000 --- a/ci/scripts/integration_hiveserver2.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -arrow_dir=${1} -source_dir=${1}/cpp -build_dir=${2}/cpp - -${arrow_dir}/ci/scripts/util_wait_for_it.sh impala:21050 -t 300 -s -- echo "impala is up" - -pushd ${build_dir} - -# ninja hiveserver2-test -debug/hiveserver2-test - -popd diff --git a/ci/scripts/integration_kartothek.sh b/ci/scripts/integration_kartothek.sh deleted file mode 100755 index 379569b9c996..000000000000 --- a/ci/scripts/integration_kartothek.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -# check that optional pyarrow modules are available -# because pytest would just skip the pyarrow tests -python -c "import pyarrow.parquet" - -# check that kartothek is correctly installed -python -c "import kartothek" - -pushd /kartothek -# See ARROW-12314, test_load_dataframes_columns_raises_missing skipped because of changed error message -pytest -n0 --ignore tests/cli/test_query.py -k "not test_load_dataframes_columns_raises_missing" diff --git a/ci/scripts/integration_spark.sh b/ci/scripts/integration_spark.sh deleted file mode 100755 index a53a62971caf..000000000000 --- a/ci/scripts/integration_spark.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# exit on any error -set -eu - -source_dir=${1} -spark_dir=${2} - -# Test Spark with latest PyArrow only, don't build with latest Arrow Java -test_pyarrow_only=${3:-false} - -# Spark branch to checkout -spark_version=${SPARK_VERSION:-master} - -# Use old behavior that always dropped tiemzones. -export PYARROW_IGNORE_TIMEZONE=1 - -if [ "${SPARK_VERSION:0:2}" == "2." ]; then - # https://github.com/apache/spark/blob/master/docs/sql-pyspark-pandas-with-arrow.md#compatibility-setting-for-pyarrow--0150-and-spark-23x-24x - export ARROW_PRE_0_15_IPC_FORMAT=1 -fi - -# Get Arrow Java version -pushd ${source_dir}/java - arrow_version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | sed -n -e '/^\[.*\]/ !{ /^[0-9]/ { p; q } }'` -popd - -export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=warn" -export MAVEN_OPTS="${MAVEN_OPTS} -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" - -pushd ${spark_dir} - - if [ "${test_pyarrow_only}" == "true" ]; then - echo "Building Spark ${SPARK_VERSION} to test pyarrow only" - - # Build Spark only - build/mvn -B -DskipTests package - - else - - # Update Spark pom with the Arrow version just installed and build Spark, need package phase for pyspark - echo "Building Spark ${SPARK_VERSION} with Arrow ${arrow_version}" - mvn versions:set-property -Dproperty=arrow.version -DnewVersion=${arrow_version} - - # Build Spark with new Arrow Java - build/mvn -B -DskipTests package -pl sql/core -pl assembly -am - - spark_scala_tests=( - "org.apache.spark.sql.execution.arrow" - "org.apache.spark.sql.execution.vectorized.ColumnarBatchSuite" - "org.apache.spark.sql.execution.vectorized.ArrowColumnVectorSuite") - - (echo "Testing Spark:"; IFS=$'\n'; echo "${spark_scala_tests[*]}") - - # TODO: should be able to only build spark-sql tests with adding "-pl sql/core" but not currently working - build/mvn -B -Dtest=none -DwildcardSuites=$(IFS=,; echo "${spark_scala_tests[*]}") test - fi - - # Run pyarrow related Python tests only - spark_python_tests=( - "pyspark.sql.tests.test_arrow" - "pyspark.sql.tests.test_pandas_map" - "pyspark.sql.tests.test_pandas_cogrouped_map" - "pyspark.sql.tests.test_pandas_grouped_map" - "pyspark.sql.tests.test_pandas_udf" - "pyspark.sql.tests.test_pandas_udf_scalar" - "pyspark.sql.tests.test_pandas_udf_grouped_agg" - "pyspark.sql.tests.test_pandas_udf_window") - - (echo "Testing PySpark:"; IFS=$'\n'; echo "${spark_python_tests[*]}") - python/run-tests --testnames "$(IFS=,; echo "${spark_python_tests[*]}")" --python-executables python -popd diff --git a/ci/scripts/integration_turbodbc.sh b/ci/scripts/integration_turbodbc.sh deleted file mode 100755 index f56074358a6e..000000000000 --- a/ci/scripts/integration_turbodbc.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -source_dir=${1} -build_dir=${2}/turbodbc - -# check that optional pyarrow modules are available -# because pytest would just skip the pyarrow tests -python -c "import pyarrow.orc" -python -c "import pyarrow.parquet" - -mkdir -p ${build_dir} -pushd ${build_dir} - -cmake -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ - -DCMAKE_CXX_FLAGS=${CXXFLAGS} \ - -DPYTHON_EXECUTABLE=$(which python) \ - -GNinja \ - ${source_dir} -ninja install - -# TODO(ARROW-5074) -export LD_LIBRARY_PATH="${ARROW_HOME}/lib:${LD_LIBRARY_PATH}" -export ODBCSYSINI="${source_dir}/travis/odbc/" - -service postgresql start -ctest --output-on-failure - -popd diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh deleted file mode 100755 index b8a7f7ced747..000000000000 --- a/ci/scripts/java_build.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/java -cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug} -with_docs=${3:-false} - -if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then - # Since some files for s390_64 are not available at maven central, - # download pre-build files from bintray and install them explicitly - mvn_install="mvn install:install-file" - wget="wget" - bintray_base_url="https://dl.bintray.com/apache/arrow" - - bintray_dir="protoc-binary" - group="com.google.protobuf" - artifact="protoc" - ver="3.7.1" - classifier="linux-s390_64" - extension="exe" - target=${artifact}-${ver}-${classifier}.${extension} - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} - # protoc requires libprotoc.so.18 libprotobuf.so.18 - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/libprotoc.so.18 - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/libprotobuf.so.18 - mkdir -p ${ARROW_HOME}/lib - cp lib*.so.18 ${ARROW_HOME}/lib - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ARROW_HOME}/lib - - bintray_dir="protoc-gen-grpc-java-binary" - group="io.grpc" - artifact="protoc-gen-grpc-java" - ver="1.30.2" - classifier="linux-s390_64" - extension="exe" - target=${artifact}-${ver}-${classifier}.${extension} - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} - - bintray_dir="netty-binary" - group="io.netty" - artifact="netty-transport-native-unix-common" - ver="4.1.48.Final" - classifier="linux-s390_64" - extension="jar" - target=${artifact}-${ver}-${classifier}.${extension} - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} - artifact="netty-transport-native-epoll" - extension="jar" - target=${artifact}-${ver}-${classifier}.${extension} - ${wget} ${bintray_base_url}/${bintray_dir}/${ver}/${target} - ${mvn_install} -DgroupId=${group} -DartifactId=${artifact} -Dversion=${ver} -Dclassifier=${classifier} -Dpackaging=${extension} -Dfile=$(pwd)/${target} -fi - -mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -# Use `2 * ncores` threads -mvn="${mvn} -T 2C" - -pushd ${source_dir} - -${mvn} install - -if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then - ${mvn} -Pshade-flatbuffers install -fi - -if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then - ${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install -fi - -if [ "${ARROW_PLASMA}" = "ON" ]; then - pushd ${source_dir}/plasma - ${mvn} clean install - popd -fi - -if [ "${with_docs}" == "true" ]; then - ${mvn} -Dcheckstyle.skip=true install site -fi - -popd diff --git a/ci/scripts/java_test.sh b/ci/scripts/java_test.sh deleted file mode 100755 index da9e45280ec9..000000000000 --- a/ci/scripts/java_test.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} -source_dir=${1}/java -cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug} - -# For JNI and Plasma tests -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PLASMA_STORE=${ARROW_HOME}/bin/plasma-store-server - -mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -# Use `2 * ncores` threads -mvn="${mvn} -T 2C" - -pushd ${source_dir} - -${mvn} test - -if [ "${ARROW_JNI}" = "ON" ]; then - ${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir} -fi - -if [ "${ARROW_PLASMA}" = "ON" ]; then - pushd ${source_dir}/plasma - java -cp target/test-classes:target/classes \ - -Djava.library.path=${cpp_build_dir} \ - org.apache.arrow.plasma.PlasmaClientTest - popd -fi - -popd diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh deleted file mode 100755 index 10ceb41ee658..000000000000 --- a/ci/scripts/js_build.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/js -with_docs=${2:-false} - -pushd ${source_dir} - -yarn --frozen-lockfile -# TODO(kszucs): linting should be moved to archery -yarn lint:ci -yarn build - -if [ "${with_docs}" == "true" ]; then - yarn doc -fi - -popd diff --git a/ci/scripts/js_test.sh b/ci/scripts/js_test.sh deleted file mode 100755 index 345d6cb811e7..000000000000 --- a/ci/scripts/js_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/js - -pushd ${source_dir} - -yarn lint -yarn test - -popd diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh deleted file mode 100755 index cb6ca30a64ea..000000000000 --- a/ci/scripts/msys2_setup.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -target=$1 - -packages=() -case "${target}" in - cpp|c_glib|ruby) - packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp) - packages+=(${MINGW_PACKAGE_PREFIX}-boost) - packages+=(${MINGW_PACKAGE_PREFIX}-brotli) - packages+=(${MINGW_PACKAGE_PREFIX}-ccache) - packages+=(${MINGW_PACKAGE_PREFIX}-clang) - packages+=(${MINGW_PACKAGE_PREFIX}-cmake) - packages+=(${MINGW_PACKAGE_PREFIX}-gcc) - packages+=(${MINGW_PACKAGE_PREFIX}-gflags) - packages+=(${MINGW_PACKAGE_PREFIX}-grpc) - packages+=(${MINGW_PACKAGE_PREFIX}-gtest) - packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc) - packages+=(${MINGW_PACKAGE_PREFIX}-llvm) - packages+=(${MINGW_PACKAGE_PREFIX}-lz4) - packages+=(${MINGW_PACKAGE_PREFIX}-ninja) - packages+=(${MINGW_PACKAGE_PREFIX}-polly) - packages+=(${MINGW_PACKAGE_PREFIX}-protobuf) - packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy) - packages+=(${MINGW_PACKAGE_PREFIX}-rapidjson) - packages+=(${MINGW_PACKAGE_PREFIX}-re2) - packages+=(${MINGW_PACKAGE_PREFIX}-snappy) - packages+=(${MINGW_PACKAGE_PREFIX}-thrift) - packages+=(${MINGW_PACKAGE_PREFIX}-zlib) - packages+=(${MINGW_PACKAGE_PREFIX}-zstd) - ;; -esac - -case "${target}" in - c_glib|ruby) - packages+=(${MINGW_PACKAGE_PREFIX}-gobject-introspection) - packages+=(${MINGW_PACKAGE_PREFIX}-gtk-doc) - packages+=(${MINGW_PACKAGE_PREFIX}-meson) - ;; -esac - -pacman \ - --needed \ - --noconfirm \ - --refresh \ - --sync \ - "${packages[@]}" - -"$(dirname $0)/ccache_setup.sh" -echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV diff --git a/ci/scripts/msys2_system_clean.sh b/ci/scripts/msys2_system_clean.sh deleted file mode 100755 index a356aee66600..000000000000 --- a/ci/scripts/msys2_system_clean.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -pacman \ - --cascade \ - --noconfirm \ - --nosave \ - --recursive \ - --remove \ - ${MINGW_PACKAGE_PREFIX}-clang-tools-extra \ - ${MINGW_PACKAGE_PREFIX}-gcc-ada \ - ${MINGW_PACKAGE_PREFIX}-gcc-fortran \ - ${MINGW_PACKAGE_PREFIX}-gcc-libgfortran \ - ${MINGW_PACKAGE_PREFIX}-gcc-objc \ - ${MINGW_PACKAGE_PREFIX}-libgccjit diff --git a/ci/scripts/msys2_system_upgrade.sh b/ci/scripts/msys2_system_upgrade.sh deleted file mode 100755 index aecd30893320..000000000000 --- a/ci/scripts/msys2_system_upgrade.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -# https://www.msys2.org/news/#2020-06-29-new-packagers -msys2_repo_base_url=https://repo.msys2.org/msys -# Mirror -msys2_repo_base_url=https://sourceforge.net/projects/msys2/files/REPOS/MSYS2 -msys2_keyring_pkg=msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz -for suffix in "" ".sig"; do - curl \ - --location \ - --remote-name \ - --show-error \ - --silent \ - ${msys2_repo_base_url}/x86_64/${msys2_keyring_pkg}${suffix} -done -pacman-key --verify ${msys2_keyring_pkg}.sig -pacman \ - --noconfirm \ - --upgrade \ - ${msys2_keyring_pkg} - - -pacman \ - --noconfirm \ - --refresh \ - --refresh \ - --sync \ - --sysupgrade \ - --sysupgrade diff --git a/ci/scripts/python_benchmark.sh b/ci/scripts/python_benchmark.sh deleted file mode 100755 index 3a35298dc044..000000000000 --- a/ci/scripts/python_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Check the ASV benchmarking setup. -# Unfortunately this won't ensure that all benchmarks succeed -# (see https://github.com/airspeed-velocity/asv/issues/449) -source deactivate -conda create -y -q -n pyarrow_asv python=$PYTHON_VERSION -conda activate pyarrow_asv -pip install -q git+https://github.com/pitrou/asv.git@customize_commands - -export PYARROW_WITH_PARQUET=1 -export PYARROW_WITH_PLASMA=1 -export PYARROW_WITH_ORC=0 -export PYARROW_WITH_GANDIVA=0 - -pushd $ARROW_PYTHON_DIR -# Workaround for https://github.com/airspeed-velocity/asv/issues/631 -git fetch --depth=100 origin master:master -# Generate machine information (mandatory) -asv machine --yes -# Run benchmarks on the changeset being tested -asv run --no-pull --show-stderr --quick HEAD^! -popd # $ARROW_PYTHON_DIR diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh deleted file mode 100755 index ec6d723b2a73..000000000000 --- a/ci/scripts/python_build.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/python -build_dir=${2}/python - -if [ ! -z "${CONDA_PREFIX}" ]; then - echo -e "===\n=== Conda environment for build\n===" - conda list -fi - -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja} -export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug} -export PYARROW_WITH_S3=${ARROW_S3:-OFF} -export PYARROW_WITH_ORC=${ARROW_ORC:-OFF} -export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF} -export PYARROW_WITH_HDFS=${ARROW_HDFS:-OFF} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF} -export PYARROW_WITH_PLASMA=${ARROW_PLASMA:-OFF} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF} -export PYARROW_WITH_DATASET=${ARROW_DATASET:-OFF} - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} - -pushd ${source_dir} - -relative_build_dir=$(realpath --relative-to=. $build_dir) - -# not nice, but prevents mutating the mounted the source directory for docker -${PYTHON:-python} \ - setup.py build --build-base $build_dir \ - install --single-version-externally-managed \ - --record $relative_build_dir/record.txt - -popd diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh deleted file mode 100755 index f9e9359b6f66..000000000000 --- a/ci/scripts/python_sdist_build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -source_dir=${1}/python - -pushd ${source_dir} -export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-} -${PYTHON:-python} setup.py sdist -popd diff --git a/ci/scripts/python_sdist_test.sh b/ci/scripts/python_sdist_test.sh deleted file mode 100755 index 1388ca09e436..000000000000 --- a/ci/scripts/python_sdist_test.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -arrow_dir=${1} - -export ARROW_SOURCE_DIR=${arrow_dir} -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data - -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja} -export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug} -export PYARROW_WITH_S3=${ARROW_S3:-OFF} -export PYARROW_WITH_ORC=${ARROW_ORC:-OFF} -export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF} -export PYARROW_WITH_HDFS=${ARROW_HDFS:-OFF} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF} -export PYARROW_WITH_PLASMA=${ARROW_PLASMA:-OFF} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF} -export PYARROW_WITH_DATASET=${ARROW_DATASET:-OFF} - -# TODO: Users should not require ARROW_HOME and pkg-config to find Arrow C++. -# Related: ARROW-9171 -# unset ARROW_HOME -# apt purge -y pkg-config - -if [ -n "${PYARROW_VERSION:-}" ]; then - sdist="${arrow_dir}/python/dist/pyarrow-${PYARROW_VERSION}.tar.gz" -else - sdist=$(ls "${arrow_dir}/python/dist/pyarrow-*.tar.gz" | sort -r | head -n1) -fi -${PYTHON:-python} -m pip install ${sdist} - -pytest -r s ${PYTEST_ARGS:-} --pyargs pyarrow diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh deleted file mode 100755 index 80a9cdef4a31..000000000000 --- a/ci/scripts/python_test.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -arrow_dir=${1} - -export ARROW_SOURCE_DIR=${arrow_dir} -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} - -# Enable some checks inside Python itself -export PYTHONDEVMODE=1 - -pytest -r s ${PYTEST_ARGS} --pyargs pyarrow diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh deleted file mode 100755 index 93e4939af23d..000000000000 --- a/ci/scripts/python_wheel_macos_build.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1} -build_dir=${2} - -echo "=== (${PYTHON_VERSION}) Clear output directories and leftovers ===" -# Clear output directories and leftovers -rm -rf ${build_dir}/install -rm -rf ${source_dir}/python/dist -rm -rf ${source_dir}/python/build -rm -rf ${source_dir}/python/repaired_wheels -rm -rf ${source_dir}/python/pyarrow/*.so -rm -rf ${source_dir}/python/pyarrow/*.so.* - -echo "=== (${PYTHON_VERSION}) Set OSX SDK and C flags ===" -# Arrow is 64-bit-only at the moment -export CFLAGS="-fPIC -arch x86_64 ${CFLAGS//-arch i386/}" -export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//-arch i386} -std=c++11" -export SDKROOT="$(xcrun --show-sdk-path)" - -echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" -: ${ARROW_DATASET:=ON} -: ${ARROW_FLIGHT:=ON} -: ${ARROW_GANDIVA:=OFF} -: ${ARROW_HDFS:=ON} -: ${ARROW_JEMALLOC:=ON} -: ${ARROW_MIMALLOC:=ON} -: ${ARROW_ORC:=ON} -: ${ARROW_PARQUET:=ON} -: ${ARROW_PLASMA:=ON} -: ${ARROW_S3:=ON} -: ${ARROW_TENSORFLOW:=ON} -: ${ARROW_WITH_BROTLI:=ON} -: ${ARROW_WITH_BZ2:=ON} -: ${ARROW_WITH_LZ4:=ON} -: ${ARROW_WITH_SNAPPY:=ON} -: ${ARROW_WITH_ZLIB:=ON} -: ${ARROW_WITH_ZSTD:=ON} -: ${CMAKE_BUILD_TYPE:=release} -: ${CMAKE_GENERATOR:=Ninja} -: ${VCPKG_FEATURE_FLAGS:=-manifests} -: ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}} - -mkdir -p ${build_dir}/build -pushd ${build_dir}/build -cmake \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BUILD_STATIC=OFF \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_DATASET=${ARROW_DATASET} \ - -DARROW_DEPENDENCY_SOURCE="VCPKG" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_FLIGHT==${ARROW_FLIGHT} \ - -DARROW_GANDIVA=${ARROW_GANDIVA} \ - -DARROW_HDFS=${ARROW_HDFS} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC} \ - -DARROW_ORC=${ARROW_ORC} \ - -DARROW_PACKAGE_KIND="python-wheel-macos" \ - -DARROW_PARQUET=${ARROW_PARQUET} \ - -DARROW_PLASMA=${ARROW_PLASMA} \ - -DARROW_PYTHON=ON \ - -DARROW_RPATH_ORIGIN=ON \ - -DARROW_S3=${ARROW_S3} \ - -DARROW_TENSORFLOW=${ARROW_TENSORFLOW} \ - -DARROW_USE_CCACHE=ON \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_INSTALL_PREFIX=${build_dir}/install \ - -DCMAKE_UNITY_BUILD=ON \ - -DOPENSSL_USE_STATIC_LIBS=ON \ - -DVCPKG_MANIFEST_MODE=OFF \ - -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \ - -G ${CMAKE_GENERATOR} \ - ${source_dir}/cpp -cmake --build . --target install -popd - -# Check that we don't expose any unwanted symbols -# check_arrow_visibility - -echo "=== (${PYTHON_VERSION}) Building wheel ===" -export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE} -export PYARROW_BUNDLE_ARROW_CPP=1 -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR} -export PYARROW_INSTALL_TESTS=1 -export PYARROW_WITH_DATASET=${ARROW_DATASET} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA} -export PYARROW_WITH_HDFS=${ARROW_HDFS} -export PYARROW_WITH_ORC=${ARROW_ORC} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET} -export PYARROW_WITH_PLASMA=${ARROW_PLASMA} -export PYARROW_WITH_S3=${ARROW_S3} -# PyArrow build configuration -export PKG_CONFIG_PATH=/usr/lib/pkgconfig:${build_dir}/install/lib/pkgconfig - -pushd ${source_dir}/python -python setup.py bdist_wheel -popd - -echo "=== (${PYTHON_VERSION}) Show dynamic libraries the wheel depend on ===" -deps=$(delocate-listdeps ${source_dir}/python/dist/*.whl) - -if echo $deps | grep -v "^@rpath/lib\(arrow\|gandiva\|parquet\|plasma\)"; then - echo "There are non-bundled shared library dependencies." - exit 1 -fi diff --git a/ci/scripts/python_wheel_macos_test.sh b/ci/scripts/python_wheel_macos_test.sh deleted file mode 100755 index 6ac8576d484c..000000000000 --- a/ci/scripts/python_wheel_macos_test.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1} - -: ${ARROW_S3:=ON} - -export PYARROW_TEST_CYTHON=OFF -export PYARROW_TEST_DATASET=ON -export PYARROW_TEST_GANDIVA=OFF -export PYARROW_TEST_HDFS=ON -export PYARROW_TEST_ORC=ON -export PYARROW_TEST_PANDAS=ON -export PYARROW_TEST_PARQUET=ON -export PYARROW_TEST_PLASMA=ON -export PYARROW_TEST_S3=${ARROW_S3} -export PYARROW_TEST_TENSORFLOW=ON -export PYARROW_TEST_FLIGHT=ON - -export ARROW_TEST_DATA=${source_dir}/testing/data -export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data - -# Install the built wheels -pip install ${source_dir}/python/dist/*.whl - -# Test that the modules are importable -python -c " -import pyarrow -import pyarrow._hdfs -import pyarrow.csv -import pyarrow.dataset -import pyarrow.flight -import pyarrow.fs -import pyarrow.json -import pyarrow.orc -import pyarrow.parquet -import pyarrow.plasma -" - -if [ "${PYARROW_TEST_S3}" == "ON" ]; then - python -c "import pyarrow._s3fs" -fi - -# Install testing dependencies -pip install -r ${source_dir}/python/requirements-wheel-test.txt - -# Execute unittest -pytest -r s --pyargs pyarrow diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh deleted file mode 100755 index 312e1c3b9b7c..000000000000 --- a/ci/scripts/python_wheel_manylinux_build.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -function check_arrow_visibility { - nm --demangle --dynamic /tmp/arrow-dist/lib/libarrow.so > nm_arrow.log - - # Filter out Arrow symbols and see if anything remains. - # '_init' and '_fini' symbols may or not be present, we don't care. - # (note we must ignore the grep exit status when no match is found) - grep ' T ' nm_arrow.log | grep -v -E '(arrow|\b_init\b|\b_fini\b)' | cat - > visible_symbols.log - - if [[ -f visible_symbols.log && `cat visible_symbols.log | wc -l` -eq 0 ]]; then - return 0 - else - echo "== Unexpected symbols exported by libarrow.so ==" - cat visible_symbols.log - echo "================================================" - - exit 1 - fi -} - -echo "=== (${PYTHON_VERSION}) Clear output directories and leftovers ===" -# Clear output directories and leftovers -rm -rf /tmp/arrow-build -rm -rf /arrow/python/dist -rm -rf /arrow/python/build -rm -rf /arrow/python/repaired_wheels -rm -rf /arrow/python/pyarrow/*.so -rm -rf /arrow/python/pyarrow/*.so.* - -echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" -: ${ARROW_DATASET:=ON} -: ${ARROW_FLIGHT:=ON} -: ${ARROW_GANDIVA:=OFF} -: ${ARROW_HDFS:=ON} -: ${ARROW_JEMALLOC:=ON} -: ${ARROW_MIMALLOC:=ON} -: ${ARROW_ORC:=ON} -: ${ARROW_PARQUET:=ON} -: ${ARROW_PLASMA:=ON} -: ${ARROW_S3:=ON} -: ${ARROW_TENSORFLOW:=ON} -: ${ARROW_WITH_BROTLI:=ON} -: ${ARROW_WITH_BZ2:=ON} -: ${ARROW_WITH_LZ4:=ON} -: ${ARROW_WITH_SNAPPY:=ON} -: ${ARROW_WITH_ZLIB:=ON} -: ${ARROW_WITH_ZSTD:=ON} -: ${CMAKE_BUILD_TYPE:=release} -: ${CMAKE_UNITY_BUILD:=ON} -: ${CMAKE_GENERATOR:=Ninja} -: ${VCPKG_FEATURE_FLAGS:=-manifests} -: ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-linux-static-${CMAKE_BUILD_TYPE}}} - -mkdir /tmp/arrow-build -pushd /tmp/arrow-build -cmake \ - -DARROW_BROTLI_USE_SHARED=OFF \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BUILD_STATIC=OFF \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_DATASET=${ARROW_DATASET} \ - -DARROW_DEPENDENCY_SOURCE="VCPKG" \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_FLIGHT==${ARROW_FLIGHT} \ - -DARROW_GANDIVA=${ARROW_GANDIVA} \ - -DARROW_HDFS=${ARROW_HDFS} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC} \ - -DARROW_ORC=${ARROW_ORC} \ - -DARROW_PACKAGE_KIND="python-wheel-manylinux${MANYLINUX_VERSION}" \ - -DARROW_PARQUET=${ARROW_PARQUET} \ - -DARROW_PLASMA=${ARROW_PLASMA} \ - -DARROW_PYTHON=ON \ - -DARROW_RPATH_ORIGIN=ON \ - -DARROW_S3=${ARROW_S3} \ - -DARROW_TENSORFLOW=${ARROW_TENSORFLOW} \ - -DARROW_USE_CCACHE=ON \ - -DARROW_UTF8PROC_USE_SHARED=OFF \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ - -DOPENSSL_USE_STATIC_LIBS=ON \ - -DVCPKG_MANIFEST_MODE=OFF \ - -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \ - -G ${CMAKE_GENERATOR} \ - /arrow/cpp -cmake --build . --target install -popd - -# Check that we don't expose any unwanted symbols -check_arrow_visibility - -echo "=== (${PYTHON_VERSION}) Building wheel ===" -export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE} -export PYARROW_BUNDLE_ARROW_CPP=1 -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR} -export PYARROW_INSTALL_TESTS=1 -export PYARROW_WITH_DATASET=${ARROW_DATASET} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA} -export PYARROW_WITH_HDFS=${ARROW_HDFS} -export PYARROW_WITH_ORC=${ARROW_ORC} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET} -export PYARROW_WITH_PLASMA=${ARROW_PLASMA} -export PYARROW_WITH_S3=${ARROW_S3} -# PyArrow build configuration -export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/tmp/arrow-dist/lib/pkgconfig - -pushd /arrow/python -python setup.py bdist_wheel - -echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux${MANYLINUX_VERSION} ===" -auditwheel repair -L . dist/pyarrow-*.whl -w repaired_wheels -popd diff --git a/ci/scripts/python_wheel_manylinux_test.sh b/ci/scripts/python_wheel_manylinux_test.sh deleted file mode 100755 index 21987748f73f..000000000000 --- a/ci/scripts/python_wheel_manylinux_test.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -x -set -o pipefail - -case $# in - 1) KIND="$1" - case $KIND in - imports|unittests) ;; - *) echo "Invalid argument: '${KIND}', valid options are 'imports', 'unittests'" - exit 1 - ;; - esac - ;; - *) echo "Usage: $0 imports|unittests" - exit 1 - ;; -esac - -export PYARROW_TEST_CYTHON=OFF -export PYARROW_TEST_DATASET=ON -export PYARROW_TEST_GANDIVA=OFF -export PYARROW_TEST_HDFS=ON -export PYARROW_TEST_ORC=ON -export PYARROW_TEST_PANDAS=ON -export PYARROW_TEST_PARQUET=ON -export PYARROW_TEST_PLASMA=ON -export PYARROW_TEST_S3=ON -export PYARROW_TEST_TENSORFLOW=ON -export PYARROW_TEST_FLIGHT=ON - -export ARROW_TEST_DATA=/arrow/testing/data -export PARQUET_TEST_DATA=/arrow/submodules/parquet-testing/data - -# Install the built wheels -pip install /arrow/python/repaired_wheels/*.whl - -if [ "${KIND}" == "imports" ]; then - # Test that the modules are importable - python -c " -import pyarrow -import pyarrow._hdfs -import pyarrow._s3fs -import pyarrow.csv -import pyarrow.dataset -import pyarrow.flight -import pyarrow.fs -import pyarrow.json -import pyarrow.orc -import pyarrow.parquet -import pyarrow.plasma" -elif [ "${KIND}" == "unittests" ]; then - # Execute unittest, test dependencies must be installed - pytest -r s --pyargs pyarrow -fi diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat deleted file mode 100644 index 23be7f512d6e..000000000000 --- a/ci/scripts/python_wheel_windows_build.bat +++ /dev/null @@ -1,109 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@echo on - -echo "Building windows wheel..." - -call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" - -echo "=== (%PYTHON_VERSION%) Clear output directories and leftovers ===" -del /s /q C:\arrow-build -del /s /q C:\arrow-dist -del /s /q C:\arrow\python\dist -del /s /q C:\arrow\python\build -del /s /q C:\arrow\python\pyarrow\*.so -del /s /q C:\arrow\python\pyarrow\*.so.* - -echo "=== (%PYTHON_VERSION%) Building Arrow C++ libraries ===" -set ARROW_DATASET=ON -set ARROW_FLIGHT=ON -set ARROW_GANDIVA=OFF -set ARROW_HDFS=ON -set ARROW_ORC=OFF -set ARROW_PARQUET=ON -set ARROW_MIMALLOC=ON -set ARROW_S3=ON -set ARROW_TENSORFLOW=ON -set ARROW_WITH_BROTLI=ON -set ARROW_WITH_BZ2=ON -set ARROW_WITH_LZ4=ON -set ARROW_WITH_SNAPPY=ON -set ARROW_WITH_ZLIB=ON -set ARROW_WITH_ZSTD=ON -set CMAKE_UNITY_BUILD=ON -set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 -set VCPKG_FEATURE_FLAGS=-manifests - -mkdir C:\arrow-build -pushd C:\arrow-build -cmake ^ - -DARROW_BUILD_SHARED=ON ^ - -DARROW_BUILD_STATIC=OFF ^ - -DARROW_BUILD_TESTS=OFF ^ - -DARROW_CXXFLAGS="/MP" ^ - -DARROW_DATASET=%ARROW_DATASET% ^ - -DARROW_DEPENDENCY_SOURCE=VCPKG ^ - -DARROW_DEPENDENCY_USE_SHARED=OFF ^ - -DARROW_FLIGHT=%ARROW_FLIGHT% ^ - -DARROW_GANDIVA=%ARROW_GANDIVA% ^ - -DARROW_HDFS=%ARROW_HDFS% ^ - -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^ - -DARROW_ORC=%ARROW_ORC% ^ - -DARROW_PACKAGE_KIND="python-wheel-windows" ^ - -DARROW_PARQUET=%ARROW_PARQUET% ^ - -DARROW_PYTHON=ON ^ - -DARROW_S3=%ARROW_S3% ^ - -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^ - -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^ - -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^ - -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^ - -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ - -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ - -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ - -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ - -DCMAKE_CXX_COMPILER=clcache ^ - -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ - -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^ - -DMSVC_LINK_VERBOSE=ON ^ - -DVCPKG_MANIFEST_MODE=OFF ^ - -DVCPKG_TARGET_TRIPLET=x64-windows-static-md-%CMAKE_BUILD_TYPE% ^ - -G "%CMAKE_GENERATOR%" ^ - C:\arrow\cpp || exit /B -cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B -popd - -echo "=== (%PYTHON_VERSION%) Building wheel ===" -set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE% -set PYARROW_BUNDLE_ARROW_CPP=ON -set PYARROW_BUNDLE_BOOST=OFF -set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR% -set PYARROW_INSTALL_TESTS=ON -set PYARROW_WITH_DATASET=%ARROW_DATASET% -set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT% -set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA% -set PYARROW_WITH_HDFS=%ARROW_HDFS% -set PYARROW_WITH_ORC=%ARROW_ORC% -set PYARROW_WITH_PARQUET=%ARROW_PARQUET% -set PYARROW_WITH_S3=%ARROW_S3% -set ARROW_HOME=C:\arrow-dist - -pushd C:\arrow\python -@REM bundle the msvc runtime -cp "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\14.16.27012\x64\Microsoft.VC141.CRT\msvcp140.dll" pyarrow\ -python setup.py bdist_wheel || exit /B -popd diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat deleted file mode 100755 index 8352e5862266..000000000000 --- a/ci/scripts/python_wheel_windows_test.bat +++ /dev/null @@ -1,54 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@echo on - -set PYARROW_TEST_CYTHON=OFF -set PYARROW_TEST_DATASET=ON -set PYARROW_TEST_GANDIVA=OFF -set PYARROW_TEST_HDFS=ON -set PYARROW_TEST_ORC=OFF -set PYARROW_TEST_PANDAS=ON -set PYARROW_TEST_PARQUET=ON -set PYARROW_TEST_PLASMA=OFF -set PYARROW_TEST_S3=OFF -set PYARROW_TEST_TENSORFLOW=ON -set PYARROW_TEST_FLIGHT=ON - -set ARROW_TEST_DATA=C:\arrow\testing\data -set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data - -@REM Install the built wheels -python -m pip install numpy -python -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B - -@REM Test that the modules are importable -python -c "import pyarrow" -python -c "import pyarrow._hdfs" -python -c "import pyarrow._s3fs" -python -c "import pyarrow.csv" -python -c "import pyarrow.dataset" -python -c "import pyarrow.flight" -python -c "import pyarrow.fs" -python -c "import pyarrow.json" -python -c "import pyarrow.parquet" - -@REM Install testing dependencies -pip install -r C:\arrow\python\requirements-wheel-test.txt - -@REM Execute unittest -pytest -r s --pyargs pyarrow diff --git a/ci/scripts/r_build.sh b/ci/scripts/r_build.sh deleted file mode 100755 index d1907a87c894..000000000000 --- a/ci/scripts/r_build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=R} -source_dir=${1}/r -with_docs=${2:-false} - -pushd ${source_dir} - -${R_BIN} CMD INSTALL . - -if [ "${with_docs}" == "true" ]; then - ${R_BIN} -e "pkgdown::build_site(install = FALSE)" -fi - -popd \ No newline at end of file diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh deleted file mode 100755 index 7e9d2eac7a96..000000000000 --- a/ci/scripts/r_deps.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=R} - -source_dir=${1}/r - -pushd ${source_dir} - -# Install R package dependencies -${R_BIN} -e "install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys'))" -${R_BIN} -e "remotes::install_deps(dependencies = TRUE)" - -popd diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh deleted file mode 100755 index 3e553fe9edda..000000000000 --- a/ci/scripts/r_docker_configure.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=R} - -# The Dockerfile should have put this file here -if [ -f "/arrow/ci/etc/rprofile" ]; then - # Ensure parallel R package installation, set CRAN repo mirror, - # and use pre-built binaries where possible - cat /arrow/ci/etc/rprofile >> $(${R_BIN} RHOME)/etc/Rprofile.site -fi - -# Ensure parallel compilation of C/C++ code -echo "MAKEFLAGS=-j$(${R_BIN} -s -e 'cat(parallel::detectCores())')" >> $(${R_BIN} RHOME)/etc/Makeconf - -# Special hacking to try to reproduce quirks on fedora-clang-devel on CRAN -# which uses a bespoke clang compiled to use libc++ -# https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang -if [ "$RHUB_PLATFORM" = "linux-x86_64-fedora-clang" ]; then - dnf install -y libcxx-devel - sed -i.bak -E -e 's/(CXX1?1? =.*)/\1 -stdlib=libc++/g' $(${R_BIN} RHOME)/etc/Makeconf - rm -rf $(${R_BIN} RHOME)/etc/Makeconf.bak -fi - -# Special hacking to try to reproduce quirks on centos using non-default build -# tooling. -if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then - if [ "`which dnf`" ]; then - dnf install -y centos-release-scl - dnf install -y "devtoolset-$DEVTOOLSET_VERSION" - else - yum install -y centos-release-scl - yum install -y "devtoolset-$DEVTOOLSET_VERSION" - fi -fi - -# Install openssl for S3 support -if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then - if [ "`which dnf`" ]; then - dnf install -y libcurl-devel openssl-devel - elif [ "`which yum`" ]; then - yum install -y libcurl-devel openssl-devel - elif [ "`which zypper`" ]; then - zypper install -y libcurl-devel libopenssl-devel - else - apt-get update - apt-get install -y libcurl4-openssl-dev libssl-dev - fi - - # The Dockerfile should have put this file here - if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then - /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local - fi -fi - -# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786 -Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}' - -if [ "`which curl`" ]; then - # We need this on R >= 4.0 - curl -L https://sourceforge.net/projects/checkbaskisms/files/2.0.0.2/checkbashisms/download > /usr/local/bin/checkbashisms - chmod 755 /usr/local/bin/checkbashisms -fi diff --git a/ci/scripts/r_pkgdown_check.sh b/ci/scripts/r_pkgdown_check.sh deleted file mode 100755 index 327480a6b348..000000000000 --- a/ci/scripts/r_pkgdown_check.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Use this script to prevent errors in the pkgdown site being rendered due to missing YAML entries - -# all .Rd files in the repo -all_rd_files=`find ./r/man -maxdepth 1 -name "*.Rd" | sed -e 's/.\/r\/man\///g' | sed -e 's/.Rd//g' | sort` - -# .Rd files to exclude from search (i.e. are internal) -exclusions=`grep "\keyword{internal}" -rl ./r/man --include=*.Rd | sed -e 's/.\/r\/man\///g' | sed -e 's/.Rd//g' | sort` - -# .Rd files to check against pkgdown.yml -rd_files=`echo ${exclusions[@]} ${all_rd_files[@]} | tr ' ' '\n' | sort | uniq -u` - -# pkgdown sections -pkgdown_sections=`awk '/^[^ ]/{ f=/reference:/; next } f{ if (sub(/:$/,"")) pkg=$2; else print pkg, $2 }' ./r/_pkgdown.yml | grep -v "title:" | sort` - -# get things that appear in man files that don't appear in pkgdown sections -pkgdown_missing=`echo ${pkgdown_sections[@]} ${pkgdown_sections[@]} ${rd_files[@]} | tr ' ' '\n' | sort | uniq -u` - -# if any sections are missing raise an error -if ([ ${#pkgdown_missing} -ge 1 ]); then - echo "Error! $pkgdown_missing missing from ./r/_pkgdown.yml" - exit 1 -fi diff --git a/ci/scripts/r_sanitize.sh b/ci/scripts/r_sanitize.sh deleted file mode 100755 index 89963eb2dd86..000000000000 --- a/ci/scripts/r_sanitize.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=RDsan} - -source_dir=${1}/r - -${R_BIN} CMD INSTALL ${source_dir} -pushd ${source_dir}/tests - -export TEST_R_WITH_ARROW=TRUE -export UBSAN_OPTIONS="print_stacktrace=1,suppressions=/arrow/r/tools/ubsan.supp" -${R_BIN} < testthat.R > testthat.out 2>&1 || { cat testthat.out; exit 1; } - -cat testthat.out -if grep -q "runtime error" testthat.out; then - exit 1 -fi -popd diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh deleted file mode 100755 index 1f6a8721efc1..000000000000 --- a/ci/scripts/r_test.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${R_BIN:=R} - -source_dir=${1}/r - -pushd ${source_dir} - -printenv - -if [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then - export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} - export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} -fi -export _R_CHECK_COMPILATION_FLAGS_KNOWN_=${ARROW_R_CXXFLAGS} -if [ "$ARROW_R_DEV" = "TRUE" ]; then - # These are used in the Arrow C++ build and are not a problem - export _R_CHECK_COMPILATION_FLAGS_KNOWN_="${_R_CHECK_COMPILATION_FLAGS_KNOWN_} -Wno-attributes -msse4.2" - # Note that NOT_CRAN=true means (among other things) that optional dependencies are built - export NOT_CRAN=true -fi -: ${TEST_R_WITH_ARROW:=TRUE} -export TEST_R_WITH_ARROW=$TEST_R_WITH_ARROW - -export _R_CHECK_CRAN_INCOMING_REMOTE_=FALSE -# --run-donttest was used in R < 4.0, this is used now -export _R_CHECK_DONTTEST_EXAMPLES_=$TEST_R_WITH_ARROW -# Not all Suggested packages are needed for checking, so in case they aren't installed don't fail -export _R_CHECK_FORCE_SUGGESTS_=FALSE -export _R_CHECK_LIMIT_CORES_=FALSE -export _R_CHECK_TESTS_NLINES_=0 - -# By default, aws-sdk tries to contact a non-existing local ip host -# to retrieve metadata. Disable this so that S3FileSystem tests run faster. -export AWS_EC2_METADATA_DISABLED=TRUE - -# Hack so that texlive2020 doesn't pollute the home dir -export TEXMFCONFIG=/tmp/texmf-config -export TEXMFVAR=/tmp/texmf-var - -if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then - # enable the devtoolset version to use it - source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable -fi - -# Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check) -BEFORE=$(ls -alh ~/) - -SCRIPT="as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true') - run_donttest <- identical(tolower(Sys.getenv('_R_CHECK_DONTTEST_EXAMPLES_', 'true')), 'true') - if (as_cran) { - rcmdcheck::rcmdcheck(args = c('--as-cran', if (run_donttest) '--run-donttest'), error_on = 'warning', check_dir = 'check', timeout = 3600) - } else { - if (nzchar(Sys.which('minio'))) { - message('Running minio for S3 tests (if build supports them)') - minio_dir <- tempfile() - dir.create(minio_dir) - pid <- sys::exec_background('minio', c('server', minio_dir)) - on.exit(tools::pskill(pid)) - } - rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', if (run_donttest) '--run-donttest'), error_on = 'warning', check_dir = 'check', timeout = 3600) - }" -echo "$SCRIPT" | ${R_BIN} --no-save - -AFTER=$(ls -alh ~/) -if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then - ls -alh ~/.cmake/packages - exit 1 -fi -popd diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh deleted file mode 100755 index 9988dfb64946..000000000000 --- a/ci/scripts/r_windows_build.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -: ${ARROW_HOME:=$(pwd)} -# Make sure it is absolute and exported -export ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" - -if [ "$RTOOLS_VERSION" = "35" ]; then - # Use rtools-backports if building with rtools35 - curl https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf > /etc/pacman.conf - # Update keys: https://www.msys2.org/news/#2020-06-29-new-packagers - msys2_repo_base_url=https://repo.msys2.org/msys - curl -OSsL "${msys2_repo_base_url}/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" - pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz - pacman --noconfirm -Scc - pacman --noconfirm -Syy - # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) - RWINLIB_LIB_DIR="lib-4.9.3" -else - # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN - # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf - # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" - # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz - # pacman --noconfirm -Scc - - pacman --noconfirm -Syy - RWINLIB_LIB_DIR="lib" -fi - -cp $ARROW_HOME/ci/scripts/PKGBUILD . -printenv -makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild - -VERSION=$(grep Version $ARROW_HOME/r/DESCRIPTION | cut -d " " -f 2) -DST_DIR="arrow-$VERSION" - -# Collect the build artifacts and make the shape of zip file that rwinlib expects -ls -mkdir -p build -mv mingw* build -cd build - -# This may vary by system/CI provider -MSYS_LIB_DIR="/c/rtools40" - -ls $MSYS_LIB_DIR/mingw64/lib/ -ls $MSYS_LIB_DIR/mingw32/lib/ - -# Untar the two builds we made -ls *.xz | xargs -n 1 tar -xJf -mkdir -p $DST_DIR -# Grab the headers from one, either one is fine -# (if we're building twice to combine old and new toolchains, this may already exist) -if [ ! -d $DST_DIR/include ]; then - mv mingw64/include $DST_DIR -fi - -# Make the rest of the directory structure -# lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) -mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/x64 -mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/i386 -# lib is for the new gcc 8 toolchain (Rtools 4.0) -mkdir -p $DST_DIR/lib/x64 -mkdir -p $DST_DIR/lib/i386 - -# Move the 64-bit versions of libarrow into the expected location -mv mingw64/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/x64 -# Same for the 32-bit versions -mv mingw32/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 - -# These may be from https://dl.bintray.com/rtools/backports/ -cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x64 -cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 - -# These are from https://dl.bintray.com/rtools/mingw{32,64}/ -cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/x64 -cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/i386 - -# Create build artifact -zip -r ${DST_DIR}.zip $DST_DIR - -# Copy that to a file name/path that does not vary by version number so we -# can easily find it in the R package tests on CI -cp ${DST_DIR}.zip ../libarrow.zip diff --git a/ci/scripts/release_test.sh b/ci/scripts/release_test.sh deleted file mode 100755 index ae2ab328884b..000000000000 --- a/ci/scripts/release_test.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eux - -arrow_dir=${1} - -pushd ${arrow_dir} - -dev/release/run-test.rb - -popd diff --git a/ci/scripts/ruby_test.sh b/ci/scripts/ruby_test.sh deleted file mode 100755 index 03d20e19831f..000000000000 --- a/ci/scripts/ruby_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/ruby -build_dir=${2}/ruby - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig -export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 - -rake -f ${source_dir}/Rakefile BUILD_DIR=${build_dir} USE_BUNDLER=yes diff --git a/ci/scripts/rust_build.sh b/ci/scripts/rust_build.sh deleted file mode 100755 index 726ecd80f1ab..000000000000 --- a/ci/scripts/rust_build.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -source_dir=${1}/rust - -# This file is used to build the rust binaries needed for the -# archery integration tests. Testing of the rust implementation -# in normal CI is handled by github workflows - -# Disable full debug symbol generation to speed up CI build / reduce memory required -export RUSTFLAGS="-C debuginfo=1" - -export ARROW_TEST_DATA=${arrow_dir}/testing/data -export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data - -# show activated toolchain -rustup show - -pushd ${source_dir} - -# build only the integration testing binaries -cargo build -p arrow-integration-testing - -# Remove incremental build artifacts to save space -rm -rf target/debug/deps/ target/debug/build/ - -popd diff --git a/ci/scripts/util_checkout.sh b/ci/scripts/util_checkout.sh deleted file mode 100755 index 25fe69aa13f0..000000000000 --- a/ci/scripts/util_checkout.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# this script is github actions specific to check out the submodules and tags - -# TODO(kszucs): remove it once the "submodules: recursive" feature is released -auth_header="$(git config --local --get http.https://github.com/.extraheader)" -git submodule sync --recursive -git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 - -# fetch all the tags -git fetch --depth=1 origin +refs/tags/*:refs/tags/* diff --git a/ci/scripts/util_cleanup.sh b/ci/scripts/util_cleanup.sh deleted file mode 100755 index 3a13a1a784af..000000000000 --- a/ci/scripts/util_cleanup.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This script is Github Actions-specific to free up disk space, -# to avoid disk full errors on some builds - -if [ $RUNNER_OS = "Linux" ]; then - df -h - - # remove swap - sudo swapoff -a - sudo rm -f /swapfile - - # clean apt cache - sudo apt clean - - # remove haskell, consumes 8.6 GB - sudo rm -rf /opt/ghc - - # 1 GB - sudo rm -rf /home/linuxbrew/.linuxbrew - - # 1+ GB - sudo rm -rf /opt/hostedtoolcache/CodeQL - - # 1+ GB - sudo rm -rf /usr/share/swift - - # 12 GB, but takes a lot of time to delete - #sudo rm -rf /usr/local/lib/android - - # remove cached docker images, around 13 GB - docker rmi $(docker image ls -aq) - - # NOTE: /usr/share/dotnet is 25 GB -fi - -df -h diff --git a/ci/scripts/util_download_apache.sh b/ci/scripts/util_download_apache.sh deleted file mode 100755 index 5d37f9c0333e..000000000000 --- a/ci/scripts/util_download_apache.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -x -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -tarball_path=$1 -target_dir=$2 - -APACHE_MIRRORS=( - "http://www.apache.org/dyn/closer.cgi?action=download&filename=" - "https://downloads.apache.org" - "https://apache.claz.org" - "https://apache.cs.utah.edu" - "https://apache.mirrors.lucidnetworks.net" - "https://apache.osuosl.org" - "https://ftp.wayne.edu/apache" - "https://mirror.olnevhost.net/pub/apache" - "https://mirrors.gigenet.com/apache" - "https://mirrors.koehn.com/apache" - "https://mirrors.ocf.berkeley.edu/apache" - "https://mirrors.sonic.net/apache" - "https://us.mirrors.quenda.co/apache" -) - -mkdir -p "${target_dir}" - -for mirror in ${APACHE_MIRRORS[*]} -do - curl -SL "${mirror}/${tarball_path}" | tar -xzf - -C "${target_dir}" - if [ $? == 0 ]; then - exit 0 - fi -done - -exit 1 diff --git a/ci/scripts/util_wait_for_it.sh b/ci/scripts/util_wait_for_it.sh deleted file mode 100755 index 51ce816eb8c2..000000000000 --- a/ci/scripts/util_wait_for_it.sh +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env bash - -# The MIT License (MIT) -# Copyright (c) 2016 Giles Hall -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Use this script to test if a given TCP host/port are available - -cmdname=$(basename $0) - -echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $TIMEOUT -gt 0 ]]; then - echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" - else - echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" - fi - start_ts=$(date +%s) - while : - do - if [[ $ISBUSY -eq 1 ]]; then - nc -z $HOST $PORT - result=$? - else - (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 - result=$? - fi - if [[ $result -eq 0 ]]; then - end_ts=$(date +%s) - echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" - break - fi - sleep 1 - done - return $result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $QUIET -eq 1 ]]; then - timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - else - timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - fi - PID=$! - trap "kill -INT -$PID" INT - wait $PID - RESULT=$? - if [[ $RESULT -ne 0 ]]; then - echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" - fi - return $RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - hostport=(${1//:/ }) - HOST=${hostport[0]} - PORT=${hostport[1]} - shift 1 - ;; - --child) - CHILD=1 - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -s | --strict) - STRICT=1 - shift 1 - ;; - -h) - HOST="$2" - if [[ $HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - HOST="${1#*=}" - shift 1 - ;; - -p) - PORT="$2" - if [[ $PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - PORT="${1#*=}" - shift 1 - ;; - -t) - TIMEOUT="$2" - if [[ $TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$HOST" == "" || "$PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -TIMEOUT=${TIMEOUT:-15} -STRICT=${STRICT:-0} -CHILD=${CHILD:-0} -QUIET=${QUIET:-0} - -# check to see if timeout is from busybox? -# check to see if timeout is from busybox? -TIMEOUT_PATH=$(realpath $(which timeout)) -if [[ $TIMEOUT_PATH =~ "busybox" ]]; then - ISBUSY=1 - BUSYTIMEFLAG="-t" -else - ISBUSY=0 - BUSYTIMEFLAG="" -fi - -if [[ $CHILD -gt 0 ]]; then - wait_for - RESULT=$? - exit $RESULT -else - if [[ $TIMEOUT -gt 0 ]]; then - wait_for_wrapper - RESULT=$? - else - wait_for - RESULT=$? - fi -fi - -if [[ $CLI != "" ]]; then - if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then - echoerr "$cmdname: strict mode, refusing to execute subprocess" - exit $RESULT - fi - exec "${CLI[@]}" -else - exit $RESULT -fi diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh index f63426601035..b1285cbc3f53 100755 --- a/dev/rust_lint.sh +++ b/dev/rust_lint.sh @@ -16,6 +16,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +# This script runs all the Rust lints locally the same way the +# DataFusion CI does + set -e if ! command -v cargo-tomlfmt &> /dev/null; then echo "Installing cargo-tomlfmt using cargo" @@ -24,4 +28,4 @@ fi ci/scripts/rust_fmt.sh ci/scripts/rust_clippy.sh -ci/scripts/rust_toml_fmt.sh \ No newline at end of file +ci/scripts/rust_toml_fmt.sh