From a416b1c7d711800b4bc4ddafba28427920ab85e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 17 Sep 2024 16:56:40 +0200 Subject: [PATCH 01/10] GH-40570: [CI] Default environment to Ubuntu 22.04 instead of 20.04 --- .env | 2 +- .github/workflows/python.yml | 2 +- dev/tasks/tasks.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env b/.env index f41a1424907..215aa49109b 100644 --- a/.env +++ b/.env @@ -50,7 +50,7 @@ ALMALINUX=8 ALPINE_LINUX=3.16 DEBIAN=12 FEDORA=39 -UBUNTU=20.04 +UBUNTU=22.04 # Default versions for various dependencies CLANG_TOOLS=14 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4916287556b..1462e0faf32 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -93,7 +93,7 @@ jobs: python: "3.11" env: PYTHON: ${{ matrix.python || 3.9 }} - UBUNTU: ${{ matrix.ubuntu || 20.04 }} + UBUNTU: ${{ matrix.ubuntu || 22.04 }} PANDAS: ${{ matrix.pandas || 'latest' }} NUMPY: ${{ matrix.numpy || 'latest' }} steps: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 10b5426b750..82c071eb241 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1118,7 +1118,7 @@ tasks: template: docker-tests/github.linux.yml params: env: - UBUNTU: 20.04 + UBUNTU: 22.04 flags: >- -e ARROW_AZURE=OFF -e ARROW_GANDIVA=OFF From f52056f1a086ecf44913060275cc46990a861c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Sep 2024 11:49:30 +0200 Subject: [PATCH 02/10] Add Ubuntu env for cuda Ubuntu 20.04 --- dev/tasks/tasks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 82c071eb241..c529569bfff 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1509,6 +1509,8 @@ tasks: ci: github template: docker-tests/github.cuda.yml params: + env: + UBUNTU: 20.04 image: ubuntu-cuda-cpp test-cuda-cpp-ubuntu-22.04-cuda-11.7.1: From 753cfdea1d0b2d7a435bd539f5606a0292c4a9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Sep 2024 11:59:08 +0200 Subject: [PATCH 03/10] Update cpp-minimal-with-formats to use Ubuntu 22.04 --- dev/tasks/tasks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c529569bfff..b7ff10fd388 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1188,12 +1188,12 @@ tasks: UBUNTU: 24.04 image: ubuntu-cpp-thread-sanitizer - test-ubuntu-20.04-cpp-minimal-with-formats: + test-ubuntu-24.04-cpp-minimal-with-formats: ci: github template: docker-tests/github.linux.yml params: env: - UBUNTU: 20.04 + UBUNTU: 24.04 flags: "-e ARROW_CSV=ON -e ARROW_PARQUET=ON" image: ubuntu-cpp-minimal From bf0ea8f9ac067d9c60faff62735593a706250141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 19 Sep 2024 12:36:37 +0200 Subject: [PATCH 04/10] Revert back skyhook test to use ubuntu 20.04 --- dev/tasks/tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index b7ff10fd388..77e188f08ec 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1118,7 +1118,7 @@ tasks: template: docker-tests/github.linux.yml params: env: - UBUNTU: 22.04 + UBUNTU: 20.04 flags: >- -e ARROW_AZURE=OFF -e ARROW_GANDIVA=OFF From ac33ea91a3a369234c8895a80d6c1648a711dd08 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 24 Sep 2024 09:37:25 +0900 Subject: [PATCH 05/10] Enable Skyhook integration --- ci/docker/ubuntu-22.04-cpp.dockerfile | 3 +++ ci/docker/ubuntu-24.04-cpp.dockerfile | 3 +++ dev/tasks/tasks.yml | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index f12e7456add..4447dea6f9e 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -91,6 +91,7 @@ RUN apt-get update -y -q && \ libprotobuf-dev \ libprotoc-dev \ libpsl-dev \ + libradospp-dev \ libre2-dev \ librtmp-dev \ libsnappy-dev \ @@ -112,7 +113,9 @@ RUN apt-get update -y -q && \ protobuf-compiler-grpc \ python3-dev \ python3-pip \ + python3-rados \ python3-venv \ + rados-objclass-dev \ rapidjson-dev \ rsync \ tzdata \ diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index 7d0772c33a2..6957bdda4f7 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -91,6 +91,7 @@ RUN apt-get update -y -q && \ libprotobuf-dev \ libprotoc-dev \ libpsl-dev \ + libradospp-dev \ libre2-dev \ librtmp-dev \ libsnappy-dev \ @@ -112,7 +113,9 @@ RUN apt-get update -y -q && \ protobuf-compiler-grpc \ python3-dev \ python3-pip \ + python3-rados \ python3-venv \ + rados-objclass-dev \ rapidjson-dev \ rsync \ tzdata \ diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 77e188f08ec..97adb9864f4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1118,7 +1118,7 @@ tasks: template: docker-tests/github.linux.yml params: env: - UBUNTU: 20.04 + UBUNTU: 24.04 flags: >- -e ARROW_AZURE=OFF -e ARROW_GANDIVA=OFF From 7c2c57405367d1bb1ee5c5bad0e5d22293059fcd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 24 Sep 2024 09:37:48 +0900 Subject: [PATCH 06/10] Use 22.04 --- dev/tasks/tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 97adb9864f4..b7ff10fd388 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1118,7 +1118,7 @@ tasks: template: docker-tests/github.linux.yml params: env: - UBUNTU: 24.04 + UBUNTU: 22.04 flags: >- -e ARROW_AZURE=OFF -e ARROW_GANDIVA=OFF From b27223513c32f5a7bd6dec61158e3417a556ad68 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 24 Sep 2024 10:08:29 +0900 Subject: [PATCH 07/10] Add uuid-runtime for uuidgen --- ci/docker/ubuntu-22.04-cpp.dockerfile | 1 + ci/docker/ubuntu-24.04-cpp.dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 4447dea6f9e..96b10c8dda4 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -119,6 +119,7 @@ RUN apt-get update -y -q && \ rapidjson-dev \ rsync \ tzdata \ + uuid-runtime \ wget \ xz-utils && \ apt-get clean && \ diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index 6957bdda4f7..0f2a400f7cb 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -120,6 +120,7 @@ RUN apt-get update -y -q && \ rsync \ tzdata \ tzdata-legacy \ + uuid-runtime \ wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists* From 6d7ddd1a3e5e8eb5802e084d4b0fae5c947419bd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 24 Sep 2024 10:21:52 +0900 Subject: [PATCH 08/10] Add ceph --- ci/docker/ubuntu-22.04-cpp.dockerfile | 1 + ci/docker/ubuntu-24.04-cpp.dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 96b10c8dda4..f22dce54441 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -68,6 +68,7 @@ RUN apt-get update -y -q && \ bzip2 \ ca-certificates \ ccache \ + ceph \ cmake \ curl \ gdb \ diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index 0f2a400f7cb..ed2a8cec49c 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -68,6 +68,7 @@ RUN apt-get update -y -q && \ autoconf \ ca-certificates \ ccache \ + ceph \ cmake \ curl \ gdb \ From 456892744740999a74f27567c51a59f9288933c3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 24 Sep 2024 10:40:52 +0900 Subject: [PATCH 09/10] Add ceph-mds --- ci/docker/ubuntu-22.04-cpp.dockerfile | 1 + ci/docker/ubuntu-24.04-cpp.dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index f22dce54441..8779d347b6a 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -69,6 +69,7 @@ RUN apt-get update -y -q && \ ca-certificates \ ccache \ ceph \ + ceph-mds \ cmake \ curl \ gdb \ diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index ed2a8cec49c..6999dcee85c 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -69,6 +69,7 @@ RUN apt-get update -y -q && \ ca-certificates \ ccache \ ceph \ + ceph-mds \ cmake \ curl \ gdb \ From 812da521192813012e7281de3f48fa47ae672bf9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 24 Sep 2024 11:25:20 +0900 Subject: [PATCH 10/10] Add ceph-fuse --- ci/docker/ubuntu-22.04-cpp.dockerfile | 1 + ci/docker/ubuntu-24.04-cpp.dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 8779d347b6a..bf5a282e7b8 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -69,6 +69,7 @@ RUN apt-get update -y -q && \ ca-certificates \ ccache \ ceph \ + ceph-fuse \ ceph-mds \ cmake \ curl \ diff --git a/ci/docker/ubuntu-24.04-cpp.dockerfile b/ci/docker/ubuntu-24.04-cpp.dockerfile index 6999dcee85c..f662edc5365 100644 --- a/ci/docker/ubuntu-24.04-cpp.dockerfile +++ b/ci/docker/ubuntu-24.04-cpp.dockerfile @@ -69,6 +69,7 @@ RUN apt-get update -y -q && \ ca-certificates \ ccache \ ceph \ + ceph-fuse \ ceph-mds \ cmake \ curl \