GH-39958: [Python][CI] Remove upper pin on pytest

ci/conda_env_python.txt

@@ -23,7 +23,7 @@ cloudpickle
 fsspec
 hypothesis
 numpy>=1.16.6
-pytest<8
+pytest
 pytest-faulthandler
 s3fs>=2023.10.0
 setuptools

docker-compose.yml

@@ -1208,15 +1208,17 @@ services:
       LANG: "C.UTF-8"
       BUILD_DOCS_CPP: "ON"
       BUILD_DOCS_PYTHON: "ON"
-      # GH-31506/GH-33609: Remove --disable-warnings once
-      # https://github.com/lgpage/pytest-cython/issues/24 is resolved
-      # and a new version that includes the fix is released.
-      PYTEST_ARGS: "--doctest-modules --doctest-cython --disable-warnings"
+      PYTEST_ARGS: "--doctest-modules --doctest-cython"
     volumes: *conda-volumes
+    # pytest is installed with an upper pin of 8.0.0 because
+    # newer version breaks cython doctesting, see:
+    # https://github.com/lgpage/pytest-cython/issues/58
+    # Remove pip install pytest~=7 when upstream issue is resolved
     command:
       ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/python_build.sh /arrow /build &&
         pip install -e /arrow/dev/archery[numpydoc] &&
+        pip install pytest~=7.4 &&
         archery numpydoc --allow-rule GL10,PR01,PR03,PR04,PR05,PR10,RT03,YD01 &&
         /arrow/ci/scripts/python_test.sh /arrow"]
 

python/pyarrow/tests/parquet/conftest.py

@@ -81,6 +81,7 @@ def s3_example_fs(s3_server):
     host, port, access_key, secret_key = s3_server['connection']
     uri = (
         "s3://{}:{}@mybucket/data.parquet?scheme=http&endpoint_override={}:{}"
+        "&allow_bucket_creation=True"
         .format(access_key, secret_key, host, port)
     )
     fs, path = FileSystem.from_uri(uri)

python/requirements-test.txt

@@ -1,5 +1,5 @@
 cffi
 hypothesis
 pandas
-pytest<8
+pytest
 pytz

python/requirements-wheel-test.txt

@@ -1,7 +1,7 @@
 cffi
 cython
 hypothesis
-pytest<8
+pytest
 pytz
 tzdata; sys_platform == 'win32'