diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 75c9cc61..e5388437 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -47,8 +47,9 @@ jobs:
     - name: Tests
       env:
         SUPER_SECRET: ${{ secrets.HF_HUB_TOKEN }}
+      # skip slow inference tests here, see inference-test.yml
       run: |
-        python -m pytest -s -v --cov-report=xml skops/
+        python -m pytest -s -v --cov-report=xml -m "not inference" skops/
 
     - name: Mypy
       run: mypy --config-file pyproject.toml skops
diff --git a/.github/workflows/inference-test-full.yml b/.github/workflows/inference-test-full.yml
new file mode 100644
index 00000000..7d48b56b
--- /dev/null
+++ b/.github/workflows/inference-test-full.yml
@@ -0,0 +1,48 @@
+# Tests hitting the inference API are slow and prone to hanging, resulting in
+# timeouts. Therefore, we only run inference tests on the full test matrix if
+# the head commit message contains the text "[CI inference]". See #118
+
+name: inference-tests-full
+
+on:
+  - push
+  - pull_request
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  inference-pytest-full:
+
+    runs-on: ${{ matrix.os }}
+    if: ${{ (github.repository == 'skops-dev/skops') && contains(github.event.head_commit.message, '[CI inference]') }}
+    strategy:
+      fail-fast: false  # need to see which ones fail
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python: ["3.7", "3.8", "3.9", "3.10"]
+
+    # Timeout: https://stackoverflow.com/a/59076067/4521646
+    timeout-minutes: 15
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Install dependencies
+      run: |
+        pip install .[docs,tests]
+        python --version
+        pip --version
+        pip list
+      shell: bash
+
+    - name: Full inference tests
+      env:
+        SUPER_SECRET: ${{ secrets.HF_HUB_TOKEN }}
+      run: |
+        python -m pytest -s -v -m "inference" skops/
diff --git a/.github/workflows/inference-test-partial.yml b/.github/workflows/inference-test-partial.yml
new file mode 100644
index 00000000..daadcbfd
--- /dev/null
+++ b/.github/workflows/inference-test-partial.yml
@@ -0,0 +1,48 @@
+# Tests hitting the inference API are slow and prone to hanging, resulting in
+# timeouts. Therefore, we only run a subset of settings with inference tests by
+# default. See #118
+
+name: inference-tests-partial
+
+on:
+  - push
+  - pull_request
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  inference-pytest-partial:
+
+    runs-on: ${{ matrix.os }}
+    if: ${{ (github.repository == 'skops-dev/skops') && (!contains(github.event.head_commit.message, '[CI inference]')) }}
+    strategy:
+      fail-fast: false  # need to see which ones fail
+      matrix:
+        os: [ubuntu-latest]
+        python: ["3.10"]
+
+    # Timeout: https://stackoverflow.com/a/59076067/4521646
+    timeout-minutes: 15
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Install dependencies
+      run: |
+        pip install .[docs,tests]
+        python --version
+        pip --version
+        pip list
+      shell: bash
+
+    - name: Partial inference tests
+      env:
+        SUPER_SECRET: ${{ secrets.HF_HUB_TOKEN }}
+      run: |
+        python -m pytest -s -v -m "inference" skops/
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 03bc4a26..306accc8 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -73,7 +73,12 @@ internet with:
 
 .. code:: bash
 
-   pytest -m "not network"
+   pytest -m "not network" skops
+
+Similarly, there is a flag, ``-m inference`` for tests that hit the Hugging Face
+Inference API, which can be quite slow or even hang. Skip these tests as long as
+you don't make any changes to this functionality. If you already skip network
+tests, the inference tests will also be skipped.
 
 
 Releases
diff --git a/pyproject.toml b/pyproject.toml
index 64a3866b..4a880028 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,6 +13,7 @@ filterwarnings = [
 ]
 markers = [
     "network: marks tests as requiring internet (deselect with '-m \"not network\"')",
+    "inference: marks tests that call inference API (deselect with '-m \"not inference\"')",
 ]
 addopts = "--cov=skops --cov-report=term-missing --doctest-modules"
 
diff --git a/skops/hub_utils/tests/test_hf_hub.py b/skops/hub_utils/tests/test_hf_hub.py
index 362c2664..82df2268 100644
--- a/skops/hub_utils/tests/test_hf_hub.py
+++ b/skops/hub_utils/tests/test_hf_hub.py
@@ -391,6 +391,7 @@ def repo_path_for_inference():
         yield Path(repo_path)
 
 
+@pytest.mark.inference
 @pytest.mark.network
 @flaky(max_runs=3)
 @pytest.mark.parametrize(
@@ -408,7 +409,8 @@ def test_inference(
     repo_path_for_inference,
     destination_path,
 ):
-    # test inference backend for classifier and regressor models.
+    # Test inference backend for classifier and regressor models. This test can
+    # take a lot of time.
     client = HfApi()
 
     repo_path = repo_path_for_inference