From 2d0a1f2296661d313f8c8fbb2f10ba1ac8a58882 Mon Sep 17 00:00:00 2001
From: cvanelteren <caspervanelteren@gmail.com>
Date: Tue, 21 Oct 2025 15:55:39 +0200
Subject: [PATCH 1/8] get border axes

---
 ultraplot/figure.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/ultraplot/figure.py b/ultraplot/figure.py
index d44f31e61..79d656a88 100644
--- a/ultraplot/figure.py
+++ b/ultraplot/figure.py
@@ -898,11 +898,7 @@ def _get_align_axes(self, side):
         if not axs:
             return []
         ranges = np.array([ax._range_subplotspec(x) for ax in axs])
-        edge = ranges[:, 0].min() if side in ("left", "top") else ranges[:, 1].max()
-        idx = 0 if side in ("left", "top") else 1
-        axs = [ax for ax in axs if ax._range_subplotspec(x)[idx] == edge]
-        axs = [ax for ax in sorted(axs, key=lambda ax: ax._range_subplotspec(y)[0])]
-        axs = [ax for ax in axs if ax.get_visible()]
+        axs = self._get_border_axes()[side]
         return axs
 
     def _get_border_axes(

From d219544544627eaa941f13ac49532710077befb9 Mon Sep 17 00:00:00 2001
From: cvanelteren <caspervanelteren@gmail.com>
Date: Tue, 21 Oct 2025 15:57:58 +0200
Subject: [PATCH 2/8] rm dead code

---
 ultraplot/figure.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ultraplot/figure.py b/ultraplot/figure.py
index 79d656a88..7df45e816 100644
--- a/ultraplot/figure.py
+++ b/ultraplot/figure.py
@@ -897,7 +897,6 @@ def _get_align_axes(self, side):
         axs = self._subplot_dict.values()
         if not axs:
             return []
-        ranges = np.array([ax._range_subplotspec(x) for ax in axs])
         axs = self._get_border_axes()[side]
         return axs
 

From e83a449627adccf88cb9e38f6c31af98ec06b2f0 Mon Sep 17 00:00:00 2001
From: cvanelteren <caspervanelteren@gmail.com>
Date: Tue, 21 Oct 2025 16:45:28 +0200
Subject: [PATCH 3/8] allow non-rectangular grids but error when not possible

---
 ultraplot/figure.py              |  1 -
 ultraplot/tests/test_subplots.py | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/ultraplot/figure.py b/ultraplot/figure.py
index 7df45e816..42b242825 100644
--- a/ultraplot/figure.py
+++ b/ultraplot/figure.py
@@ -893,7 +893,6 @@ def _get_align_axes(self, side):
         """
         Return the main axes along the edge of the figure.
         """
-        x, y = "xy" if side in ("left", "right") else "yx"
         axs = self._subplot_dict.values()
         if not axs:
             return []
diff --git a/ultraplot/tests/test_subplots.py b/ultraplot/tests/test_subplots.py
index e215a90ee..10110c1c5 100644
--- a/ultraplot/tests/test_subplots.py
+++ b/ultraplot/tests/test_subplots.py
@@ -327,3 +327,37 @@ def test_uneven_span_subplots(rng):
     axs[-1, -1].format(fc="gray4", grid=False)
     axs[0].plot((rng.random((50, 10)) - 0.5).cumsum(axis=0), cycle="Grays_r", lw=2)
     return fig
+
+
+@pytest.mark.mpl_image_compare
+def test_non_rectangular_outside_labels_left():
+    """
+    Check that non-rectangular layouts work with outside labels.
+    """
+    layout = [
+        [1, 1, 2, 2],
+        [0, 3, 3, 0],
+        [4, 4, 5, 5],
+    ]
+    fig, ax = uplt.subplots(layout)
+    ax.format(
+        leftlabels=[1, 2, 3],
+    )
+    return fig
+
+
+def test_non_rectangular_outside_labels_top():
+    """
+    Check that non-rectangular layouts work with outside labels.
+    """
+    layout = [
+        [1, 1, 2, 2],
+        [0, 3, 3, 0],
+        [4, 4, 5, 5],
+    ]
+    fig, ax = uplt.subplots(layout)
+    with pytest.raises(ValueError):
+        ax.format(
+            toplabels=[1, 2, 3],
+        )
+    return fig

From 8373401da4993b1c154882ca1013b41eaf904a64 Mon Sep 17 00:00:00 2001
From: cvanelteren <caspervanelteren@gmail.com>
Date: Wed, 22 Oct 2025 07:10:10 +0200
Subject: [PATCH 4/8] proper fix

---
 ultraplot/figure.py              | 33 ++++++++++++++++++++++++++++++--
 ultraplot/tests/test_subplots.py |  9 +++++----
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/ultraplot/figure.py b/ultraplot/figure.py
index 42b242825..2e39a1134 100644
--- a/ultraplot/figure.py
+++ b/ultraplot/figure.py
@@ -892,11 +892,40 @@ def _parse_proj(
     def _get_align_axes(self, side):
         """
         Return the main axes along the edge of the figure.
+
+        For 'left'/'right': select one extreme axis per row (leftmost/rightmost).
+        For 'top'/'bottom': select one extreme axis per column (topmost/bottommost).
         """
-        axs = self._subplot_dict.values()
+        axs = tuple(self._subplot_dict.values())
         if not axs:
             return []
-        axs = self._get_border_axes()[side]
+        if side not in ("left", "right", "top", "bottom"):
+            raise ValueError(f"Invalid side {side!r}.")
+        from .utils import _get_subplot_layout
+
+        grid = _get_subplot_layout(self._gridspec, list(self._iter_axes()))[0]
+        # From the @side we find the first non-zero
+        # entry in each row or column and collect the axes
+        if side == "left":
+            options = grid
+        elif side == "right":
+            options = grid[:, ::-1]
+        elif side == "top":
+            options = grid.T
+        else:  # bottom
+            options = grid.T[:, ::-1]
+        uids = set()
+        for option in options:
+            idx = np.where(option > 0)[0]
+            if idx.size > 0:
+                first = idx.min()
+                number = option[first].astype(int)
+                uids.add(number)
+        axs = []
+        # Collect correct axes
+        for axi in self._iter_axes():
+            if axi.number in uids and axi not in axs:
+                axs.append(axi)
         return axs
 
     def _get_border_axes(
diff --git a/ultraplot/tests/test_subplots.py b/ultraplot/tests/test_subplots.py
index 10110c1c5..ca2a92093 100644
--- a/ultraplot/tests/test_subplots.py
+++ b/ultraplot/tests/test_subplots.py
@@ -343,6 +343,7 @@ def test_non_rectangular_outside_labels_left():
     ax.format(
         leftlabels=[1, 2, 3],
     )
+    uplt.show(block=1)
     return fig
 
 
@@ -356,8 +357,8 @@ def test_non_rectangular_outside_labels_top():
         [4, 4, 5, 5],
     ]
     fig, ax = uplt.subplots(layout)
-    with pytest.raises(ValueError):
-        ax.format(
-            toplabels=[1, 2, 3],
-        )
+    ax.format(
+        toplabels=[1, 2, 3],
+    )
+    uplt.show(block=1)
     return fig

From 1be7abb69b1e3f7b5d35e5746b9bc8eefb40f7c5 Mon Sep 17 00:00:00 2001
From: cvanelteren <caspervanelteren@gmail.com>
Date: Wed, 22 Oct 2025 07:10:58 +0200
Subject: [PATCH 5/8] update unittest

---
 ultraplot/tests/test_subplots.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/ultraplot/tests/test_subplots.py b/ultraplot/tests/test_subplots.py
index ca2a92093..50bddc9b2 100644
--- a/ultraplot/tests/test_subplots.py
+++ b/ultraplot/tests/test_subplots.py
@@ -356,9 +356,12 @@ def test_non_rectangular_outside_labels_top():
         [0, 3, 3, 0],
         [4, 4, 5, 5],
     ]
-    fig, ax = uplt.subplots(layout)
-    ax.format(
-        toplabels=[1, 2, 3],
+
+    fig, ax = uplt.subplots(
+        gridspec,
     )
-    uplt.show(block=1)
+    ax.format(rightlabels=[2, 3, 5])
+    ax.format(bottomlabels=[4, 5])
+    ax.format(leftlabels=[1, 3, 4])
+    ax.format(toplabels=[1, 2])
     return fig

From 5bfeeb48c8a3c0423d20d539a7bd229f4a12908c Mon Sep 17 00:00:00 2001
From: cvanelteren <caspervanelteren@gmail.com>
Date: Wed, 22 Oct 2025 07:25:18 +0200
Subject: [PATCH 6/8] remove red. test

---
 ultraplot/tests/test_subplots.py | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/ultraplot/tests/test_subplots.py b/ultraplot/tests/test_subplots.py
index 50bddc9b2..8cb218167 100644
--- a/ultraplot/tests/test_subplots.py
+++ b/ultraplot/tests/test_subplots.py
@@ -330,23 +330,6 @@ def test_uneven_span_subplots(rng):
 
 
 @pytest.mark.mpl_image_compare
-def test_non_rectangular_outside_labels_left():
-    """
-    Check that non-rectangular layouts work with outside labels.
-    """
-    layout = [
-        [1, 1, 2, 2],
-        [0, 3, 3, 0],
-        [4, 4, 5, 5],
-    ]
-    fig, ax = uplt.subplots(layout)
-    ax.format(
-        leftlabels=[1, 2, 3],
-    )
-    uplt.show(block=1)
-    return fig
-
-
 def test_non_rectangular_outside_labels_top():
     """
     Check that non-rectangular layouts work with outside labels.

From 9c8019189ba1f973e868605c482b44dcb70d9570 Mon Sep 17 00:00:00 2001
From: cvanelteren <caspervanelteren@gmail.com>
Date: Wed, 22 Oct 2025 07:34:34 +0200
Subject: [PATCH 7/8] minor bump

---
 ultraplot/figure.py              | 4 +++-
 ultraplot/tests/test_subplots.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ultraplot/figure.py b/ultraplot/figure.py
index 2e39a1134..981fa2424 100644
--- a/ultraplot/figure.py
+++ b/ultraplot/figure.py
@@ -903,7 +903,9 @@ def _get_align_axes(self, side):
             raise ValueError(f"Invalid side {side!r}.")
         from .utils import _get_subplot_layout
 
-        grid = _get_subplot_layout(self._gridspec, list(self._iter_axes()))[0]
+        grid = _get_subplot_layout(
+            self._gridspec, list(self._iter_axes(panels=False, hidden=False))
+        )[0]
         # From the @side we find the first non-zero
         # entry in each row or column and collect the axes
         if side == "left":
diff --git a/ultraplot/tests/test_subplots.py b/ultraplot/tests/test_subplots.py
index 8cb218167..e54109033 100644
--- a/ultraplot/tests/test_subplots.py
+++ b/ultraplot/tests/test_subplots.py
@@ -341,7 +341,7 @@ def test_non_rectangular_outside_labels_top():
     ]
 
     fig, ax = uplt.subplots(
-        gridspec,
+        layout,
     )
     ax.format(rightlabels=[2, 3, 5])
     ax.format(bottomlabels=[4, 5])

From a1b720163828074b4ee0d617808f3fcd583913c9 Mon Sep 17 00:00:00 2001
From: cvanelteren <caspervanelteren@gmail.com>
Date: Wed, 22 Oct 2025 07:40:19 +0200
Subject: [PATCH 8/8] add a panel test

---
 ultraplot/tests/test_subplots.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ultraplot/tests/test_subplots.py b/ultraplot/tests/test_subplots.py
index e54109033..9a6d6d10d 100644
--- a/ultraplot/tests/test_subplots.py
+++ b/ultraplot/tests/test_subplots.py
@@ -348,3 +348,18 @@ def test_non_rectangular_outside_labels_top():
     ax.format(leftlabels=[1, 3, 4])
     ax.format(toplabels=[1, 2])
     return fig
+
+
+@pytest.mark.mpl_image_compare
+def test_outside_labels_with_panels():
+    fig, ax = uplt.subplots(
+        ncols=2,
+        nrows=2,
+    )
+    # Create extreme case where we add a lot of panels
+    # This should push the left labels further left
+    for idx in range(5):
+        ax[0].panel("left")
+    ax.format(leftlabels=["A", "B"])
+    uplt.show(block=1)
+    return fig