BUG: DataFrame.join reordering index levels when joining on subset of levels (#55370)

* BUG: DataFrame.join reordering index levels when joining on subset of levels

* update more tests

* move whatsnew to notable section

Author: lukemanley

doc/source/whatsnew/v2.2.0.rst

@@ -133,10 +133,36 @@ and ``sort=False``:
 
     result
 
-.. _whatsnew_220.notable_bug_fixes.notable_bug_fix2:
+.. _whatsnew_220.notable_bug_fixes.multiindex_join_different_levels:
 
-notable_bug_fix2
-^^^^^^^^^^^^^^^^
+:func:`merge` and :meth:`DataFrame.join` no longer reorder levels when levels differ
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In previous versions of pandas, :func:`merge` and :meth:`DataFrame.join` would reorder
+index levels when joining on two indexes with different levels (:issue:`34133`).
+
+.. ipython:: python
+
+    left = pd.DataFrame({"left": 1}, index=pd.MultiIndex.from_tuples([("x", 1), ("x", 2)], names=["A", "B"]))
+    right = pd.DataFrame({"right": 2}, index=pd.MultiIndex.from_tuples([(1, 1), (2, 2)], names=["B", "C"]))
+    result = left.join(right)
+
+*Old Behavior*
+
+.. code-block:: ipython
+
+    In [5]: result
+    Out[5]:
+           left  right
+    B A C
+    1 x 1     1      2
+    2 x 2     1      2
+
+*New Behavior*
+
+.. ipython:: python
+
+    result
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_220.api_breaking:
@@ -342,6 +368,7 @@ Reshaping
 ^^^^^^^^^
 - Bug in :func:`concat` ignoring ``sort`` parameter when passed :class:`DatetimeIndex` indexes (:issue:`54769`)
 - Bug in :func:`merge` returning columns in incorrect order when left and/or right is empty (:issue:`51929`)
+-
 
 Sparse
 ^^^^^^

pandas/core/indexes/base.py

@@ -4725,6 +4725,13 @@ def _join_multi(self, other: Index, how: JoinHow):
 
             multi_join_idx = multi_join_idx.remove_unused_levels()
 
+            # maintain the order of the index levels
+            if how == "right":
+                level_order = other_names_list + ldrop_names
+            else:
+                level_order = self_names_list + rdrop_names
+            multi_join_idx = multi_join_idx.reorder_levels(level_order)
+
             return multi_join_idx, lidx, ridx
 
         jl = next(iter(overlap))

pandas/tests/reshape/merge/test_join.py

@@ -902,7 +902,7 @@ def test_join_inner_multiindex_deterministic_order():
     result = left.join(right, how="inner")
     expected = DataFrame(
         {"e": [5], "f": [6]},
-        index=MultiIndex.from_tuples([(2, 1, 4, 3)], names=("b", "a", "d", "c")),
+        index=MultiIndex.from_tuples([(1, 2, 4, 3)], names=("a", "b", "d", "c")),
     )
     tm.assert_frame_equal(result, expected)
 
@@ -926,10 +926,16 @@ def test_join_multiindex_one_level(join_type):
     )
     right = DataFrame(data={"d": 4}, index=MultiIndex.from_tuples([(2,)], names=("b",)))
     result = left.join(right, how=join_type)
-    expected = DataFrame(
-        {"c": [3], "d": [4]},
-        index=MultiIndex.from_tuples([(2, 1)], names=["b", "a"]),
-    )
+    if join_type == "right":
+        expected = DataFrame(
+            {"c": [3], "d": [4]},
+            index=MultiIndex.from_tuples([(2, 1)], names=["b", "a"]),
+        )
+    else:
+        expected = DataFrame(
+            {"c": [3], "d": [4]},
+            index=MultiIndex.from_tuples([(1, 2)], names=["a", "b"]),
+        )
     tm.assert_frame_equal(result, expected)
 
 

pandas/tests/reshape/merge/test_multi.py

@@ -69,11 +69,6 @@ def on_cols_multi():
     return ["Origin", "Destination", "Period"]
 
 
-@pytest.fixture
-def idx_cols_multi():
-    return ["Origin", "Destination", "Period", "TripPurp", "LinkType"]
-
-
 class TestMergeMulti:
     def test_merge_on_multikey(self, left, right, join_type):
         on_cols = ["key1", "key2"]
@@ -815,9 +810,13 @@ def test_join_multi_levels2(self):
 
 
 class TestJoinMultiMulti:
-    def test_join_multi_multi(
-        self, left_multi, right_multi, join_type, on_cols_multi, idx_cols_multi
-    ):
+    def test_join_multi_multi(self, left_multi, right_multi, join_type, on_cols_multi):
+        left_names = left_multi.index.names
+        right_names = right_multi.index.names
+        if join_type == "right":
+            level_order = right_names + left_names.difference(right_names)
+        else:
+            level_order = left_names + right_names.difference(left_names)
         # Multi-index join tests
         expected = (
             merge(
@@ -826,27 +825,34 @@ def test_join_multi_multi(
                 how=join_type,
                 on=on_cols_multi,
             )
-            .set_index(idx_cols_multi)
+            .set_index(level_order)
             .sort_index()
         )
 
         result = left_multi.join(right_multi, how=join_type).sort_index()
         tm.assert_frame_equal(result, expected)
 
     def test_join_multi_empty_frames(
-        self, left_multi, right_multi, join_type, on_cols_multi, idx_cols_multi
+        self, left_multi, right_multi, join_type, on_cols_multi
     ):
         left_multi = left_multi.drop(columns=left_multi.columns)
         right_multi = right_multi.drop(columns=right_multi.columns)
 
+        left_names = left_multi.index.names
+        right_names = right_multi.index.names
+        if join_type == "right":
+            level_order = right_names + left_names.difference(right_names)
+        else:
+            level_order = left_names + right_names.difference(left_names)
+
         expected = (
             merge(
                 left_multi.reset_index(),
                 right_multi.reset_index(),
                 how=join_type,
                 on=on_cols_multi,
             )
-            .set_index(idx_cols_multi)
+            .set_index(level_order)
             .sort_index()
         )
 

pandas/tests/series/methods/test_align.py

@@ -240,10 +240,10 @@ def test_align_left_different_named_levels():
     result_left, result_right = left.align(right)
 
     expected_left = Series(
-        [2], index=pd.MultiIndex.from_tuples([(1, 3, 4, 2)], names=["a", "c", "d", "b"])
+        [2], index=pd.MultiIndex.from_tuples([(1, 4, 3, 2)], names=["a", "d", "c", "b"])
     )
     expected_right = Series(
-        [1], index=pd.MultiIndex.from_tuples([(1, 3, 4, 2)], names=["a", "c", "d", "b"])
+        [1], index=pd.MultiIndex.from_tuples([(1, 4, 3, 2)], names=["a", "d", "c", "b"])
     )
     tm.assert_series_equal(result_left, expected_left)
     tm.assert_series_equal(result_right, expected_right)

pandas/tests/series/test_arithmetic.py

@@ -941,8 +941,8 @@ def test_series_varied_multiindex_alignment():
     expected = Series(
         [1000, 2001, 3002, 4003],
         index=pd.MultiIndex.from_tuples(
-            [("x", 1, "a"), ("x", 2, "a"), ("y", 1, "a"), ("y", 2, "a")],
-            names=["xy", "num", "ab"],
+            [("a", "x", 1), ("a", "x", 2), ("a", "y", 1), ("a", "y", 2)],
+            names=["ab", "xy", "num"],
         ),
     )
     tm.assert_series_equal(result, expected)