BUG: read_csv usecols and names parameters inconsistent between c and python (#38445)

Author: phofl

doc/source/whatsnew/v1.3.0.rst

@@ -217,6 +217,7 @@ MultiIndex
 I/O
 ^^^
 
+- Bug in :func:`read_csv` not accepting ``usecols`` with different length than ``names`` for ``engine="python"`` (:issue:`16469`)
 - Bug in :func:`read_csv` raising ``TypeError`` when ``names`` and ``parse_dates`` is specified for ``engine="c"`` (:issue:`33699`)
 -
 

pandas/io/parsers.py

@@ -2487,9 +2487,8 @@ def read(self, rows=None):
             content = content[1:]
 
         alldata = self._rows_to_cols(content)
-        data = self._exclude_implicit_index(alldata)
+        data, columns = self._exclude_implicit_index(alldata)
 
-        columns = self._maybe_dedup_names(self.columns)
         columns, data = self._do_date_conversions(columns, data)
 
         data = self._convert_data(data)
@@ -2500,19 +2499,14 @@ def read(self, rows=None):
     def _exclude_implicit_index(self, alldata):
         names = self._maybe_dedup_names(self.orig_names)
 
+        offset = 0
         if self._implicit_index:
-            excl_indices = self.index_col
+            offset = len(self.index_col)
 
-            data = {}
-            offset = 0
-            for i, col in enumerate(names):
-                while i + offset in excl_indices:
-                    offset += 1
-                data[col] = alldata[i + offset]
-        else:
-            data = {k: v for k, v in zip(names, alldata)}
+        if self._col_indices is not None and len(names) != len(self._col_indices):
+            names = [names[i] for i in sorted(self._col_indices)]
 
-        return data
+        return {name: alldata[i + offset] for i, name in enumerate(names)}, names
 
     # legacy
     def get_chunk(self, size=None):
@@ -2694,9 +2688,7 @@ def _infer_columns(self):
                 self._clear_buffer()
 
             if names is not None:
-                if (self.usecols is not None and len(names) != len(self.usecols)) or (
-                    self.usecols is None and len(names) != len(columns[0])
-                ):
+                if len(names) > len(columns[0]):
                     raise ValueError(
                         "Number of passed names did not match "
                         "number of header fields in the file"

pandas/tests/io/parser/test_usecols.py

@@ -559,12 +559,7 @@ def test_raises_on_usecols_names_mismatch(all_parsers, usecols, kwargs, expected
 
 
 @pytest.mark.parametrize("usecols", [["A", "C"], [0, 2]])
-def test_usecols_subset_names_mismatch_orig_columns(all_parsers, usecols, request):
-    if all_parsers.engine != "c":
-        reason = "see gh-16469: works on the C engine but not the Python engine"
-        # Number of passed names did not match number of header fields in the file
-        request.node.add_marker(pytest.mark.xfail(reason=reason, raises=ValueError))
-
+def test_usecols_subset_names_mismatch_orig_columns(all_parsers, usecols):
     data = "a,b,c,d\n1,2,3,4\n5,6,7,8"
     names = ["A", "B", "C", "D"]
     parser = all_parsers