more flexibile io.parquet.engine

Author: jreback

doc/source/io.rst

@@ -4576,7 +4576,10 @@ Several caveats.
 - Non supported types include ``Period`` and actual python object types. These will raise a helpful error message
   on an attempt at serialization.
 
-You can specifiy an ``engine`` to direct the serialization, defaulting to ``pyarrow`` and controlled by the ``pd.options.io.parquet``.
+You can specifiy an ``engine`` to direct the serialization. This can be one of ``pyarrow``, or ``fastparquet``, or ``auto``.
+If the engine is NOT specified, then the ``pd.options.io.parquet.engine`` option is checked; if this is also ``auto``, then
+then ``pyarrow`` is tried, and falling back to ``fastparquet``.
+
 See the documentation for `pyarrow <http://arrow.apache.org/docs/python/`__ and `fastparquet <https://fastparquet.readthedocs.io/en/latest/>`__
 
 .. note::

doc/source/options.rst

@@ -414,8 +414,9 @@ io.hdf.default_format               None         default format writing format,
                                                  'table'
 io.hdf.dropna_table                 True         drop ALL nan rows when appending
                                                  to a table
-io.parquet.engine                   pyarrow      The engine to use as a default for
-                                                 parquet reading and writing.
+io.parquet.engine                   None         The engine to use as a default for
+                                                 parquet reading and writing. If None
+                                                 then try 'pyarrow' and 'fastparquet'
 mode.chained_assignment             warn         Raise an exception, warn, or no
                                                  action if trying to use chained
                                                  assignment, The default is warn

pandas/core/config_init.py

@@ -467,9 +467,10 @@ def _register_xlsx(engine, other):
 parquet_engine_doc = """
 : string
     The default parquet reader/writer engine. Available options:
-    'pyarrow', 'fastparquet', the default is 'pyarrow'
+    'auto', 'pyarrow', 'fastparquet', the default is 'auto'
 """
 
 with cf.config_prefix('io.parquet'):
-    cf.register_option('engine', 'pyarrow', parquet_engine_doc,
-                       validator=is_one_of_factory(['pyarrow', 'fastparquet']))
+    cf.register_option(
+        'engine', 'auto', parquet_engine_doc,
+        validator=is_one_of_factory(['auto', 'pyarrow', 'fastparquet']))

pandas/core/frame.py

@@ -1598,7 +1598,7 @@ def to_feather(self, fname):
         from pandas.io.feather_format import to_feather
         to_feather(self, fname)
 
-    def to_parquet(self, fname, engine=None, compression='snappy',
+    def to_parquet(self, fname, engine='auto', compression='snappy',
                    **kwargs):
         """
         Write a DataFrame to the binary parquet format.
@@ -1609,10 +1609,10 @@ def to_parquet(self, fname, engine=None, compression='snappy',
         ----------
         fname : str
             string file path
-        engine : str, optional
-            The parquet engine, one of {'pyarrow', 'fastparquet'}
-            If None, will use the option: `io.parquet.engine`, which
-            defaults to 'pyarrow'
+        engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
+            Parquet reader library to use. If 'auto', then the option
+            'io.parquet.engine' is used. If 'auto', then the first
+            library to be installed is used.
         compression : str, optional, default 'snappy'
             compression method, includes {'gzip', 'snappy', 'brotli'}
         kwargs

pandas/io/parquet.py

@@ -10,9 +10,21 @@
 def get_engine(engine):
     """ return our implementation """
 
-    if engine is None:
+    if engine is 'auto':
         engine = get_option('io.parquet.engine')
 
+    if engine is 'auto':
+        # try engines in this order
+        try:
+            return PyArrowImpl()
+        except ImportError:
+            pass
+
+        try:
+            return FastParquetImpl()
+        except ImportError:
+            pass
+
     if engine not in ['pyarrow', 'fastparquet']:
         raise ValueError("engine must be one of 'pyarrow', 'fastparquet'")
 
@@ -98,7 +110,7 @@ def read(self, path):
         return self.api.ParquetFile(path).to_pandas()
 
 
-def to_parquet(df, path, engine=None, compression='snappy', **kwargs):
+def to_parquet(df, path, engine='auto', compression='snappy', **kwargs):
     """
     Write a DataFrame to the parquet format.
 
@@ -107,10 +119,10 @@ def to_parquet(df, path, engine=None, compression='snappy', **kwargs):
     df : DataFrame
     path : string
         File path
-    engine : str, optional
-        The parquet engine, one of {'pyarrow', 'fastparquet'}
-        If None, will use the option: `io.parquet.engine`, which
-        defaults to 'pyarrow'
+    engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
+        Parquet reader library to use. If 'auto', then the option
+        'io.parquet.engine' is used. If 'auto', then the first
+        library to be installed is used.
     compression : str, optional, default 'snappy'
         compression method, includes {'gzip', 'snappy', 'brotli'}
     kwargs
@@ -156,7 +168,7 @@ def to_parquet(df, path, engine=None, compression='snappy', **kwargs):
     return impl.write(df, path, compression=compression)
 
 
-def read_parquet(path, engine=None, **kwargs):
+def read_parquet(path, engine='auto', **kwargs):
     """
     Load a parquet object from the file path, returning a DataFrame.
 
@@ -166,9 +178,10 @@ def read_parquet(path, engine=None, **kwargs):
     ----------
     path : string
         File path
-    engine : str, optional
-        The parquet engine, one of {'pyarrow', 'fastparquet'}
-        if None, will use the option: `io.parquet.engine`
+    engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
+        Parquet reader library to use. If 'auto', then the option
+        'io.parquet.engine' is used. If 'auto', then the first
+        library to be installed is used.
     kwargs are passed to the engine
 
     Returns

pandas/tests/io/test_parquet.py

@@ -7,7 +7,8 @@
 import numpy as np
 import pandas as pd
 from pandas.compat import PY3, is_platform_windows
-from pandas.io.parquet import to_parquet, read_parquet
+from pandas.io.parquet import (to_parquet, read_parquet, get_engine,
+                               PyArrowImpl, FastParquetImpl)
 from pandas.util import testing as tm
 
 try:
@@ -100,6 +101,38 @@ def test_options_fp(df_compat, fp):
             tm.assert_frame_equal(result, df)
 
 
+def test_options_auto(df_compat, fp, pa):
+
+    df = df_compat
+    with tm.ensure_clean() as path:
+
+        with pd.option_context('io.parquet.engine', 'auto'):
+            df.to_parquet(path)
+
+            result = read_parquet(path, compression=None)
+            tm.assert_frame_equal(result, df)
+
+
+def test_options_get_engine(fp, pa):
+    assert isinstance(get_engine('pyarrow'), PyArrowImpl)
+    assert isinstance(get_engine('fastparquet'), FastParquetImpl)
+
+    with pd.option_context('io.parquet.engine', 'pyarrow'):
+        assert isinstance(get_engine('auto'), PyArrowImpl)
+        assert isinstance(get_engine('pyarrow'), PyArrowImpl)
+        assert isinstance(get_engine('fastparquet'), FastParquetImpl)
+
+    with pd.option_context('io.parquet.engine', 'fastparquet'):
+        assert isinstance(get_engine('auto'), FastParquetImpl)
+        assert isinstance(get_engine('pyarrow'), PyArrowImpl)
+        assert isinstance(get_engine('fastparquet'), FastParquetImpl)
+
+    with pd.option_context('io.parquet.engine', 'auto'):
+        assert isinstance(get_engine('auto'), PyArrowImpl)
+        assert isinstance(get_engine('pyarrow'), PyArrowImpl)
+        assert isinstance(get_engine('fastparquet'), FastParquetImpl)
+
+
 @pytest.mark.xfail(reason="fp does not ignore pa index __index_level_0__")
 def test_cross_engine_pa_fp(df_cross_compat, pa, fp):
     # cross-compat with differing reading/writing engines