hdfs: Allow hdfs read/write for files

Now the following will work:

If hdfs3 is not installed, Throws:
  ImportError: The hdfs3 library is required to handle hdfs files

If hdfs3 is installed but libhdfs3 is not installed, Throws:
  ImportError: Can not find the shared library: libhdfs3.so

If hdfs3 is installed it works for the code:
  pd.read_csv("hdfs://localhost:9000/tmp/a.csv")

If hdfs3 is installed and HADOOP_CONF_DIR is set, it works for the code:
  HADOOP_CONF_DIR=/usr/local/Cellar/hadoop/2.7.0/libexec/etc/hadoop/
  pd.read_csv("hdfs:///tmp/a.csv")

Author: AbdealiLoKo

doc/source/whatsnew/v0.22.0.txt

@@ -155,6 +155,7 @@ I/O
 ^^^
 
 - :func:`read_html` now rewinds seekable IO objects after parse failure, before attempting to parse with a new parser. If a parser errors and the object is non-seekable, an informative error is raised suggesting the use of a different parser (:issue:`17975`)
+- :func:`read_csv` now supports reading from hdfs by giving "hdfs:///tmp/data.csv". The hadoop configs will try to be automatically found. The configs can also be mentioned using the format "hdfs://nodenamehost:nodenameport/tmp/data.csv"
 - Bug in :func:`read_msgpack` with a non existent file is passed in Python 2 (:issue:`15296`)
 - Bug in :func:`read_csv` where a ``MultiIndex`` with duplicate columns was not being mangled appropriately (:issue:`18062`)
 - Bug in :func:`read_sas` where a file with 0 variables gave an ``AttributeError`` incorrectly. Now it gives an ``EmptyDataError`` (:issue:`18184`)

pandas/io/common.py

@@ -99,6 +99,14 @@ def _is_s3_url(url):
         return False
 
 
+def _is_hdfs_url(url):
+    """Check for an hdfs url"""
+    try:
+        return parse_url(url).scheme == 'hdfs'
+    except:
+        return False
+
+
 def _expand_user(filepath_or_buffer):
     """Return the argument with an initial component of ~ or ~user
        replaced by that user's home directory.
@@ -201,6 +209,12 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
                                          encoding=encoding,
                                          compression=compression)
 
+    if _is_hdfs_url(filepath_or_buffer):
+        from pandas.io import hdfs
+        return hdfs.get_filepath_or_buffer(filepath_or_buffer,
+                                           encoding=encoding,
+                                           compression=compression)
+
     if isinstance(filepath_or_buffer, (compat.string_types,
                                        compat.binary_type,
                                        mmap.mmap)):
@@ -314,11 +328,17 @@ def _get_handle(path_or_buf, mode, encoding=None, compression=None,
     handles : list of file-like objects
         A list of file-like object that were openned in this function.
     """
+    need_text_wrapping = [BytesIO]
     try:
         from s3fs import S3File
-        need_text_wrapping = (BytesIO, S3File)
+        need_text_wrapping.append(S3File)
+    except ImportError:
+        pass
+    try:
+        from hdfs3 import HDFile
+        need_text_wrapping.append(HDFile)
     except ImportError:
-        need_text_wrapping = (BytesIO,)
+        pass
 
     handles = list()
     f = path_or_buf

pandas/io/hdfs.py

@@ -0,0 +1,27 @@
+""" s3 support for remote file interactivity """
+from pandas import compat
+try:
+    import hdfs3
+except:
+    raise ImportError("The hdfs3 library is required to handle hdfs files")
+
+if compat.PY3:
+    from urllib.parse import urlparse as parse_url
+else:
+    from urlparse import urlparse as parse_url
+
+
+def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
+                           compression=None):
+    parsed_url = parse_url(filepath_or_buffer)
+    if ":" in parsed_url.netloc:
+        host, port = parsed_url.netloc.rsplit(":", 1)
+        try:
+            port = int(port)
+            fs = hdfs3.HDFileSystem(host=host, port=port)
+        except ValueError:
+            pass
+    else:
+        fs = hdfs3.HDFileSystem()
+    filepath_or_buffer = fs.open(parsed_url.path)
+    return filepath_or_buffer, None, compression

pandas/tests/io/test_hdfs.py

@@ -0,0 +1,8 @@
+from pandas.io.common import _is_hdfs_url
+
+
+class TestHDFSURL(object):
+
+    def test_is_hdfs_url(self):
+        assert _is_hdfs_url("hdfs://pandas/somethingelse.com")
+        assert not _is_hdfs_url("hdf://pandas/somethingelse.com")