Merge pull request #5022 from jtratner/arithmetic-refactor2

CLN/ENH: Provide full suite of arithmetic (and flex) methods to all NDFrame objects.

Author: jtratner

doc/source/api.rst

@@ -275,12 +275,30 @@ Binary operator functions
    :toctree: generated/
 
    Series.add
-   Series.div
-   Series.mul
    Series.sub
+   Series.mul
+   Series.div
+   Series.truediv
+   Series.floordiv
+   Series.mod
+   Series.pow
+   Series.radd
+   Series.rsub
+   Series.rmul
+   Series.rdiv
+   Series.rtruediv
+   Series.rfloordiv
+   Series.rmod
+   Series.rpow
    Series.combine
    Series.combine_first
    Series.round
+   Series.lt
+   Series.gt
+   Series.le
+   Series.ge
+   Series.ne
+   Series.eq
 
 Function application, GroupBy
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -480,13 +498,27 @@ Binary operator functions
    :toctree: generated/
 
    DataFrame.add
-   DataFrame.div
-   DataFrame.mul
    DataFrame.sub
+   DataFrame.mul
+   DataFrame.div
+   DataFrame.truediv
+   DataFrame.floordiv
+   DataFrame.mod
+   DataFrame.pow
    DataFrame.radd
-   DataFrame.rdiv
-   DataFrame.rmul
    DataFrame.rsub
+   DataFrame.rmul
+   DataFrame.rdiv
+   DataFrame.rtruediv
+   DataFrame.rfloordiv
+   DataFrame.rmod
+   DataFrame.rpow
+   DataFrame.lt
+   DataFrame.gt
+   DataFrame.le
+   DataFrame.ge
+   DataFrame.ne
+   DataFrame.eq
    DataFrame.combine
    DataFrame.combineAdd
    DataFrame.combine_first
@@ -710,9 +742,27 @@ Binary operator functions
    :toctree: generated/
 
    Panel.add
-   Panel.div
-   Panel.mul
    Panel.sub
+   Panel.mul
+   Panel.div
+   Panel.truediv
+   Panel.floordiv
+   Panel.mod
+   Panel.pow
+   Panel.radd
+   Panel.rsub
+   Panel.rmul
+   Panel.rdiv
+   Panel.rtruediv
+   Panel.rfloordiv
+   Panel.rmod
+   Panel.rpow
+   Panel.lt
+   Panel.gt
+   Panel.le
+   Panel.ge
+   Panel.ne
+   Panel.eq
 
 Function application, GroupBy
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/release.rst

@@ -263,6 +263,10 @@ API Changes
   - Begin removing methods that don't make sense on ``GroupBy`` objects
     (:issue:`4887`).
   - Remove deprecated ``read_clipboard/to_clipboard/ExcelFile/ExcelWriter`` from ``pandas.io.parsers`` (:issue:`3717`)
+  - All non-Index NDFrames (``Series``, ``DataFrame``, ``Panel``, ``Panel4D``,
+    ``SparsePanel``, etc.), now support the entire set of arithmetic operators
+    and arithmetic flex methods (add, sub, mul, etc.). ``SparsePanel`` does not
+    support ``pow`` or ``mod`` with non-scalars. (:issue:`3765`)
 
 Internal Refactoring
 ~~~~~~~~~~~~~~~~~~~~

doc/source/v0.13.0.txt

@@ -68,6 +68,11 @@ API changes
         df1 and df2
         s1 and s2
 
+  - All non-Index NDFrames (``Series``, ``DataFrame``, ``Panel``, ``Panel4D``,
+    ``SparsePanel``, etc.), now support the entire set of arithmetic operators
+    and arithmetic flex methods (add, sub, mul, etc.). ``SparsePanel`` does not
+    support ``pow`` or ``mod`` with non-scalars. (:issue:`3765`)
+
 
 Prior Version Deprecations/Changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

pandas/computation/expressions.py

@@ -15,6 +15,8 @@
 except ImportError:  # pragma: no cover
     _NUMEXPR_INSTALLED = False
 
+_TEST_MODE = None
+_TEST_RESULT = None
 _USE_NUMEXPR = _NUMEXPR_INSTALLED
 _evaluate = None
 _where = None
@@ -55,9 +57,10 @@ def set_numexpr_threads(n=None):
 
 def _evaluate_standard(op, op_str, a, b, raise_on_error=True, **eval_kwargs):
     """ standard evaluation """
+    if _TEST_MODE:
+        _store_test_result(False)
     return op(a, b)
 
-
 def _can_use_numexpr(op, op_str, a, b, dtype_check):
     """ return a boolean if we WILL be using numexpr """
     if op_str is not None:
@@ -88,11 +91,8 @@ def _evaluate_numexpr(op, op_str, a, b, raise_on_error=False, **eval_kwargs):
 
     if _can_use_numexpr(op, op_str, a, b, 'evaluate'):
         try:
-            a_value, b_value = a, b
-            if hasattr(a_value, 'values'):
-                a_value = a_value.values
-            if hasattr(b_value, 'values'):
-                b_value = b_value.values
+            a_value = getattr(a, "values", a)
+            b_value = getattr(b, "values", b)
             result = ne.evaluate('a_value %s b_value' % op_str,
                                  local_dict={'a_value': a_value,
                                              'b_value': b_value},
@@ -104,6 +104,9 @@ def _evaluate_numexpr(op, op_str, a, b, raise_on_error=False, **eval_kwargs):
             if raise_on_error:
                 raise
 
+    if _TEST_MODE:
+        _store_test_result(result is not None)
+
     if result is None:
         result = _evaluate_standard(op, op_str, a, b, raise_on_error)
 
@@ -119,13 +122,9 @@ def _where_numexpr(cond, a, b, raise_on_error=False):
     if _can_use_numexpr(None, 'where', a, b, 'where'):
 
         try:
-            cond_value, a_value, b_value = cond, a, b
-            if hasattr(cond_value, 'values'):
-                cond_value = cond_value.values
-            if hasattr(a_value, 'values'):
-                a_value = a_value.values
-            if hasattr(b_value, 'values'):
-                b_value = b_value.values
+            cond_value = getattr(cond, 'values', cond)
+            a_value = getattr(a, 'values', a)
+            b_value = getattr(b, 'values', b)
             result = ne.evaluate('where(cond_value, a_value, b_value)',
                                  local_dict={'cond_value': cond_value,
                                              'a_value': a_value,
@@ -189,3 +188,28 @@ def where(cond, a, b, raise_on_error=False, use_numexpr=True):
     if use_numexpr:
         return _where(cond, a, b, raise_on_error=raise_on_error)
     return _where_standard(cond, a, b, raise_on_error=raise_on_error)
+
+
+def set_test_mode(v = True):
+    """
+    Keeps track of whether numexpr  was used.  Stores an additional ``True`` for
+    every successful use of evaluate with numexpr since the last
+    ``get_test_result``
+    """
+    global _TEST_MODE, _TEST_RESULT
+    _TEST_MODE = v
+    _TEST_RESULT = []
+
+
+def _store_test_result(used_numexpr):
+    global _TEST_RESULT
+    if used_numexpr:
+        _TEST_RESULT.append(used_numexpr)
+
+
+def get_test_result():
+    """get test result and reset test_results"""
+    global _TEST_RESULT
+    res = _TEST_RESULT
+    _TEST_RESULT = []
+    return res

pandas/computation/tests/test_eval.py

@@ -2,9 +2,7 @@
 
 import unittest
 import functools
-import numbers
 from itertools import product
-import ast
 
 import nose
 from nose.tools import assert_raises, assert_true, assert_false, assert_equal
@@ -250,12 +248,6 @@ def check_complex_cmp_op(self, lhs, cmp1, rhs, binop, cmp2):
                     not np.isscalar(rhs_new) and binop in skip_these):
                 with tm.assertRaises(TypeError):
                     _eval_single_bin(lhs_new, binop, rhs_new, self.engine)
-            elif _bool_and_frame(lhs_new, rhs_new):
-                with tm.assertRaises(TypeError):
-                    _eval_single_bin(lhs_new, binop, rhs_new, self.engine)
-                with tm.assertRaises(TypeError):
-                    pd.eval('lhs_new & rhs_new'.format(binop),
-                            engine=self.engine, parser=self.parser)
             else:
                 expected = _eval_single_bin(lhs_new, binop, rhs_new, self.engine)
                 result = pd.eval(ex, engine=self.engine, parser=self.parser)
@@ -301,28 +293,15 @@ def check_operands(left, right, cmp_op):
         rhs_new = check_operands(mid, rhs, cmp2)
 
         if lhs_new is not None and rhs_new is not None:
-            # these are not compatible operands
-            if isinstance(lhs_new, Series) and isinstance(rhs_new, DataFrame):
-                self.assertRaises(TypeError, _eval_single_bin, lhs_new, '&',
-                                  rhs_new, self.engine)
-            elif (_bool_and_frame(lhs_new, rhs_new)):
-                self.assertRaises(TypeError, _eval_single_bin, lhs_new, '&',
-                                  rhs_new, self.engine)
-            elif _series_and_2d_ndarray(lhs_new, rhs_new):
-                # TODO: once #4319 is fixed add this test back in
-                #self.assertRaises(Exception, _eval_single_bin, lhs_new, '&',
-                                  #rhs_new, self.engine)
-                pass
-            else:
-                ex1 = 'lhs {0} mid {1} rhs'.format(cmp1, cmp2)
-                ex2 = 'lhs {0} mid and mid {1} rhs'.format(cmp1, cmp2)
-                ex3 = '(lhs {0} mid) & (mid {1} rhs)'.format(cmp1, cmp2)
-                expected = _eval_single_bin(lhs_new, '&', rhs_new, self.engine)
-
-                for ex in (ex1, ex2, ex3):
-                    result = pd.eval(ex, engine=self.engine,
-                                     parser=self.parser)
-                    assert_array_equal(result, expected)
+            ex1 = 'lhs {0} mid {1} rhs'.format(cmp1, cmp2)
+            ex2 = 'lhs {0} mid and mid {1} rhs'.format(cmp1, cmp2)
+            ex3 = '(lhs {0} mid) & (mid {1} rhs)'.format(cmp1, cmp2)
+            expected = _eval_single_bin(lhs_new, '&', rhs_new, self.engine)
+
+            for ex in (ex1, ex2, ex3):
+                result = pd.eval(ex, engine=self.engine,
+                                    parser=self.parser)
+                assert_array_equal(result, expected)
 
     @skip_incompatible_operand
     def check_simple_cmp_op(self, lhs, cmp1, rhs):

pandas/core/common.py

@@ -8,6 +8,7 @@
 import codecs
 import csv
 import sys
+import types
 
 from datetime import timedelta
 
@@ -27,6 +28,7 @@
 from pandas.core.config import get_option
 from pandas.core import array as pa
 
+
 class PandasError(Exception):
     pass
 
@@ -74,6 +76,31 @@ def __instancecheck__(cls, inst):
 
 ABCGeneric = _ABCGeneric("ABCGeneric", tuple(), {})
 
+
+def bind_method(cls, name, func):
+    """Bind a method to class, python 2 and python 3 compatible.
+
+    Parameters
+    ----------
+
+    cls : type
+        class to receive bound method
+    name : basestring
+        name of method on class instance
+    func : function
+        function to be bound as method
+
+
+    Returns
+    -------
+    None
+    """
+    # only python 2 has bound/unbound method issue
+    if not compat.PY3:
+        setattr(cls, name, types.MethodType(func, None, cls))
+    else:
+        setattr(cls, name, func)
+
 def isnull(obj):
     """Detect missing values (NaN in numeric arrays, None/NaN in object arrays)
 
@@ -360,10 +387,10 @@ def _take_2d_multi_generic(arr, indexer, out, fill_value, mask_info):
         if col_needs:
             out[:, col_mask] = fill_value
     for i in range(len(row_idx)):
-        u = row_idx[i]
+        u_ = row_idx[i]
         for j in range(len(col_idx)):
             v = col_idx[j]
-            out[i, j] = arr[u, v]
+            out[i, j] = arr[u_, v]
 
 
 def _take_nd_generic(arr, indexer, out, axis, fill_value, mask_info):
@@ -2348,3 +2375,10 @@ def save(obj, path):  # TODO remove in 0.13
     warnings.warn("save is deprecated, use obj.to_pickle", FutureWarning)
     from pandas.io.pickle import to_pickle
     return to_pickle(obj, path)
+
+
+def _maybe_match_name(a, b):
+    name = None
+    if a.name == b.name:
+        name = a.name
+    return name