diff --git a/CMakeLists.txt b/CMakeLists.txt
index 825da5a64181..f48e7d18d094 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -129,7 +129,7 @@ file(GLOB COMPILER_SRCS
     src/ir/*.cc
     src/target/*.cc
     src/api/*.cc
-    src/arithmetic/*.cc
+    src/arith/*.cc
     src/autotvm/*.cc
     src/codegen/*.cc
     src/lang/*.cc
diff --git a/include/tvm/arithmetic.h b/include/tvm/arith/analyzer.h
similarity index 55%
rename from include/tvm/arithmetic.h
rename to include/tvm/arith/analyzer.h
index a7b12336e663..78f12855cb93 100644
--- a/include/tvm/arithmetic.h
+++ b/include/tvm/arith/analyzer.h
@@ -18,24 +18,23 @@
  */
 
 /*!
- * \file tvm/arithmetic.h
- * \brief Algebra and set operations and simplifications.
+ * \file tvm/arith/analyzer.h
+ * \brief Algebra expression simplifications.
  */
-#ifndef TVM_ARITHMETIC_H_
-#define TVM_ARITHMETIC_H_
+#ifndef TVM_ARITH_ANALYZER_H_
+#define TVM_ARITH_ANALYZER_H_
 
 #include <tvm/support/with.h>
+#include <tvm/ir/expr.h>
+#include <tvm/arith/int_set.h>
 
 #include <vector>
 #include <unordered_map>
 #include <memory>
 #include <limits>
-#include "expr.h"
 
 namespace tvm {
-// forward delcare Tensor
-class Tensor;
-/*! \brief namespace of arithmetic */
+/*! \brief namespace of arithmetic analysis. */
 namespace arith {
 //-------------------------------------------------------
 // Base integer analysis API.
@@ -332,113 +331,6 @@ class ConstraintContext {
   std::function<void()> exit_;
 };
 
-//-----------------------------------------------
-// Integer set data structure.
-//
-// This is a API build on top of the base
-// integer analysis API to provide set analysis.
-//------------------------------------------------
-/*!
- * \brief Sign type of an integer expression.
- */
-enum SignType {
-  kPositive,
-  kNegative,
-  kZero,
-  kUnknown
-};
-
-/*!
- * \brief Base class of all IntSet containers.
- */
-struct IntSetNode : public Object {
-  static constexpr const char* _type_key = "IntSet";
-  TVM_DECLARE_BASE_OBJECT_INFO(IntSetNode, Object);
-};
-
-/*!
- * \brief Integer set class, represent a set of integers in one dimension.
- */
-class IntSet : public ObjectRef {
- public:
-  /*! \brief constructor */
-  IntSet() {}
-  // constructor from not container.
-  explicit IntSet(ObjectPtr<Object> n) : ObjectRef(n) {}
-  /*!
-   * \brief access the internal node container
-   * \return the pointer to the internal node container
-   */
-  inline const IntSetNode* operator->() const;
-  /*!
-   * \brief Find a range that covers the region.
-   * \param max_range The range to be covered.
-   * \return The covering range.
-   */
-  Range cover_range(Range max_range) const;
-  /*! \return Lower bound of the set */
-  PrimExpr min() const;
-  /*! \return upper bound of the set */
-  PrimExpr max() const;
-  /*! \return Whether the set represent nothing  */
-  bool is_nothing() const;
-  /*! \return Whether the set represent everything  */
-  bool is_everything() const;
-  /*! \return Whether the set is a single point */
-  bool is_single_point() const;
-  /*! \return Whether the set is proved to be bigger than 0 */
-  bool can_prove_positive() const;
-  /*! \return Whether the set is proved to be smaller than 0 */
-  bool can_prove_negative() const;
-  /*! \return Whether the set is proved to be smaller than or equal to 0 */
-  bool can_prove_non_positive() const;
-  /*! \return Whether the set is proved to be larger than or equal to 0 */
-  bool can_prove_non_negative() const;
-  /*! \return The sign of the elements in the integer set */
-  SignType sign_type() const;
-  /*!
-   * \brief The single point value, call only if is_single_point is true
-   * \return The point value.
-   */
-  PrimExpr point_value() const;
-  /*!
-   * \brief Try to match IntSet with range r.
-   *
-   * \note It is guanrateed that IntSet::range(r).match_range(r) == true
-   * \return true if we can prove they are the same.
-   */
-  bool match_range(const Range& r) const;
-  /*! \return The set contains nothing */
-  static IntSet nothing();
-  /*! \return The set contains everything */
-  static IntSet everything();
-  /*!
-   * \brief construct a point set.
-   * \param point The point in the set.
-   * \return construct a single point set
-   */
-  static IntSet single_point(PrimExpr point);
-  /*!
-   * \brief construct a integer set from vector expression.
-   * \param vec The vector expression, can also be single point.
-   * \return The result set containing the indices in the vector.
-   */
-  static IntSet vector(PrimExpr vec);
-  /*!
-   * \brief Construct a set representing a range.
-   * \param r The range
-   * \return constructed set.
-   */
-  static IntSet range(Range r);
-  /*!
-   * \brief Construct a set representing a interval.
-   * \param min The minimum value of the interval.
-   * \param max The maximum value of the interval.
-   * \return constructed set.
-   */
-  static IntSet interval(PrimExpr min, PrimExpr max);
-};
-
 /*!
  * \brief Integer set analyzer.
  */
@@ -545,157 +437,6 @@ class Analyzer {
   PrimExpr Simplify(const PrimExpr& expr);
 };
 
-//-----------------------------------------------
-// Integer set legacy API.
-//------------------------------------------------
-/*!
- * \brief Find an symbolic integer set that contains all possible values of
- *  e given the domain of each iteration variables.
- *
- * \param e The expression to be evaluated.
- * \param dom_map The domain of each variable.
- * \return An integer set that can cover all the possible values of e.
- */
-IntSet EvalSet(PrimExpr e,
-               const Map<IterVar, IntSet>& dom_map);
-/*!
- * \brief Same as EvalSet, but takes unordered_map
- *
- * \param e The expression to be evaluated.
- * \param dom_map The domain of each variable.
- * \return An integer set that can cover all the possible values of e.
- */
-IntSet EvalSet(PrimExpr e,
-               const std::unordered_map<const VarNode*, IntSet>& dom_map);
-
-/*!
- * \brief Find an symbolic integer set that contains is union over
- *  all the possible conditional values in dom_map.
- *
- * \param r The initial range.
- * \param dom_map The domain of each variable.
- * \return An integer set that can cover all the possible values.
- */
-IntSet EvalSet(Range r,
-               const Map<IterVar, IntSet>& dom_map);
-
-/*!
- * \brief Find an symbolic integer set that contains is union over
- *  all the possible conditional values in dom_map.
- *
- * \param s The initial set.
- * \param dom_map The domain of each variable.
- * \return An integer set that can cover all the possible values.
- */
-IntSet EvalSet(IntSet s,
-               const std::unordered_map<const VarNode*, IntSet>& dom_map);
-/*!
- * \brief Same as EvalSet, but takes unordered_map
- *
- * \param r The range to be evaluated.
- * \param dom_map The domain of each variable.
- * \return An integer set that can cover all the possible values of e.
- */
-IntSet EvalSet(Range r,
-               const std::unordered_map<const VarNode*, IntSet>& dom_map);
-
-/*! \brief Map from Expr to IntSet */
-using ExprIntSetMap = std::unordered_map<PrimExpr, IntSet, ObjectHash, ObjectEqual>;
-/*!
- * \brief Find the integer set of every sub-expression, given the
- *  domain of each iteration variables.
- *
- * \param e The expression to be evaluated.
- * \param dom_map The domain of each variable.
- * \return the map from the expression to its possible value.
- */
-ExprIntSetMap EvalSetForEachSubExpr(
-    PrimExpr e,
-    const std::unordered_map<const VarNode*, IntSet>& dom_map);
-
-/*!
- * \brief Create an union set of all sets
- * \param sets The sets to be unioned
- * \return the set after union
- */
-IntSet Union(const Array<IntSet>& sets);
-
-/*!
- * \brief Create an union set of all sets
- * \param sets The sets to be intersected
- * \return the set after intersected
- */
-IntSet Intersect(const Array<IntSet>& sets);
-
-/*!
- * \brief Deduce the bound of the target variable in a expression,
- *  give the domain of each variables. Return undefined IntSet to
- *  represent failure.
- *
- * \note The returned set may be smaller than set that
- *       contains all possible values of v that satisfies the bound.
- *
- * \param v The target variable to be deduced.
- * \param cond The conditional expression.
- * \param hint_map The domain of variable, used to help deduce.
- * \param relax_map The domain of each variable, used to relax the domain,
- *        The deduce bound must implies e for all value in relax_map
- * \return An integer set that always satisfies the condition.
- */
-IntSet DeduceBound(PrimExpr v, PrimExpr cond,
-                   const Map<Var, IntSet>& hint_map,
-                   const Map<Var, IntSet>& relax_map);
-/*!
- * \brief Same as DeduceBound with  unordered_map signature.
- *
- * \param v The target variable to be deduced.
- * \param cond The conditional expression.
- * \param hint_map The domain of variable, used to help deduce.
- * \param relax_map The domain of each variable, used to relax the domain,
- *        The deduce bound mush implies e for all value in relax_map
- * \return An integer set that always satisfies the condition.
- */
-IntSet DeduceBound(PrimExpr v, PrimExpr cond,
-                   const std::unordered_map<const VarNode*, IntSet>& hint_map,
-                   const std::unordered_map<const VarNode*, IntSet>& relax_map);
-
-/*!
- * \brief Infer a regular domain that covers all the calls or provides within the given statement.
- * \param body The given statement.
- * \param tensor The name of the calls or provides.
- * \param consider_calls If calls (read) are considered.
- * \param consider_provides If provides (write) are considered.
- * \return The domain that covers all the calls or provides within the given statement.
- */
-Domain DomainTouched(Stmt body, const Tensor &tensor, bool consider_calls, bool consider_provides);
-
-// Expression pattern detector.
-/*!
- * \brief Detect if e can be rewritten as e = sum_{i=0}^{n-1} var[i] * coeff[i] + coeff[n]
- *  Where coeff[i] and base are invariant of var[j] for all i and j.
- *
- * \param e The expression to be detected.
- * \param vars List of variables to be used in detection.
- * \return [coeff[i]] if it is possible, empty array if it is not.
- */
-Array<PrimExpr> DetectLinearEquation(const PrimExpr& e,
-                                 const Array<Var>& vars);
-
-/*!
- * \brief Detect if expression corresponds to clip bound of the vars
- *
- * \param e The expression to be detected.
- * \param vars List of variables to be used in detection.
- * \return concat([min_value[i], max_value[i]]), None is returned if there is no min or max value
- *          return empty if the e does not match the pattern.
- */
-Array<PrimExpr> DetectClipBound(const PrimExpr& e,
-                            const Array<Var>& vars);
-
-// implementation
-inline const IntSetNode* IntSet::operator->() const {
-  return static_cast<const IntSetNode*>(get());
-}
 }  // namespace arith
 }  // namespace tvm
-#endif  // TVM_ARITHMETIC_H_
+#endif  // TVM_ARITH_ANALYZER_H_
diff --git a/include/tvm/arith/bound.h b/include/tvm/arith/bound.h
new file mode 100644
index 000000000000..73c0733c41d3
--- /dev/null
+++ b/include/tvm/arith/bound.h
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * \file tvm/arith/bound.h
+ * \brief Bound deducers.
+ */
+#ifndef TVM_ARITH_BOUND_H_
+#define TVM_ARITH_BOUND_H_
+
+#include <tvm/node/container.h>
+#include <tvm/ir/expr.h>
+#include <tvm/arith/int_set.h>
+#include <tvm/expr.h>
+
+#include <unordered_map>
+
+namespace tvm {
+// forward delcare Tensor
+class Tensor;
+namespace arith {
+
+/*!
+ * \brief Deduce the bound of the target variable in a expression,
+ *  give the domain of each variables. Return undefined IntSet to
+ *  represent failure.
+ *
+ * \note The returned set may be smaller than set that
+ *       contains all possible values of v that satisfies the bound.
+ *
+ * \param v The target variable to be deduced.
+ * \param cond The conditional expression.
+ * \param hint_map The domain of variable, used to help deduce.
+ * \param relax_map The domain of each variable, used to relax the domain,
+ *        The deduce bound must implies e for all value in relax_map
+ * \return An integer set that always satisfies the condition.
+ */
+IntSet DeduceBound(PrimExpr v, PrimExpr cond,
+                   const Map<Var, IntSet>& hint_map,
+                   const Map<Var, IntSet>& relax_map);
+/*!
+ * \brief Same as DeduceBound with  unordered_map signature.
+ *
+ * \param v The target variable to be deduced.
+ * \param cond The conditional expression.
+ * \param hint_map The domain of variable, used to help deduce.
+ * \param relax_map The domain of each variable, used to relax the domain,
+ *        The deduce bound mush implies e for all value in relax_map
+ * \return An integer set that always satisfies the condition.
+ */
+IntSet DeduceBound(PrimExpr v, PrimExpr cond,
+                   const std::unordered_map<const VarNode*, IntSet>& hint_map,
+                   const std::unordered_map<const VarNode*, IntSet>& relax_map);
+
+/*!
+ * \brief Infer a regular domain that covers all the calls or provides within the given statement.
+ * \param body The given statement.
+ * \param tensor The name of the calls or provides.
+ * \param consider_calls If calls (read) are considered.
+ * \param consider_provides If provides (write) are considered.
+ * \return The domain that covers all the calls or provides within the given statement.
+ */
+Domain DomainTouched(Stmt body, const Tensor &tensor, bool consider_calls, bool consider_provides);
+
+}  // namespace arith
+}  // namespace tvm
+#endif  // TVM_ARITH_BOUND_H_
diff --git a/include/tvm/arith/int_set.h b/include/tvm/arith/int_set.h
new file mode 100644
index 000000000000..ca06cfd01c26
--- /dev/null
+++ b/include/tvm/arith/int_set.h
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file tvm/arith/int_set.h
+ * \brief Integer set
+ */
+#ifndef TVM_ARITH_INT_SET_H_
+#define TVM_ARITH_INT_SET_H_
+
+#include <tvm/ir/expr.h>
+#include <tvm/expr.h>
+#include <unordered_map>
+
+namespace tvm {
+namespace arith {
+
+//-----------------------------------------------
+// Integer set data structure.
+//
+// This is a API build on top of the base
+// integer analysis API to provide set analysis.
+//------------------------------------------------
+/*!
+ * \brief Sign type of an integer expression.
+ */
+enum SignType {
+  kPositive,
+  kNegative,
+  kZero,
+  kUnknown
+};
+
+/*!
+ * \brief Base class of all Integer set containers.
+ *        represent a set of integers in one dimension.
+ * \sa IntSet
+ */
+class IntSetNode : public Object {
+ public:
+  static constexpr const char* _type_key = "IntSet";
+  TVM_DECLARE_BASE_OBJECT_INFO(IntSetNode, Object);
+};
+
+/*!
+ * \brief Managed reference to IntSetNode.
+ * \sa IntSetNode
+ */
+class IntSet : public ObjectRef {
+ public:
+  /*! \brief constructor */
+  IntSet() {}
+  // constructor from not container.
+  explicit IntSet(ObjectPtr<Object> n) : ObjectRef(n) {}
+  /*!
+   * \brief access the internal node container
+   * \return the pointer to the internal node container
+   */
+  const IntSetNode* operator->() const {
+    return static_cast<const IntSetNode*>(get());
+  }
+  /*!
+   * \brief Find a range that covers the region.
+   * \param max_range The range to be covered.
+   * \return The covering range.
+   */
+  Range cover_range(Range max_range) const;
+  /*! \return Lower bound of the set */
+  PrimExpr min() const;
+  /*! \return upper bound of the set */
+  PrimExpr max() const;
+  /*! \return Whether the set represent nothing  */
+  bool is_nothing() const;
+  /*! \return Whether the set represent everything  */
+  bool is_everything() const;
+  /*! \return Whether the set is a single point */
+  bool is_single_point() const;
+  /*! \return Whether the set is proved to be bigger than 0 */
+  bool can_prove_positive() const;
+  /*! \return Whether the set is proved to be smaller than 0 */
+  bool can_prove_negative() const;
+  /*! \return Whether the set is proved to be smaller than or equal to 0 */
+  bool can_prove_non_positive() const;
+  /*! \return Whether the set is proved to be larger than or equal to 0 */
+  bool can_prove_non_negative() const;
+  /*! \return The sign of the elements in the integer set */
+  SignType sign_type() const;
+  /*!
+   * \brief The single point value, call only if is_single_point is true
+   * \return The point value.
+   */
+  PrimExpr point_value() const;
+  /*!
+   * \brief Try to match IntSet with range r.
+   *
+   * \note It is guanrateed that IntSet::range(r).match_range(r) == true
+   * \return true if we can prove they are the same.
+   */
+  bool match_range(const Range& r) const;
+  /*! \return The set contains nothing */
+  static IntSet nothing();
+  /*! \return The set contains everything */
+  static IntSet everything();
+  /*!
+   * \brief construct a point set.
+   * \param point The point in the set.
+   * \return construct a single point set
+   */
+  static IntSet single_point(PrimExpr point);
+  /*!
+   * \brief construct a integer set from vector expression.
+   * \param vec The vector expression, can also be single point.
+   * \return The result set containing the indices in the vector.
+   */
+  static IntSet vector(PrimExpr vec);
+  /*!
+   * \brief Construct a set representing a range.
+   * \param r The range
+   * \return constructed set.
+   */
+  static IntSet range(Range r);
+  /*!
+   * \brief Construct a set representing a interval.
+   * \param min The minimum value of the interval.
+   * \param max The maximum value of the interval.
+   * \return constructed set.
+   */
+  static IntSet interval(PrimExpr min, PrimExpr max);
+};
+
+//-----------------------------------------------
+// Integer set legacy API.
+//------------------------------------------------
+/*!
+ * \brief Find an symbolic integer set that contains all possible values of
+ *  e given the domain of each iteration variables.
+ *
+ * \param e The expression to be evaluated.
+ * \param dom_map The domain of each variable.
+ * \return An integer set that can cover all the possible values of e.
+ */
+IntSet EvalSet(PrimExpr e,
+               const Map<IterVar, IntSet>& dom_map);
+/*!
+ * \brief Same as EvalSet, but takes unordered_map
+ *
+ * \param e The expression to be evaluated.
+ * \param dom_map The domain of each variable.
+ * \return An integer set that can cover all the possible values of e.
+ */
+IntSet EvalSet(PrimExpr e,
+               const std::unordered_map<const VarNode*, IntSet>& dom_map);
+
+/*!
+ * \brief Find an symbolic integer set that contains is union over
+ *  all the possible conditional values in dom_map.
+ *
+ * \param r The initial range.
+ * \param dom_map The domain of each variable.
+ * \return An integer set that can cover all the possible values.
+ */
+IntSet EvalSet(Range r,
+               const Map<IterVar, IntSet>& dom_map);
+
+/*!
+ * \brief Find an symbolic integer set that contains is union over
+ *  all the possible conditional values in dom_map.
+ *
+ * \param s The initial set.
+ * \param dom_map The domain of each variable.
+ * \return An integer set that can cover all the possible values.
+ */
+IntSet EvalSet(IntSet s,
+               const std::unordered_map<const VarNode*, IntSet>& dom_map);
+/*!
+ * \brief Same as EvalSet, but takes unordered_map
+ *
+ * \param r The range to be evaluated.
+ * \param dom_map The domain of each variable.
+ * \return An integer set that can cover all the possible values of e.
+ */
+IntSet EvalSet(Range r,
+               const std::unordered_map<const VarNode*, IntSet>& dom_map);
+
+/*! \brief Map from Expr to IntSet */
+using ExprIntSetMap = std::unordered_map<PrimExpr, IntSet, ObjectHash, ObjectEqual>;
+/*!
+ * \brief Find the integer set of every sub-expression, given the
+ *  domain of each iteration variables.
+ *
+ * \param e The expression to be evaluated.
+ * \param dom_map The domain of each variable.
+ * \return the map from the expression to its possible value.
+ */
+ExprIntSetMap EvalSetForEachSubExpr(
+    PrimExpr e,
+    const std::unordered_map<const VarNode*, IntSet>& dom_map);
+
+/*!
+ * \brief Create an union set of all sets
+ * \param sets The sets to be unioned
+ * \return the set after union
+ */
+IntSet Union(const Array<IntSet>& sets);
+
+/*!
+ * \brief Create an union set of all sets
+ * \param sets The sets to be intersected
+ * \return the set after intersected
+ */
+IntSet Intersect(const Array<IntSet>& sets);
+
+}  // namespace arith
+}  // namespace tvm
+#endif  // TVM_ARITH_INT_SET_H_
diff --git a/include/tvm/arith/pattern.h b/include/tvm/arith/pattern.h
new file mode 100644
index 000000000000..a531d309f5c9
--- /dev/null
+++ b/include/tvm/arith/pattern.h
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file tvm/arith/pattern.h
+ * \brief Expression pattern detectors.
+ */
+#ifndef TVM_ARITH_PATTERN_H_
+#define TVM_ARITH_PATTERN_H_
+
+#include <tvm/node/container.h>
+#include <tvm/ir/expr.h>
+#include <tvm/expr.h>
+
+namespace tvm {
+namespace arith {
+/*!
+ * \brief Detect if e can be rewritten as e = sum_{i=0}^{n-1} var[i] * coeff[i] + coeff[n]
+ *  Where coeff[i] and base are invariant of var[j] for all i and j.
+ *
+ * \param e The expression to be detected.
+ * \param vars List of variables to be used in detection.
+ * \return [coeff[i]] if it is possible, empty array if it is not.
+ */
+Array<PrimExpr> DetectLinearEquation(const PrimExpr& e,
+                                     const Array<Var>& vars);
+
+/*!
+ * \brief Detect if expression corresponds to clip bound of the vars
+ *
+ * \param e The expression to be detected.
+ * \param vars List of variables to be used in detection.
+ * \return concat([min_value[i], max_value[i]]), None is returned if there is no min or max value
+ *          return empty if the e does not match the pattern.
+ */
+Array<PrimExpr> DetectClipBound(const PrimExpr& e,
+                                const Array<Var>& vars);
+
+}  // namespace arith
+}  // namespace tvm
+#endif  // TVM_ARITH_PATTERN_H_
diff --git a/include/tvm/build_module.h b/include/tvm/build_module.h
index 891918864d44..29903786a2b3 100644
--- a/include/tvm/build_module.h
+++ b/include/tvm/build_module.h
@@ -25,7 +25,7 @@
 #define TVM_BUILD_MODULE_H_
 
 #include <tvm/target/target.h>
-
+#include <tvm/support/with.h>
 #include <string>
 #include <vector>
 #include <utility>
diff --git a/include/tvm/operation.h b/include/tvm/operation.h
index 039e26ecfe90..54c98a38991c 100644
--- a/include/tvm/operation.h
+++ b/include/tvm/operation.h
@@ -24,14 +24,16 @@
 #ifndef TVM_OPERATION_H_
 #define TVM_OPERATION_H_
 
+#include <tvm/arith/analyzer.h>
+
 #include <string>
 #include <vector>
 #include <unordered_map>
+
 #include "expr.h"
 #include "expr_operator.h"
 #include "tensor.h"
 #include "schedule.h"
-#include "arithmetic.h"
 #include "buffer.h"
 
 namespace tvm {
diff --git a/include/tvm/tensor.h b/include/tvm/tensor.h
index 9962762d30d9..91c0c96b94ef 100644
--- a/include/tvm/tensor.h
+++ b/include/tvm/tensor.h
@@ -25,6 +25,8 @@
 #define TVM_TENSOR_H_
 
 #include <tvm/node/container.h>
+#include <tvm/arith/bound.h>
+
 #include <string>
 #include <vector>
 #include <utility>
@@ -32,7 +34,6 @@
 
 #include "expr.h"
 #include "expr_operator.h"
-#include "arithmetic.h"
 
 namespace tvm {
 
diff --git a/src/README.md b/src/README.md
index 6a7bf514adb2..b0172b665eb8 100644
--- a/src/README.md
+++ b/src/README.md
@@ -26,7 +26,7 @@ There can be internal header files within each module that sit in src.
 - node: base infra for IR/AST nodes that is dialect independent.
 - api: API function registration.
 - lang: The definition of DSL related data structure.
-- arithmetic: Arithmetic expression and set simplification.
+- arith: Arithmetic expression and set simplification.
 - op: The detail implementations about each operation(compute, scan, placeholder).
 - schedule: The operations on the schedule graph before converting to IR.
 - pass: The optimization pass on the IR structure.
diff --git a/src/api/api_arith.cc b/src/api/api_arith.cc
index 7150d2723ab7..b4020eb944ea 100644
--- a/src/api/api_arith.cc
+++ b/src/api/api_arith.cc
@@ -21,6 +21,11 @@
  *  Implementation of API functions related to arith
  * \file api_arith.cc
  */
+#include <tvm/arith/bound.h>
+#include <tvm/arith/int_set.h>
+#include <tvm/arith/pattern.h>
+#include <tvm/arith/analyzer.h>
+
 #include <tvm/expr.h>
 #include <tvm/ir.h>
 #include <tvm/runtime/registry.h>
diff --git a/src/arithmetic/analyzer.cc b/src/arith/analyzer.cc
similarity index 98%
rename from src/arithmetic/analyzer.cc
rename to src/arith/analyzer.cc
index e03e5e2387bf..a36c4093684c 100644
--- a/src/arithmetic/analyzer.cc
+++ b/src/arith/analyzer.cc
@@ -18,10 +18,10 @@
  */
 
 /*!
- * \file tvm/arithmetic/analyzer.cc
+ * \file tvm/arith/analyzer.cc
  */
 #include <tvm/ir.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/expr_operator.h>
 
 namespace tvm {
diff --git a/src/arithmetic/bound_deducer.cc b/src/arith/bound_deducer.cc
similarity index 99%
rename from src/arithmetic/bound_deducer.cc
rename to src/arith/bound_deducer.cc
index 5128096eb40f..f1da23ba596f 100644
--- a/src/arithmetic/bound_deducer.cc
+++ b/src/arith/bound_deducer.cc
@@ -24,14 +24,14 @@
 #include <tvm/expr.h>
 #include <tvm/ir_pass.h>
 #include <tvm/ir_functor_ext.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/runtime/registry.h>
 #include <tvm/packed_func_ext.h>
 
 
 #include <unordered_set>
 #include <unordered_map>
-#include "int_set.h"
+#include "interval_set.h"
 
 namespace tvm {
 namespace arith {
diff --git a/src/arithmetic/canonical_simplify.cc b/src/arith/canonical_simplify.cc
similarity index 99%
rename from src/arithmetic/canonical_simplify.cc
rename to src/arith/canonical_simplify.cc
index 90c6e48ded1e..b3cfc725aa41 100644
--- a/src/arithmetic/canonical_simplify.cc
+++ b/src/arith/canonical_simplify.cc
@@ -21,7 +21,7 @@
  * \file canonical_simplify.cc
  * \brief Canonical form based simplification.
  */
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/expr_operator.h>
 #include "const_fold.h"
 #include "pattern_match.h"
diff --git a/src/arithmetic/compute_expr.h b/src/arith/compute_expr.h
similarity index 96%
rename from src/arithmetic/compute_expr.h
rename to src/arith/compute_expr.h
index d78838f85ae5..b53543caae61 100644
--- a/src/arithmetic/compute_expr.h
+++ b/src/arith/compute_expr.h
@@ -21,8 +21,8 @@
  * \file compute_expr.h
  * \brief Utility to invoke certan compute operations.
  */
-#ifndef TVM_ARITHMETIC_COMPUTE_EXPR_H_
-#define TVM_ARITHMETIC_COMPUTE_EXPR_H_
+#ifndef TVM_ARITH_COMPUTE_EXPR_H_
+#define TVM_ARITH_COMPUTE_EXPR_H_
 
 #include <tvm/ir.h>
 #include <limits>
@@ -126,4 +126,4 @@ inline PrimExpr ComputeReduce(const Array<PrimExpr>& values, PrimExpr empty_valu
 
 }  // namespace arith
 }  // namespace tvm
-#endif   // TVM_ARITHMETIC_COMPUTE_EXPR_H_
+#endif   // TVM_ARITH_COMPUTE_EXPR_H_
diff --git a/src/arithmetic/const_fold.h b/src/arith/const_fold.h
similarity index 99%
rename from src/arithmetic/const_fold.h
rename to src/arith/const_fold.h
index d82ac892a5bb..0d8e2abe2102 100644
--- a/src/arithmetic/const_fold.h
+++ b/src/arith/const_fold.h
@@ -21,8 +21,8 @@
  * \file const_fold.h
  * \brief Centralized location for constant folding.
  */
-#ifndef TVM_ARITHMETIC_CONST_FOLD_H_
-#define TVM_ARITHMETIC_CONST_FOLD_H_
+#ifndef TVM_ARITH_CONST_FOLD_H_
+#define TVM_ARITH_CONST_FOLD_H_
 
 #include <tvm/ir.h>
 #include <tvm/expr_operator.h>
@@ -400,4 +400,4 @@ inline bool is_neg_inf(const PrimExpr& value) {
 
 }  // namespace arith
 }  // namespace tvm
-#endif  // TVM_ARITHMETIC_CONST_FOLD_H_
+#endif  // TVM_ARITH_CONST_FOLD_H_
diff --git a/src/arithmetic/const_int_bound.cc b/src/arith/const_int_bound.cc
similarity index 99%
rename from src/arithmetic/const_int_bound.cc
rename to src/arith/const_int_bound.cc
index c1cf1e08d2c3..460b9d9a54e6 100644
--- a/src/arithmetic/const_int_bound.cc
+++ b/src/arith/const_int_bound.cc
@@ -18,9 +18,9 @@
  */
 
 /*!
- * \file tvm/arithmetic/const_int_bound.cc
+ * \file tvm/arith/const_int_bound.cc
  */
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir_functor_ext.h>
 #include <algorithm>
 #include "int_operator.h"
diff --git a/src/arithmetic/detect_linear_equation.cc b/src/arith/detect_linear_equation.cc
similarity index 99%
rename from src/arithmetic/detect_linear_equation.cc
rename to src/arith/detect_linear_equation.cc
index 3de555765293..81740a0be426 100644
--- a/src/arithmetic/detect_linear_equation.cc
+++ b/src/arith/detect_linear_equation.cc
@@ -24,7 +24,7 @@
 #include <tvm/expr.h>
 #include <tvm/ir_pass.h>
 #include <tvm/ir_functor_ext.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 
 namespace tvm {
 namespace arith {
diff --git a/src/arithmetic/domain_touched.cc b/src/arith/domain_touched.cc
similarity index 100%
rename from src/arithmetic/domain_touched.cc
rename to src/arith/domain_touched.cc
diff --git a/src/arithmetic/int_operator.h b/src/arith/int_operator.h
similarity index 97%
rename from src/arithmetic/int_operator.h
rename to src/arith/int_operator.h
index fd510916e1a7..6e7f3959c5c9 100644
--- a/src/arithmetic/int_operator.h
+++ b/src/arith/int_operator.h
@@ -21,8 +21,8 @@
  * \file int_operator.h
  * \brief Additional useful operators for integer.
  */
-#ifndef TVM_ARITHMETIC_INT_OPERATOR_H_
-#define TVM_ARITHMETIC_INT_OPERATOR_H_
+#ifndef TVM_ARITH_INT_OPERATOR_H_
+#define TVM_ARITH_INT_OPERATOR_H_
 
 #include <limits>
 
@@ -143,4 +143,4 @@ inline int64_t floormod(int64_t x, int64_t y) {
 
 }  // namespace arith
 }  // namespace tvm
-#endif  // TVM_ARITHMETIC_INT_OPERATOR_H_
+#endif  // TVM_ARITH_INT_OPERATOR_H_
diff --git a/src/arithmetic/int_set.cc b/src/arith/int_set.cc
similarity index 99%
rename from src/arithmetic/int_set.cc
rename to src/arith/int_set.cc
index 3ea6ee5cc608..82f0d2b33660 100644
--- a/src/arithmetic/int_set.cc
+++ b/src/arith/int_set.cc
@@ -21,6 +21,7 @@
  * \file int_set.cc
  * \brief The integer set functions
  */
+#include <tvm/arith/int_set.h>
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
 #include <tvm/runtime/registry.h>
@@ -29,7 +30,7 @@
 #include <utility>
 #include <algorithm>
 #include <unordered_map>
-#include "int_set.h"
+#include "interval_set.h"
 #include "pattern_match.h"
 
 namespace tvm {
diff --git a/src/arithmetic/int_set.h b/src/arith/interval_set.h
similarity index 96%
rename from src/arithmetic/int_set.h
rename to src/arith/interval_set.h
index b28f1cb4d3a4..e931a7c04284 100644
--- a/src/arithmetic/int_set.h
+++ b/src/arith/interval_set.h
@@ -21,10 +21,10 @@
  * \file int_set.h
  * \brief Internal data structure for integer set.
  */
-#ifndef TVM_ARITHMETIC_INT_SET_H_
-#define TVM_ARITHMETIC_INT_SET_H_
+#ifndef TVM_ARITH_INTERVAL_SET_H_
+#define TVM_ARITH_INTERVAL_SET_H_
 
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/expr_operator.h>
 #include <limits>
 #include "const_fold.h"
@@ -141,4 +141,4 @@ TVM_DLL IntervalSet Intersect(Analyzer *analzyer, IntervalSet a, IntervalSet b);
 }  // namespace arith
 }  // namespace tvm
 
-#endif  // TVM_ARITHMETIC_INT_SET_H_
+#endif  // TVM_ARITH_INTERVAL_SET_H_
diff --git a/src/arithmetic/ir_mutator_with_analyzer.cc b/src/arith/ir_mutator_with_analyzer.cc
similarity index 99%
rename from src/arithmetic/ir_mutator_with_analyzer.cc
rename to src/arith/ir_mutator_with_analyzer.cc
index 1345e7e7a137..7c1c30a51761 100644
--- a/src/arithmetic/ir_mutator_with_analyzer.cc
+++ b/src/arith/ir_mutator_with_analyzer.cc
@@ -18,7 +18,7 @@
  */
 
 /*!
- * \file tvm/arithmetic/ir_mutator_with_analyzer.cc
+ * \file tvm/arith/ir_mutator_with_analyzer.cc
  */
 #include <tvm/ir_pass.h>
 #include <tvm/expr_operator.h>
diff --git a/src/arithmetic/ir_mutator_with_analyzer.h b/src/arith/ir_mutator_with_analyzer.h
similarity index 92%
rename from src/arithmetic/ir_mutator_with_analyzer.h
rename to src/arith/ir_mutator_with_analyzer.h
index a2297cb3c04e..10dc427116c6 100644
--- a/src/arithmetic/ir_mutator_with_analyzer.h
+++ b/src/arith/ir_mutator_with_analyzer.h
@@ -21,11 +21,11 @@
  * \file tvm/arithmetic/ir_mutator_with_analyzer.h
  * \brief IR mutator base-class with an analyzer context.
  */
-#ifndef TVM_ARITHMETIC_IR_MUTATOR_WITH_ANALYZER_H_
-#define TVM_ARITHMETIC_IR_MUTATOR_WITH_ANALYZER_H_
+#ifndef TVM_ARITH_IR_MUTATOR_WITH_ANALYZER_H_
+#define TVM_ARITH_IR_MUTATOR_WITH_ANALYZER_H_
 
 #include <tvm/ir_functor_ext.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <utility>
 
 namespace tvm {
@@ -66,4 +66,4 @@ class IRMutatorWithAnalyzer : public ir::StmtExprMutator {
 
 }  // namespace arith
 }  // namespace tvm
-#endif  // TVM_ARITHMETIC_IR_MUTATOR_WITH_ANALYZER_H_
+#endif  // TVM_ARITH_IR_MUTATOR_WITH_ANALYZER_H_
diff --git a/src/arithmetic/ir_visitor_with_analyzer.h b/src/arith/ir_visitor_with_analyzer.h
similarity index 92%
rename from src/arithmetic/ir_visitor_with_analyzer.h
rename to src/arith/ir_visitor_with_analyzer.h
index 08be59b8c423..8e5c0b19bd2f 100644
--- a/src/arithmetic/ir_visitor_with_analyzer.h
+++ b/src/arith/ir_visitor_with_analyzer.h
@@ -22,10 +22,10 @@
  * \brief IR visitor class with an analyzer context.
  */
 
-#ifndef TVM_ARITHMETIC_IR_VISITOR_WITH_ANALYZER_H_
-#define TVM_ARITHMETIC_IR_VISITOR_WITH_ANALYZER_H_
+#ifndef TVM_ARITH_IR_VISITOR_WITH_ANALYZER_H_
+#define TVM_ARITH_IR_VISITOR_WITH_ANALYZER_H_
 
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
 
@@ -73,4 +73,4 @@ class IRVisitorWithAnalyzer final : public StmtExprVisitor {
 
 }  // namespace ir
 }  // namespace tvm
-#endif  // TVM_ARITHMETIC_IR_VISITOR_WITH_ANALYZER_H_
+#endif  // TVM_ARITH_IR_VISITOR_WITH_ANALYZER_H_
diff --git a/src/arithmetic/modular_set.cc b/src/arith/modular_set.cc
similarity index 99%
rename from src/arithmetic/modular_set.cc
rename to src/arith/modular_set.cc
index c81842035c9f..c79e94f967eb 100644
--- a/src/arithmetic/modular_set.cc
+++ b/src/arith/modular_set.cc
@@ -21,7 +21,7 @@
  * \file modular_set.cc
  * \brief Modular set analysis
  */
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/expr_operator.h>
 #include <tvm/ir_functor_ext.h>
 #include <limits>
diff --git a/src/arithmetic/pattern_match.h b/src/arith/pattern_match.h
similarity index 99%
rename from src/arithmetic/pattern_match.h
rename to src/arith/pattern_match.h
index a236e65a8312..f6e6508366db 100644
--- a/src/arithmetic/pattern_match.h
+++ b/src/arith/pattern_match.h
@@ -62,8 +62,8 @@
  *       Please be aware that the filled value in a PVar
  *       can be overriden in the next call to Match.
  */
-#ifndef TVM_ARITHMETIC_PATTERN_MATCH_H_
-#define TVM_ARITHMETIC_PATTERN_MATCH_H_
+#ifndef TVM_ARITH_PATTERN_MATCH_H_
+#define TVM_ARITH_PATTERN_MATCH_H_
 
 #include <tvm/ir_pass.h>
 #include <tuple>
@@ -798,4 +798,4 @@ if_then_else(const Pattern<TCond>& cond,
 
 }  // namespace arith
 }  // namespace tvm
-#endif  // TVM_ARITHMETIC_PATTERN_MATCH_H_
+#endif  // TVM_ARITH_PATTERN_MATCH_H_
diff --git a/src/arithmetic/rewrite_simplify.cc b/src/arith/rewrite_simplify.cc
similarity index 99%
rename from src/arithmetic/rewrite_simplify.cc
rename to src/arith/rewrite_simplify.cc
index e6e1524604ce..5f486cf88deb 100644
--- a/src/arithmetic/rewrite_simplify.cc
+++ b/src/arith/rewrite_simplify.cc
@@ -22,7 +22,7 @@
  * \brief Rewrite-rule based simplification.
  */
 // Acknowledgement: Most rewrite-rules are from Halide.
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/expr_operator.h>
 #include <algorithm>
 #include "const_fold.h"
diff --git a/src/arithmetic/rewrite_simplify.h b/src/arith/rewrite_simplify.h
similarity index 96%
rename from src/arithmetic/rewrite_simplify.h
rename to src/arith/rewrite_simplify.h
index 6b4193c3ff59..3255376ebd2c 100644
--- a/src/arithmetic/rewrite_simplify.h
+++ b/src/arith/rewrite_simplify.h
@@ -21,10 +21,10 @@
  * \file rewrite_simplify.h
  * \brief Rewrite-rule based simplification.
  */
-#ifndef TVM_ARITHMETIC_REWRITE_SIMPLIFY_H_
-#define TVM_ARITHMETIC_REWRITE_SIMPLIFY_H_
+#ifndef TVM_ARITH_REWRITE_SIMPLIFY_H_
+#define TVM_ARITH_REWRITE_SIMPLIFY_H_
 
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/expr_operator.h>
 #include <unordered_map>
 #include <vector>
@@ -141,4 +141,4 @@ class RewriteSimplifier::Impl : public IRMutatorWithAnalyzer {
 
 }  // namespace arith
 }  // namespace tvm
-#endif  // TVM_ARITHMETIC_REWRITE_SIMPLIFY_H_
+#endif  // TVM_ARITH_REWRITE_SIMPLIFY_H_
diff --git a/src/arithmetic/stmt_simplify.cc b/src/arith/stmt_simplify.cc
similarity index 98%
rename from src/arithmetic/stmt_simplify.cc
rename to src/arith/stmt_simplify.cc
index dcc7e5dfbb53..0b7a4b7b416d 100644
--- a/src/arithmetic/stmt_simplify.cc
+++ b/src/arith/stmt_simplify.cc
@@ -23,9 +23,9 @@
  */
 #include <tvm/ir.h>
 #include <tvm/ir_pass.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/expr_operator.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include "ir_mutator_with_analyzer.h"
 
 namespace tvm {
diff --git a/src/codegen/codegen_c.cc b/src/codegen/codegen_c.cc
index 5ecf97056ea9..3d41d0828ace 100644
--- a/src/codegen/codegen_c.cc
+++ b/src/codegen/codegen_c.cc
@@ -24,7 +24,7 @@
 #include <cctype>
 #include "codegen_c.h"
 #include "../pass/ir_util.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace codegen {
diff --git a/src/codegen/llvm/codegen_llvm.cc b/src/codegen/llvm/codegen_llvm.cc
index 60d8146fc0e6..d214add06ac1 100644
--- a/src/codegen/llvm/codegen_llvm.cc
+++ b/src/codegen/llvm/codegen_llvm.cc
@@ -31,7 +31,7 @@
 #include "codegen_cpu.h"
 #include "../build_common.h"
 #include "../../pass/ir_util.h"
-#include "../../arithmetic/compute_expr.h"
+#include "../../arith/compute_expr.h"
 
 namespace tvm {
 namespace codegen {
diff --git a/src/codegen/llvm/codegen_llvm.h b/src/codegen/llvm/codegen_llvm.h
index b269f2423fc8..34e1fb8509a1 100644
--- a/src/codegen/llvm/codegen_llvm.h
+++ b/src/codegen/llvm/codegen_llvm.h
@@ -25,10 +25,10 @@
 #define TVM_CODEGEN_LLVM_CODEGEN_LLVM_H_
 #ifdef TVM_LLVM_VERSION
 
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
 #include <tvm/codegen.h>
-#include <tvm/arithmetic.h>
 #include <memory>
 #include <utility>
 #include <vector>
diff --git a/src/codegen/spirv/codegen_spirv.cc b/src/codegen/spirv/codegen_spirv.cc
index 985f6816a640..91eee8c345ed 100644
--- a/src/codegen/spirv/codegen_spirv.cc
+++ b/src/codegen/spirv/codegen_spirv.cc
@@ -25,7 +25,7 @@
 #include <tvm/ir_pass.h>
 #include <string>
 #include "codegen_spirv.h"
-#include "../../arithmetic/compute_expr.h"
+#include "../../arith/compute_expr.h"
 
 namespace tvm {
 namespace codegen {
diff --git a/src/codegen/spirv/codegen_spirv.h b/src/codegen/spirv/codegen_spirv.h
index 5aa7f9c49910..877bc712085b 100644
--- a/src/codegen/spirv/codegen_spirv.h
+++ b/src/codegen/spirv/codegen_spirv.h
@@ -24,6 +24,7 @@
 #ifndef TVM_CODEGEN_SPIRV_CODEGEN_SPIRV_H_
 #define TVM_CODEGEN_SPIRV_CODEGEN_SPIRV_H_
 
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
 #include <tvm/lowered_func.h>
diff --git a/src/lang/buffer.cc b/src/lang/buffer.cc
index 925e3dbaffc6..8c7108ae6819 100644
--- a/src/lang/buffer.cc
+++ b/src/lang/buffer.cc
@@ -26,7 +26,7 @@
 #include <tvm/ir_pass.h>
 #include <iterator>
 #include <stack>
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 
diff --git a/src/lang/expr_operator.cc b/src/lang/expr_operator.cc
index 5b4f4d1132e1..966087c5025d 100644
--- a/src/lang/expr_operator.cc
+++ b/src/lang/expr_operator.cc
@@ -25,7 +25,7 @@
 #include <tvm/expr_operator.h>
 #include <cmath>
 // Centralized header for constant folders.
-#include "../arithmetic/const_fold.h"
+#include "../arith/const_fold.h"
 
 namespace tvm {
 
diff --git a/src/op/compute_op.cc b/src/op/compute_op.cc
index 7c8427a1f42f..b82bab599b3e 100644
--- a/src/op/compute_op.cc
+++ b/src/op/compute_op.cc
@@ -22,7 +22,7 @@
  * \file compute_op.cc
  */
 #include <tvm/operation.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/ir_pass.h>
 #include <tvm/ir_functor_ext.h>
@@ -32,8 +32,8 @@
 #include "compute_op.h"
 #include "op_util.h"
 #include "../schedule/message_passing.h"
-#include "../arithmetic/compute_expr.h"
-#include "../arithmetic/int_set.h"
+#include "../arith/compute_expr.h"
+#include "../arith/interval_set.h"
 
 namespace tvm {
 
diff --git a/src/op/extern_op.cc b/src/op/extern_op.cc
index 6fc54a8bd506..fb9f49162be5 100644
--- a/src/op/extern_op.cc
+++ b/src/op/extern_op.cc
@@ -22,7 +22,7 @@
  * \file extern_op.cc
  */
 #include <tvm/operation.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <unordered_set>
 #include "op_util.h"
diff --git a/src/op/hybrid_op.cc b/src/op/hybrid_op.cc
index c3be2346b55b..8687ad661582 100644
--- a/src/op/hybrid_op.cc
+++ b/src/op/hybrid_op.cc
@@ -22,7 +22,7 @@
  * \file hybrid_op.cc
  */
 #include <tvm/operation.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
 #include <tvm/ir_pass.h>
diff --git a/src/op/op_util.cc b/src/op/op_util.cc
index e108ad312b0a..52d3b5aa200a 100644
--- a/src/op/op_util.cc
+++ b/src/op/op_util.cc
@@ -28,7 +28,7 @@
 #include <string>
 #include "op_util.h"
 #include "../schedule/message_passing.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace op {
diff --git a/src/op/tensor_compute_op.cc b/src/op/tensor_compute_op.cc
index 08df8b7f64ed..4c758ddd81fe 100644
--- a/src/op/tensor_compute_op.cc
+++ b/src/op/tensor_compute_op.cc
@@ -22,13 +22,13 @@
  * \file tensor_compute_op.cc
  */
 #include <tvm/operation.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/ir_pass.h>
 #include <unordered_set>
 #include "./op_util.h"
 #include "./compute_op.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 using namespace ir;
diff --git a/src/pass/arg_binder.cc b/src/pass/arg_binder.cc
index 0f350d2d732e..3b3bf4b1853e 100644
--- a/src/pass/arg_binder.cc
+++ b/src/pass/arg_binder.cc
@@ -26,7 +26,7 @@
 #include <tvm/runtime/device_api.h>
 #include "ir_util.h"
 #include "arg_binder.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/pass/hoist_if_then_else.cc b/src/pass/hoist_if_then_else.cc
index aff7d3de9eb2..6d4df47a730c 100644
--- a/src/pass/hoist_if_then_else.cc
+++ b/src/pass/hoist_if_then_else.cc
@@ -22,14 +22,14 @@
  */
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/runtime/registry.h>
 #include <tvm/packed_func_ext.h>
 
 #include <unordered_map>
 #include <unordered_set>
 #include <queue>
-#include "../arithmetic/int_set.h"
+#include "../arith/interval_set.h"
 #include "../runtime/thread_storage_scope.h"
 
 namespace tvm {
diff --git a/src/pass/inject_copy_intrin.cc b/src/pass/inject_copy_intrin.cc
index e41a868a6372..af83f47d8790 100644
--- a/src/pass/inject_copy_intrin.cc
+++ b/src/pass/inject_copy_intrin.cc
@@ -21,11 +21,12 @@
  * \brief Replace certain copy with copy intrinsics.
  * \file copy_intrin_rewrite.cc
  */
+#include <tvm/arith/pattern.h>
 #include <tvm/ir.h>
 #include <tvm/packed_func_ext.h>
 #include <tvm/ir_functor_ext.h>
 #include <tvm/ir_pass.h>
-#include "../arithmetic/pattern_match.h"
+#include "../arith/pattern_match.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/pass/inject_double_buffer.cc b/src/pass/inject_double_buffer.cc
index 9ed56062dacc..691f6a79e667 100644
--- a/src/pass/inject_double_buffer.cc
+++ b/src/pass/inject_double_buffer.cc
@@ -25,7 +25,7 @@
 #include <tvm/ir_functor_ext.h>
 #include <tvm/expr_operator.h>
 #include "ir_util.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/pass/inject_prefetch.cc b/src/pass/inject_prefetch.cc
index d7abed8c6062..d87786385a02 100644
--- a/src/pass/inject_prefetch.cc
+++ b/src/pass/inject_prefetch.cc
@@ -24,7 +24,7 @@
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
 #include <tvm/ir_pass.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <unordered_set>
 
 namespace tvm {
diff --git a/src/pass/inject_virtual_thread.cc b/src/pass/inject_virtual_thread.cc
index 83fc582cd4f1..a0a67a785a5a 100644
--- a/src/pass/inject_virtual_thread.cc
+++ b/src/pass/inject_virtual_thread.cc
@@ -24,7 +24,7 @@
 #include <tvm/ir_functor_ext.h>
 #include <tvm/ir_pass.h>
 #include <unordered_set>
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/pass/loop_partition.cc b/src/pass/loop_partition.cc
index adcd5ecd0287..7af2240294dd 100644
--- a/src/pass/loop_partition.cc
+++ b/src/pass/loop_partition.cc
@@ -23,10 +23,10 @@
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
 #include <tvm/ir_pass.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <unordered_map>
 #include <unordered_set>
-#include "../arithmetic/int_set.h"
+#include "../arith/interval_set.h"
 #include "../runtime/thread_storage_scope.h"
 
 namespace tvm {
diff --git a/src/pass/lower_intrin.cc b/src/pass/lower_intrin.cc
index 5684f4ef785f..7172a4a6dda2 100644
--- a/src/pass/lower_intrin.cc
+++ b/src/pass/lower_intrin.cc
@@ -29,8 +29,8 @@
 #include <tvm/expr_operator.h>
 #include <unordered_set>
 #include "ir_util.h"
-#include "../arithmetic/pattern_match.h"
-#include "../arithmetic/ir_mutator_with_analyzer.h"
+#include "../arith/pattern_match.h"
+#include "../arith/ir_mutator_with_analyzer.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/pass/lower_thread_allreduce.cc b/src/pass/lower_thread_allreduce.cc
index d509169df0b1..7b1378f8686c 100644
--- a/src/pass/lower_thread_allreduce.cc
+++ b/src/pass/lower_thread_allreduce.cc
@@ -26,7 +26,7 @@
 #include <tvm/ir_pass.h>
 #include <unordered_set>
 #include "ir_util.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 #include "../runtime/thread_storage_scope.h"
 
 namespace tvm {
diff --git a/src/pass/lower_tvm_builtin.cc b/src/pass/lower_tvm_builtin.cc
index 59a6473728d6..13e2504a7bf0 100644
--- a/src/pass/lower_tvm_builtin.cc
+++ b/src/pass/lower_tvm_builtin.cc
@@ -26,7 +26,7 @@
 #include <tvm/ir_pass.h>
 #include <unordered_set>
 #include "ir_util.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/pass/lower_warp_memory.cc b/src/pass/lower_warp_memory.cc
index 6a1c3c499b83..8da07f02ca29 100644
--- a/src/pass/lower_warp_memory.cc
+++ b/src/pass/lower_warp_memory.cc
@@ -25,12 +25,15 @@
  */
 // Thanks to Andrew Adams and Vinod Grover for
 // explaining the concept of warp shuffle.
+#include <tvm/arith/pattern.h>
+#include <tvm/arith/analyzer.h>
+
 #include <tvm/ir.h>
 #include <tvm/ir_functor_ext.h>
 #include <tvm/ir_pass.h>
 #include <unordered_set>
 #include "ir_util.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 #include "../runtime/thread_storage_scope.h"
 
 namespace tvm {
diff --git a/src/pass/storage_access.cc b/src/pass/storage_access.cc
index da153fc12804..43acbafc8554 100644
--- a/src/pass/storage_access.cc
+++ b/src/pass/storage_access.cc
@@ -26,7 +26,7 @@
 #include <utility>
 #include "ir_util.h"
 #include "storage_access.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/pass/storage_flatten.cc b/src/pass/storage_flatten.cc
index a6d83a825458..fc46ef3a4523 100644
--- a/src/pass/storage_flatten.cc
+++ b/src/pass/storage_flatten.cc
@@ -22,7 +22,7 @@
  */
 // Flattens storage from multi-dimensional array to 1D
 // buffer access as in Halide pipeline.
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/expr.h>
 #include <tvm/operation.h>
@@ -35,8 +35,8 @@
 #include <unordered_map>
 #include "ir_util.h"
 #include "arg_binder.h"
-#include "../arithmetic/compute_expr.h"
-#include "../arithmetic/ir_visitor_with_analyzer.h"
+#include "../arith/compute_expr.h"
+#include "../arith/ir_visitor_with_analyzer.h"
 #include "../runtime/thread_storage_scope.h"
 
 namespace tvm {
diff --git a/src/pass/storage_rewrite.cc b/src/pass/storage_rewrite.cc
index 49084209c07a..8b55dab31e9c 100644
--- a/src/pass/storage_rewrite.cc
+++ b/src/pass/storage_rewrite.cc
@@ -22,6 +22,7 @@
  * \brief Memory access pattern analysis and optimization.
  *  Re-write data access to enable memory sharing when possible.
  */
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/ir_pass.h>
 #include <tvm/ir_functor_ext.h>
@@ -30,7 +31,7 @@
 #include <unordered_set>
 #include <unordered_map>
 #include "ir_util.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 #include "../runtime/thread_storage_scope.h"
 
 namespace tvm {
diff --git a/src/pass/tensor_core.cc b/src/pass/tensor_core.cc
index dad2780be425..c5c81cacb688 100644
--- a/src/pass/tensor_core.cc
+++ b/src/pass/tensor_core.cc
@@ -33,7 +33,7 @@
 #include <tvm/runtime/device_api.h>
 #include <unordered_map>
 #include "ir_util.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 #include "../runtime/thread_storage_scope.h"
 
 namespace tvm {
diff --git a/src/pass/unroll_loop.cc b/src/pass/unroll_loop.cc
index 26ad59189671..2658e76880a2 100644
--- a/src/pass/unroll_loop.cc
+++ b/src/pass/unroll_loop.cc
@@ -28,7 +28,7 @@
 #include <unordered_set>
 #include <unordered_map>
 #include <vector>
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/pass/vectorize_loop.cc b/src/pass/vectorize_loop.cc
index c9f3441e6c47..5e5e427b126e 100644
--- a/src/pass/vectorize_loop.cc
+++ b/src/pass/vectorize_loop.cc
@@ -24,11 +24,11 @@
 #include <tvm/ir.h>
 #include <tvm/ir_pass.h>
 #include <tvm/ir_functor_ext.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <unordered_set>
 #include <unordered_map>
 #include <vector>
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace ir {
diff --git a/src/relay/op/tensor/transform.cc b/src/relay/op/tensor/transform.cc
index 0d4e0a4bfb4f..59bb432e756d 100644
--- a/src/relay/op/tensor/transform.cc
+++ b/src/relay/op/tensor/transform.cc
@@ -35,7 +35,7 @@
 #include <topi/nn.h>
 #include <vector>
 #include "../op_common.h"
-#include "../../../arithmetic/compute_expr.h"
+#include "../../../arith/compute_expr.h"
 #include "../../pass/infer_layout_util.h"
 #include "../../pass/pattern_util.h"
 #include "transform.h"
diff --git a/src/schedule/message_passing.cc b/src/schedule/message_passing.cc
index 869e3051d39f..816ea440b566 100644
--- a/src/schedule/message_passing.cc
+++ b/src/schedule/message_passing.cc
@@ -21,11 +21,11 @@
  * \file message_passing.cc
  * \brief The message passing domain.
  */
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <tvm/ir.h>
 #include <tvm/ir_pass.h>
 #include "message_passing.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 namespace schedule {
diff --git a/src/schedule/message_passing.h b/src/schedule/message_passing.h
index 66615890a50c..4b81bccf4b2f 100644
--- a/src/schedule/message_passing.h
+++ b/src/schedule/message_passing.h
@@ -28,7 +28,7 @@
 #include <tvm/expr.h>
 #include <tvm/schedule.h>
 #include <tvm/operation.h>
-#include <tvm/arithmetic.h>
+#include <tvm/arith/analyzer.h>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
diff --git a/src/schedule/schedule_dataflow_rewrite.cc b/src/schedule/schedule_dataflow_rewrite.cc
index 3bad33811d78..f06cf96c551b 100644
--- a/src/schedule/schedule_dataflow_rewrite.cc
+++ b/src/schedule/schedule_dataflow_rewrite.cc
@@ -27,7 +27,7 @@
 #include <unordered_set>
 #include "message_passing.h"
 #include "../pass/ir_util.h"
-#include "../arithmetic/compute_expr.h"
+#include "../arith/compute_expr.h"
 
 namespace tvm {
 
diff --git a/tests/cpp/pattern_match_test.cc b/tests/cpp/pattern_match_test.cc
index 193f2f206c06..ffd710fd83bb 100644
--- a/tests/cpp/pattern_match_test.cc
+++ b/tests/cpp/pattern_match_test.cc
@@ -18,7 +18,7 @@
  */
 
 #include <gtest/gtest.h>
-#include "../src/arithmetic/pattern_match.h"
+#include "../src/arith/pattern_match.h"
 
 TEST(Pattern, Basic) {
   using namespace tvm;