diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 66a685b45d3..1569889c0a6 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -165,12 +165,6 @@ PrimitiveArray::PrimitiveArray(const std::shared_ptr<DataType>& type, int64_t le
   SetData(ArrayData::Make(type, length, {null_bitmap, data}, null_count, offset));
 }
 
-template <typename T>
-NumericArray<T>::NumericArray(const std::shared_ptr<ArrayData>& data)
-    : PrimitiveArray(data) {
-  DCHECK_EQ(data->type->id(), T::type_id);
-}
-
 // ----------------------------------------------------------------------
 // BooleanArray
 
@@ -987,24 +981,4 @@ std::vector<ArrayVector> RechunkArraysConsistently(
 
 }  // namespace internal
 
-// ----------------------------------------------------------------------
-// Instantiate templates
-
-template class ARROW_TEMPLATE_EXPORT NumericArray<UInt8Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<UInt16Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<UInt32Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<UInt64Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<Int8Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<Int16Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<Int32Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<Int64Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<TimestampType>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<Date32Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<Date64Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<Time32Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<Time64Type>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<HalfFloatType>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<FloatType>;
-template class ARROW_TEMPLATE_EXPORT NumericArray<DoubleType>;
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index aead17f133d..5b4daa808c7 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -398,12 +398,12 @@ class ARROW_EXPORT PrimitiveArray : public FlatArray {
 
 /// Concrete Array class for numeric data.
 template <typename TYPE>
-class ARROW_EXPORT NumericArray : public PrimitiveArray {
+class NumericArray : public PrimitiveArray {
  public:
   using TypeClass = TYPE;
   using value_type = typename TypeClass::c_type;
 
-  explicit NumericArray(const std::shared_ptr<ArrayData>& data);
+  explicit NumericArray(const std::shared_ptr<ArrayData>& data) : PrimitiveArray(data) {}
 
   // Only enable this constructor without a type argument for types without additional
   // metadata
@@ -844,27 +844,6 @@ class ARROW_EXPORT DictionaryArray : public Array {
   std::shared_ptr<Array> indices_;
 };
 
-// ----------------------------------------------------------------------
-// extern templates and other details
-
-// Only instantiate these templates once
-ARROW_EXTERN_TEMPLATE NumericArray<Int8Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<UInt8Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<Int16Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<UInt16Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<Int32Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<UInt32Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<Int64Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<UInt64Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<HalfFloatType>;
-ARROW_EXTERN_TEMPLATE NumericArray<FloatType>;
-ARROW_EXTERN_TEMPLATE NumericArray<DoubleType>;
-ARROW_EXTERN_TEMPLATE NumericArray<Date32Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<Date64Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<Time32Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<Time64Type>;
-ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
-
 /// \brief Perform any validation checks to determine obvious inconsistencies
 /// with the array's internal data
 ///
diff --git a/cpp/src/arrow/sparse_tensor.cc b/cpp/src/arrow/sparse_tensor.cc
index 101500d3643..a55f51a5673 100644
--- a/cpp/src/arrow/sparse_tensor.cc
+++ b/cpp/src/arrow/sparse_tensor.cc
@@ -264,18 +264,18 @@ class SparseTensorConverter<TYPE, SparseCSRIndex>
 // ----------------------------------------------------------------------
 // Instantiate templates
 
-#define INSTANTIATE_SPARSE_TENSOR_CONVERTER(IndexType)                                  \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<UInt8Type, IndexType>;     \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<UInt16Type, IndexType>;    \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<UInt32Type, IndexType>;    \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<UInt64Type, IndexType>;    \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<Int8Type, IndexType>;      \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<Int16Type, IndexType>;     \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<Int32Type, IndexType>;     \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<Int64Type, IndexType>;     \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<HalfFloatType, IndexType>; \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<FloatType, IndexType>;     \
-  template class ARROW_TEMPLATE_EXPORT SparseTensorConverter<DoubleType, IndexType>
+#define INSTANTIATE_SPARSE_TENSOR_CONVERTER(IndexType)            \
+  template class SparseTensorConverter<UInt8Type, IndexType>;     \
+  template class SparseTensorConverter<UInt16Type, IndexType>;    \
+  template class SparseTensorConverter<UInt32Type, IndexType>;    \
+  template class SparseTensorConverter<UInt64Type, IndexType>;    \
+  template class SparseTensorConverter<Int8Type, IndexType>;      \
+  template class SparseTensorConverter<Int16Type, IndexType>;     \
+  template class SparseTensorConverter<Int32Type, IndexType>;     \
+  template class SparseTensorConverter<Int64Type, IndexType>;     \
+  template class SparseTensorConverter<HalfFloatType, IndexType>; \
+  template class SparseTensorConverter<FloatType, IndexType>;     \
+  template class SparseTensorConverter<DoubleType, IndexType>
 
 INSTANTIATE_SPARSE_TENSOR_CONVERTER(SparseCOOIndex);
 INSTANTIATE_SPARSE_TENSOR_CONVERTER(SparseCSRIndex);
diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc
index 792945b1740..a4db298a04d 100644
--- a/cpp/src/arrow/tensor.cc
+++ b/cpp/src/arrow/tensor.cc
@@ -123,50 +123,4 @@ Type::type Tensor::type_id() const { return type_->id(); }
 
 bool Tensor::Equals(const Tensor& other) const { return TensorEquals(*this, other); }
 
-// ----------------------------------------------------------------------
-// NumericTensor
-
-template <typename TYPE>
-NumericTensor<TYPE>::NumericTensor(const std::shared_ptr<Buffer>& data,
-                                   const std::vector<int64_t>& shape)
-    : NumericTensor(data, shape, {}, {}) {}
-
-template <typename TYPE>
-NumericTensor<TYPE>::NumericTensor(const std::shared_ptr<Buffer>& data,
-                                   const std::vector<int64_t>& shape,
-                                   const std::vector<int64_t>& strides)
-    : NumericTensor(data, shape, strides, {}) {}
-
-template <typename TYPE>
-NumericTensor<TYPE>::NumericTensor(const std::shared_ptr<Buffer>& data,
-                                   const std::vector<int64_t>& shape,
-                                   const std::vector<int64_t>& strides,
-                                   const std::vector<std::string>& dim_names)
-    : Tensor(TypeTraits<TYPE>::type_singleton(), data, shape, strides, dim_names) {}
-
-template <typename TYPE>
-int64_t NumericTensor<TYPE>::CalculateValueOffset(
-    const std::vector<int64_t>& index) const {
-  int64_t offset = 0;
-  for (size_t i = 0; i < index.size(); ++i) {
-    offset += index[i] * strides_[i];
-  }
-  return offset;
-}
-
-// ----------------------------------------------------------------------
-// Instantiate templates
-
-template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt8Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt16Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt32Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt64Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<Int8Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<Int16Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<Int32Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<Int64Type>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<HalfFloatType>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<FloatType>;
-template class ARROW_TEMPLATE_EXPORT NumericTensor<DoubleType>;
-
 }  // namespace arrow
diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h
index e81f0f0dff5..445a81f2cf2 100644
--- a/cpp/src/arrow/tensor.h
+++ b/cpp/src/arrow/tensor.h
@@ -25,6 +25,7 @@
 
 #include "arrow/buffer.h"
 #include "arrow/type.h"
+#include "arrow/type_traits.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
@@ -121,22 +122,25 @@ class ARROW_EXPORT Tensor {
 };
 
 template <typename TYPE>
-class ARROW_EXPORT NumericTensor : public Tensor {
+class NumericTensor : public Tensor {
  public:
   using TypeClass = TYPE;
   using value_type = typename TypeClass::c_type;
 
+  /// Constructor with non-negative strides and dimension names
+  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
+                const std::vector<int64_t>& strides,
+                const std::vector<std::string>& dim_names)
+      : Tensor(TypeTraits<TYPE>::type_singleton(), data, shape, strides, dim_names) {}
+
   /// Constructor with no dimension names or strides, data assumed to be row-major
-  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape);
+  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape)
+      : NumericTensor(data, shape, {}, {}) {}
 
   /// Constructor with non-negative strides
   NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
-                const std::vector<int64_t>& strides);
-
-  /// Constructor with non-negative strides and dimension names
-  NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape,
-                const std::vector<int64_t>& strides,
-                const std::vector<std::string>& dim_names);
+                const std::vector<int64_t>& strides)
+      : NumericTensor(data, shape, strides, {}) {}
 
   const value_type& Value(const std::vector<int64_t>& index) const {
     int64_t offset = CalculateValueOffset(index);
@@ -145,7 +149,13 @@ class ARROW_EXPORT NumericTensor : public Tensor {
   }
 
  protected:
-  int64_t CalculateValueOffset(const std::vector<int64_t>& index) const;
+  int64_t CalculateValueOffset(const std::vector<int64_t>& index) const {
+    int64_t offset = 0;
+    for (size_t i = 0; i < index.size(); ++i) {
+      offset += index[i] * strides_[i];
+    }
+    return offset;
+  }
 };
 
 }  // namespace arrow
diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h
index 34aa752fd21..b224717a62d 100644
--- a/cpp/src/arrow/util/visibility.h
+++ b/cpp/src/arrow/util/visibility.h
@@ -43,15 +43,6 @@
 #endif
 #endif  // Non-Windows
 
-// gcc and clang disagree about how to handle template visibility when you have
-// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
-
-#if defined(__clang__)
-#define ARROW_EXTERN_TEMPLATE extern template class ARROW_EXPORT
-#else
-#define ARROW_EXTERN_TEMPLATE extern template class
-#endif
-
 // This is a complicated topic, some reading on it:
 // http://www.codesynthesis.com/~boris/blog/2010/01/18/dll-export-cxx-templates/
 #if defined(_MSC_VER) || defined(__clang__)