diff --git a/c_glib/Brewfile b/c_glib/Brewfile
index 54738efb711..599fa7390ba 100644
--- a/c_glib/Brewfile
+++ b/c_glib/Brewfile
@@ -23,7 +23,6 @@ brew "cmake"
 brew "git"
 brew "gobject-introspection"
 brew "gtk-doc"
-brew "jemalloc"
 brew "libtool"
 brew "lua"
 brew "luarocks"
diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.yml
index 5aa3c32b417..0ae21ec6d97 100644
--- a/ci/conda_env_cpp.yml
+++ b/ci/conda_env_cpp.yml
@@ -22,7 +22,6 @@ flatbuffers
 gflags
 glog
 gtest
-jemalloc
 libprotobuf
 lz4-c
 python
diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh
index c1db2d5c5bd..4dbfce0dbaa 100755
--- a/ci/travis_install_linux.sh
+++ b/ci/travis_install_linux.sh
@@ -19,7 +19,7 @@
 
 sudo apt-get install -y -q \
     gdb binutils ccache libboost-dev libboost-filesystem-dev \
-    libboost-system-dev libboost-regex-dev libjemalloc-dev
+    libboost-system-dev libboost-regex-dev
 
 if [ "$CXX" == "g++-4.9" ]; then
     sudo apt-get install -y -q g++-4.9
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 15ff5ec8d31..b4617645ddb 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -309,6 +309,15 @@ if ("${MAKE}" STREQUAL "")
     endif()
 endif()
 
+# Using make -j in sub-make is fragile
+# see discussion https://github.com/apache/arrow/pull/2779
+if (${CMAKE_GENERATOR} MATCHES "Makefiles")
+    set(MAKE_BUILD_ARGS "")
+else()
+    # limit the maximum number of jobs for ninja
+    set(MAKE_BUILD_ARGS "-j4")
+endif()
+
 # ----------------------------------------------------------------------
 # Find pthreads
 
@@ -736,9 +745,9 @@ if (ARROW_JEMALLOC)
     CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}" "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_" "--disable-tls"
     ${EP_LOG_OPTIONS}
     BUILD_IN_SOURCE 1
-    BUILD_COMMAND ${MAKE} -j
+    BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}
     BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}"
-    INSTALL_COMMAND ${MAKE} -j1 install)
+    INSTALL_COMMAND ${MAKE} install)
 
   # Don't use the include directory directly so that we can point to a path
   # that is unique to our codebase.
diff --git a/dev/container/Dockerfile b/dev/container/Dockerfile
index de0729ba534..e77279968a1 100644
--- a/dev/container/Dockerfile
+++ b/dev/container/Dockerfile
@@ -41,9 +41,8 @@ ENV PATH="/miniconda/bin:$PATH"
 RUN conda create -y -q -n pyarrow-dev \
 	python=3.6 numpy six setuptools cython pandas pytest \
 	cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
-  	gflags brotli jemalloc lz4-c zstd -c conda-forge \
+  	gflags brotli lz4-c zstd -c conda-forge \
 	&& conda clean --all
 
 ADD script ./script
 RUN chmod u=rwx ./script/*.sh
-
diff --git a/dev/docker_common/Dockerfile.xenial.base b/dev/docker_common/Dockerfile.xenial.base
index 49d9fd72d96..bead6948c0f 100644
--- a/dev/docker_common/Dockerfile.xenial.base
+++ b/dev/docker_common/Dockerfile.xenial.base
@@ -57,7 +57,6 @@ RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -
       gtest \
       re2 \
       brotli \
-      jemalloc \
       lz4-c \
       zstd \
       setuptools \
diff --git a/dev/spark_integration/Dockerfile b/dev/spark_integration/Dockerfile
index 81b801cc62b..84e353a9b50 100644
--- a/dev/spark_integration/Dockerfile
+++ b/dev/spark_integration/Dockerfile
@@ -52,7 +52,6 @@ RUN conda create -y -q -c conda-forge -n pyarrow-dev \
       zlib \
       gflags \
       brotli \
-      jemalloc \
       lz4-c \
       zstd \
       setuptools \
diff --git a/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile b/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
index 24a7ddf7c7e..4dde574cbf9 100644
--- a/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
+++ b/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
@@ -41,7 +41,6 @@ RUN \
     libboost-system-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
-    libjemalloc-dev \
     libprotobuf-dev \
     libprotoc-dev \
     lsb-release \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
index dcf783604a5..5d3c9ba2932 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
@@ -39,7 +39,6 @@ RUN \
     libboost-system-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
-    libjemalloc-dev \
     libprotobuf-dev \
     libprotoc-dev \
     lsb-release \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
index 727e03524e6..e88c24177b4 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
@@ -40,7 +40,6 @@ RUN \
     libboost-system-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
-    libjemalloc-dev \
     lsb-release \
     pkg-config && \
   apt-get clean && \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
index 93a6d3a4b35..17cb27713f0 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
@@ -40,7 +40,6 @@ RUN \
     libboost-system-dev \
     libgirepository1.0-dev \
     libglib2.0-doc \
-    libjemalloc-dev \
     libprotobuf-dev \
     libprotoc-dev \
     lsb-release \
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/control b/dev/tasks/linux-packages/debian.ubuntu-trusty/control
index c8c9f9c8180..696f2c4b696 100644
--- a/dev/tasks/linux-packages/debian.ubuntu-trusty/control
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/control
@@ -15,7 +15,6 @@ Build-Depends:
   libboost-regex-dev,
   libboost-system-dev,
   libgirepository1.0-dev,
-  libjemalloc-dev,
   pkg-config
 Build-Depends-Indep: libglib2.0-doc
 Standards-Version: 3.9.6
diff --git a/dev/tasks/linux-packages/debian/control b/dev/tasks/linux-packages/debian/control
index 5521f9f9fee..d497a31d144 100644
--- a/dev/tasks/linux-packages/debian/control
+++ b/dev/tasks/linux-packages/debian/control
@@ -16,7 +16,6 @@ Build-Depends:
   libboost-regex-dev,
   libboost-system-dev,
   libgirepository1.0-dev,
-  libjemalloc-dev,
   libprotobuf-dev,
   libprotoc-dev,
   nvidia-cuda-toolkit,
diff --git a/dev/tasks/linux-packages/yum/arrow.spec.in b/dev/tasks/linux-packages/yum/arrow.spec.in
index c2715dd81a7..3fc82398be1 100644
--- a/dev/tasks/linux-packages/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/yum/arrow.spec.in
@@ -37,7 +37,6 @@ BuildRequires:	gcc-c++
 BuildRequires:	cmake3
 BuildRequires:	boost-devel
 BuildRequires:	git
-BuildRequires:	jemalloc-devel
 %if %{use_python}
 BuildRequires:	python34-devel
 BuildRequires:	python34-numpy
@@ -112,7 +111,6 @@ License:	Apache-2.0
 Requires:	boost-system
 Requires:	boost-filesystem
 Requires:	boost-regex
-Requires:	jemalloc
 
 %description libs
 This package contains the libraries for Apache Arrow C++.
diff --git a/dev/tasks/linux-packages/yum/centos-6/Dockerfile b/dev/tasks/linux-packages/yum/centos-6/Dockerfile
index 90bd710963c..8143b99efd1 100644
--- a/dev/tasks/linux-packages/yum/centos-6/Dockerfile
+++ b/dev/tasks/linux-packages/yum/centos-6/Dockerfile
@@ -51,7 +51,6 @@ RUN \
     git \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     pkg-config \
     python34 \
     python34-devel \
diff --git a/dev/tasks/linux-packages/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/yum/centos-7/Dockerfile
index a8710c5a0ec..d672e46bd2d 100644
--- a/dev/tasks/linux-packages/yum/centos-7/Dockerfile
+++ b/dev/tasks/linux-packages/yum/centos-7/Dockerfile
@@ -33,7 +33,6 @@ RUN \
     git \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     pkg-config \
     python34 \
     python34-devel \
diff --git a/python/Brewfile b/python/Brewfile
index dae4544e438..f8cdf222f41 100644
--- a/python/Brewfile
+++ b/python/Brewfile
@@ -16,6 +16,5 @@
 # under the License.
 
 brew "ccache"
-brew "jemalloc"
 brew "boost"
 brew "thrift"