Remove bundled Python packages

.github/workflows/coverity.yml

@@ -107,7 +107,6 @@ jobs:
             --with-perl \
             --with-pgport=5432 \
             --with-python \
-            --with-pythonsrc-ext \
             --with-ssl=openssl \
             --with-uuid=e2fs \
             --with-includes=/usr/local/xerces-c/include \

configure

@@ -570,6 +570,7 @@ ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
 #
 # Initializations.
 #
+ac_default_prefix=/usr/local
 ac_clean_files=
 ac_config_libobj_dir=.
 LIBOBJS=
@@ -742,7 +743,6 @@ with_ldap
 with_krb_srvnam
 krb_srvtab
 with_gssapi
-with_pythonsrc_ext
 with_python
 with_perl
 with_tcl
@@ -758,8 +758,8 @@ HAVE_CXX14
 enable_gpcloud
 enable_shared_postgres_backend
 enable_mapreduce
-enable_catalog_ext
 enable_serverless
+enable_catalog_ext
 enable_orca
 autodepend
 PKG_CONFIG_LIBDIR
@@ -907,7 +907,6 @@ with_tcl
 with_tclconfig
 with_perl
 with_python
-with_pythonsrc_ext
 with_gssapi
 with_krb_srvnam
 with_pam
@@ -1606,7 +1605,7 @@ Optional Features:
   --enable-cassert        enable assertion checks (for debugging)
   --disable-orca          disable ORCA optimizer
   --enable-catalog-ext    enable Cloudberry catalog extension
-  --enable-serverless     enable Cloudberry serverless mode
+  --enable-serverless     use serverless mode of Cloudberry
   --enable-mapreduce      enable Cloudberry Mapreduce support
   --disable-shared-postgres-backend
                           enable Cloudberry shared postgres backend support
@@ -1644,7 +1643,6 @@ Optional Packages:
   --with-tclconfig=DIR    tclConfig.sh is in DIR
   --with-perl             build Perl modules (PL/Perl)
   --without-python        build Python modules (PL/Python)
-  --with-pythonsrc-ext    build Python modules for gpMgmt
   --with-gssapi           build with GSSAPI support
   --with-krb-srvnam=NAME  default service principal name in Kerberos (GSSAPI)
                           [postgres]
@@ -8377,7 +8375,7 @@ $as_echo "checking whether to build with catalog extension... $enable_catalog_ex
 
 
 #
-# Enable serverless mode
+# --enable-serverless uses serverless mode of Cloudberry
 #
 
 
@@ -8404,8 +8402,8 @@ else
 fi
 
 
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to use serverless architecture of Cloudberry ... $enable_serverless" >&5
-$as_echo "checking whether to use serverless architecture of Cloudberry ... $enable_serverless" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to use serverless mode of Cloudberry... $enable_serverless" >&5
+$as_echo "checking whether to use serverless mode of Cloudberry... $enable_serverless" >&6; }
 
 
 #
@@ -9423,40 +9421,6 @@ fi
 $as_echo "$with_python" >&6; }
 
 
-#
-# Optionally build Python modules for gpMgmt
-#
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build Python modules for gpMgmt" >&5
-$as_echo_n "checking whether to build Python modules for gpMgmt... " >&6; }
-
-
-
-# Check whether --with-pythonsrc-ext was given.
-if test "${with_pythonsrc_ext+set}" = set; then :
-  withval=$with_pythonsrc_ext;
-  case $withval in
-    yes)
-      :
-      ;;
-    no)
-      :
-      ;;
-    *)
-      as_fn_error $? "no argument expected for --with-pythonsrc-ext option" "$LINENO" 5
-      ;;
-  esac
-
-else
-  with_pythonsrc_ext=no
-
-fi
-
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_pythonsrc_ext" >&5
-$as_echo "$with_pythonsrc_ext" >&6; }
-
-
-
 #
 # GSSAPI
 #

configure.ac

@@ -1011,15 +1011,6 @@ PGAC_ARG_BOOL(with, python, yes, [build Python modules (PL/Python)])
 AC_MSG_RESULT([$with_python])
 AC_SUBST(with_python)
 
-#
-# Optionally build Python modules for gpMgmt
-#
-AC_MSG_CHECKING([whether to build Python modules for gpMgmt])
-PGAC_ARG_BOOL(with, pythonsrc-ext, no, [build Python modules for gpMgmt])
-AC_MSG_RESULT([$with_pythonsrc_ext])
-AC_SUBST(with_pythonsrc_ext)
-
-
 #
 # GSSAPI
 #

gpAux/Makefile

@@ -124,13 +124,13 @@ DEFPORT=5432
 
 ORCA_CONFIG=--enable-orca
 
-rhel6_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap --with-pythonsrc-ext
-rhel7_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap --with-pythonsrc-ext
-rhel8_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap --with-pythonsrc-ext
-ubuntu18.04_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap --with-pythonsrc-ext
-sles12_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap --with-pythonsrc-ext
-rhel7_aarch64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap --with-pythonsrc-ext
-common_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap --with-pythonsrc-ext
+rhel6_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap
+rhel7_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap
+rhel8_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap
+ubuntu18.04_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap
+sles12_x86_64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap
+rhel7_aarch64_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap
+common_CONFIGFLAGS=--with-quicklz --with-lz4 --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-ssl=openssl --with-pam --with-ldap
 #BLD_CONFIGFLAGS=$($(BLD_ARCH)_CONFIGFLAGS)
 BLD_CONFIGFLAGS=$(common_CONFIGFLAGS)
 

gpMgmt/Makefile

@@ -13,22 +13,9 @@ generate_greenplum_path_file:
 
 install: generate_greenplum_path_file
 	mkdir -p $(DESTDIR)$(prefix)/lib/python
-
-	# Setup /lib/python contents
-	if [ -e bin/ext/__init__.py ]; then \
-	    cp -rp bin/ext/__init__.py $(DESTDIR)$(prefix)/lib/python ; \
-	fi
-	if [ -e bin/ext/psutil ]; then \
-	    cp -rp bin/ext/psutil $(DESTDIR)$(prefix)/lib/python ; \
-	fi
-	if [ -e bin/ext/pgdb.py ]; then \
-	    cp -rp bin/ext/pgdb.py $(DESTDIR)$(prefix)/lib/python && \
-	    cp -rp bin/ext/pg.py $(DESTDIR)$(prefix)/lib/python && \
-	    cp -rp bin/ext/_pg*.so $(DESTDIR)$(prefix)/lib/python ; \
-	fi
-	if [ -e bin/ext/yaml ]; then \
-	    cp -rp bin/ext/yaml $(DESTDIR)$(prefix)/lib/python ; \
-	fi
+
+    # Remove PyGreSQL, psutil and PyYAML specific copy operations
+    # These will be installed via pip
 
 clean distclean:
 	$(MAKE) -C bin $@

gpMgmt/bin/Makefile

@@ -54,13 +54,10 @@ uninstall:
 # SOURCE DIRECTORIES
 #
 SRC=$(CURDIR)
-PYLIB_SRC=$(SRC)/pythonSrc
-PYLIB_SRC_EXT=$(PYLIB_SRC)/ext
 SBIN_DIR=$(SRC)/../sbin
 SERVER_SRC=$(SRC)
 SERVER_SBIN=$(SERVER_SRC)/../sbin
 
-
 #
 # INSTALL DIRECTORY
 #
@@ -70,128 +67,9 @@ PYLIB_DIR=$(SRC)/ext
 core:
 	python3 gpconfig_modules/parse_guc_metadata.py $(DESTDIR)$(prefix)
 
-ifeq ($(with_pythonsrc_ext), yes)
-install: installdirs installprograms core psutil pygresql pyyaml
-else
 install: installdirs installprograms core
-endif
-
-#
-# Python Libraries
-#
-
-#
-# PyGreSQL
-#
-PYGRESQL_VERSION=5.2
-PYGRESQL_DIR=PyGreSQL-$(PYGRESQL_VERSION)
-pygresql:
-	@echo "--- PyGreSQL"
-	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYGRESQL_DIR).tar.gz
-	cd $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/ && PATH=$(DESTDIR)$(bindir):$$PATH LDFLAGS='$(LDFLAGS) $(PYGRESQL_LDFLAGS)' python3 setup.py build
-	cp -r $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build/lib*-3*/* $(PYLIB_DIR)/
-
-
-#
-# PSUTIL
-#
-PSUTIL_VERSION=5.7.0
-PSUTIL_DIR=psutil-$(PSUTIL_VERSION)
-
-psutil:
-	@echo "--- psutil"
-ifeq "$(findstring $(BLD_ARCH),aix7_ppc_64 )" ""
-	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PSUTIL_DIR).tar.gz
-	cd $(PYLIB_SRC_EXT)/$(PSUTIL_DIR)/ && env -u CC python3 setup.py build
-	cp -r $(PYLIB_SRC_EXT)/$(PSUTIL_DIR)/build/lib.*/psutil $(PYLIB_DIR)
-endif
-
-#
-# PYYAML
-#
-PYYAML_VERSION=5.3.1
-PYYAML_DIR=PyYAML-$(PYYAML_VERSION)
-
-pyyaml:
-	@echo "--- pyyaml"
-	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYYAML_DIR).tar.gz
-	cd $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/ && env -u CC python3 setup.py build
-	cp -r $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/build/lib*-3*/* $(PYLIB_DIR)/
-
-#
-# PYLINT
-#
-
-PYLINT_VERSION=0.21.0
-PYLINT_DIR=pylint-$(PYLINT_VERSION)
-PYLINT_PYTHONPATH=$(PYLIB_DIR):$(PYLIB_SRC_EXT)/$(PYLINT_DIR)/build/lib/
-MOCK_VERSION=1.0.1
-MOCK_DIR=mock-$(MOCK_VERSION)
-SETUP_TOOLS_VERSION=36.6.0
-PARSE_VERSION=1.8.2
-SETUP_TOOLS_DIR=setuptools-$(SETUP_TOOLS_VERSION)
-PARSE_DIR=parse-$(PARSE_VERSION)
-PYTHONSRC_INSTALL=$(PYLIB_SRC_EXT)/install
-PYTHON_VERSION=$(shell python3 -c "import sys; print ('%s.%s' % (sys.version_info[0:2]))")
-PYTHONSRC_INSTALL_SITE=$(PYLIB_SRC_EXT)/install/lib/python$(PYTHON_VERSION)/site-packages
-PYTHONSRC_INSTALL_PYTHON_PATH=$(PYTHONPATH):$(PYTHONSRC_INSTALL_SITE)
-# TODO: mock-1.0.1-py2.6.egg package should be updated.
-MOCK_BIN=$(PYTHONSRC_INSTALL)/lib/python$(PYTHON_VERSION)/site-packages/mock-1.0.1-py2.6.egg
-UBUNTU_PLATFORM=$(shell if lsb_release -a 2>/dev/null | grep -q 'Ubuntu' ; then echo "Ubuntu"; fi)
-
-pylint:
-	@echo "--- pylint"
-	@cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYLINT_DIR).tar.gz
-	@cd $(PYLIB_SRC_EXT)/$(PYLINT_DIR)/ && python3 setup.py build 1> /dev/null
-	@touch $(PYLIB_SRC_EXT)/$(PYLINT_DIR)/build/lib/__init__.py
-
-$(MOCK_BIN):
-	@echo "--- mock for platform $(UBUNTU_PLATFORM)"
-	@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
-       pip3 install mock;\
-     else\
-       mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
-	   cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(MOCK_DIR).tar.gz && \
-	   cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
-	   PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
-	 fi;
-
-PYTHON_FILES=`grep -l --exclude=Makefile --exclude=gplogfilter "/bin/env python3" *`\
-			 `grep -l "/bin/env python3" $(SRC)/../sbin/*`\
-			 `find ./gppylib -name "*.py"`\
-			 `find $(SRC)/../sbin -name "*.py"`
-
-checkcode: pylint
-	@echo "Running pylint on management scripts..."
-	@PYTHONPATH=$(PYTHONPATH):$(PYLINT_PYTHONPATH) $(PYLIB_SRC_EXT)/$(PYLINT_DIR)/bin/pylint -i y $(PYTHON_FILES) --rcfile=.rcfile > $(SRC)/../pylint.txt || true
-	@echo -n "pylint_score=" > $(SRC)/../pylint_score.properties
-	@grep "Your code has been rated at" $(SRC)/../pylint.txt | sed -e "s|Your .* \(.*\)/.*|\1|" >> $(SRC)/../pylint_score.properties
-
-check: $(MOCK_BIN)
-	@echo "Running pure unit and also "unit" tests that require cluster to be up..."
-	@TMPDIR=/tmp PYTHONPATH=$(SERVER_SRC):$(SERVER_SBIN):$(PYTHONPATH):$(PYTHONSRC_INSTALL_PYTHON_PATH):$(SRC)/ext:$(SBIN_DIR):$(LIB_DIR):$(PYLIB_DIR)/mock-1.0.1 \
-	gppylib/gpunit discover --verbose -s $(SRC)/gppylib -p "test_unit*.py" 2> $(SRC)/../gpMgmt_testunit_results.log 1> $(SRC)/../gpMgmt_testunit_output.log
-	@TMPDIR=/tmp PYTHONPATH=$(SERVER_SRC):$(SERVER_SBIN):$(PYTHONPATH):$(PYTHONSRC_INSTALL_PYTHON_PATH):$(SRC)/ext:$(SBIN_DIR):$(LIB_DIR):$(PYLIB_DIR)/mock-1.0.1 \
-	gppylib/gpunit discover --verbose -s $(SRC)/gppylib -p "test_cluster*.py" 2>> $(SRC)/../gpMgmt_testunit_results.log 1>> $(SRC)/../gpMgmt_testunit_output.log
-
-unitdevel:
-	@echo "Running pure unit tests..."
-	PYTHONPATH=$(SERVER_SRC):$(SERVER_SBIN):$(PYTHONPATH):$(PYTHONSRC_INSTALL_PYTHON_PATH):$(SRC)/ext:$(SBIN_DIR):$(LIB_DIR):$(PYLIB_DIR)/mock-1.0.1 \
-	    python3 -m unittest discover --verbose -s $(SRC)/gppylib -p "test_unit*.py"
-
-
-.PHONY: installcheck-bash
-installcheck-bash:
-	./test/suite.bash
-
-.PHONY: installcheck
-installcheck: installcheck-bash
-	$(MAKE) -C gpload_test $@
 
 clean distclean:
-	rm -rf $(PYLIB_SRC_EXT)/$(PYLINT_DIR)
-	rm -rf $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build
-	rm -rf $(PYLIB_SRC)/$(PYGRESQL_DIR)/build
 	rm -rf *.pyc
 	rm -f analyzedbc gpactivatestandbyc gpaddmirrorsc gpcheckcatc \
 		  gpcheckperfc gpcheckresgroupimplc gpchecksubnetcfgc gpconfigc \

gpMgmt/bin/README.md

@@ -1,15 +1,19 @@
 # For Developers
 
-To install the libraries necessary for running scripts or testing, a system python of 2.7 must be available, the version of gcc and g++ used to compile python must be available.
-On most distributions, python will compiled with the same gcc and g++ verion available from the base packages "gcc" and "gcc-c++".
+To install the libraries necessary for running scripts or testing, a system python of 3.x must be available.
 
-The command `python -VV` will show the compiler used to compile the version of python being used.
-A `make` in from gpMgmt will install the proper libraries provided a gcc and gcc-c++ are present.
+## Installing Python Dependencies
 
-To run any of these python scripts, necessary libraries must be installed, and PYTHONPATH must be modified to use the libraries in this path.
+Install required Python packages using pip:
 
+```bash
+pip3 install -r python-dependencies.txt
 ```
-PYTHONPATH="\$GPHOME/lib/python:${PYTHONPATH}"
+
+To run any of these python scripts, necessary libraries must be installed, and PYTHONPATH must be modified to use the libraries in this path.
+
+```bash
+PYTHONPATH="$GPHOME/lib/python:${PYTHONPATH}"
 ```
 
 This will be set automatically with a `source $GPHOME/greenplum_path.sh`

gpMgmt/bin/gpload.bat

@@ -2,7 +2,7 @@
 set FOUND=
 for %%X in (python.exe) do (set FOUND=%%~$PATH:X)
 if not defined FOUND (
-    echo "Ensure python2.7 64-bit is installed and set in PATH"
+    echo "Ensure python3 is installed and set in PATH"
     exit /B 1
 )