Cherrypick python dependencies from GP to Cloudberry

configure

@@ -742,7 +742,6 @@ with_ldap
 with_krb_srvnam
 krb_srvtab
 with_gssapi
-with_pythonsrc_ext
 with_python
 with_perl
 with_tcl
@@ -909,7 +908,6 @@ with_tcl
 with_tclconfig
 with_perl
 with_python
-with_pythonsrc_ext
 with_gssapi
 with_krb_srvnam
 with_pam
@@ -1647,7 +1645,6 @@ Optional Packages:
   --with-tclconfig=DIR    tclConfig.sh is in DIR
   --with-perl             build Perl modules (PL/Perl)
   --without-python        build Python modules (PL/Python)
-  --with-pythonsrc-ext    build Python modules for gpMgmt
   --with-gssapi           build with GSSAPI support
   --with-krb-srvnam=NAME  default service principal name in Kerberos (GSSAPI)
                           [postgres]
@@ -9456,39 +9453,6 @@ fi
 $as_echo "$with_python" >&6; }
 
 
-#
-# Optionally build Python modules for gpMgmt
-#
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build Python modules for gpMgmt" >&5
-$as_echo_n "checking whether to build Python modules for gpMgmt... " >&6; }
-
-
-
-# Check whether --with-pythonsrc-ext was given.
-if test "${with_pythonsrc_ext+set}" = set; then :
-  withval=$with_pythonsrc_ext;
-  case $withval in
-    yes)
-      :
-      ;;
-    no)
-      :
-      ;;
-    *)
-      as_fn_error $? "no argument expected for --with-pythonsrc-ext option" "$LINENO" 5
-      ;;
-  esac
-
-else
-  with_pythonsrc_ext=no
-
-fi
-
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_pythonsrc_ext" >&5
-$as_echo "$with_pythonsrc_ext" >&6; }
-
-
 
 #
 # GSSAPI

configure.ac

@@ -1021,14 +1021,6 @@ PGAC_ARG_BOOL(with, python, yes, [build Python modules (PL/Python)])
 AC_MSG_RESULT([$with_python])
 AC_SUBST(with_python)
 
-#
-# Optionally build Python modules for gpMgmt
-#
-AC_MSG_CHECKING([whether to build Python modules for gpMgmt])
-PGAC_ARG_BOOL(with, pythonsrc-ext, no, [build Python modules for gpMgmt])
-AC_MSG_RESULT([$with_pythonsrc_ext])
-AC_SUBST(with_pythonsrc_ext)
-
 
 #
 # GSSAPI

deploy/build/README.Rhel-Rocky.bash

@@ -10,6 +10,7 @@ sudo yum --enablerepo=powertools install -y libyaml-devel
 
 sudo yum install -y postgresql
 sudo yum install -y postgresql-devel
+sudo yum install -y python3-psycopg2
 
 pip3.9 install -r ../../python-dependencies.txt
 

deploy/build/README.Ubuntu.bash

@@ -36,7 +36,6 @@ sudo apt-get install -y \
 	python3-dev \
 	python3-pip \
 	python3-psutil \
-	python3-pygresql \
 	python3-yaml \
 	zlib1g-dev
 

gpAux/Makefile

@@ -142,7 +142,7 @@ endif
 
 
 ifneq (false, ${PG_LANG})
-CONFIGFLAGS+= --with-perl --with-python PYTHON=python3
+CONFIGFLAGS+= --with-perl --with-python PYTHON=python3.9
 ifdef TCL_CFG
 CONFIGFLAGS+= --with-tcl-config=${TCL_CFG}
 endif
@@ -218,8 +218,8 @@ perl_archlibexp:=$(shell perl -MConfig -e 'print $$Config{archlibexp}')
 # set default build steps
 define BUILD_STEPS
 	@$(MAKE) -C $(BUILDDIR)/src/backend generated-headers
-	cd $(BUILDDIR) && PYGRESQL_LDFLAGS=' -Wl,-rpath,\$$$$ORIGIN/.. -Wl,--enable-new-dtags ' $(MAKE) $(PARALLEL_MAKE_OPTS) install
-	cd $(BUILDDIR)/src/pl/plpython && $(MAKE) clean && $(MAKE) $(PARALLEL_MAKE_OPTS) install && cd $(BUILDDIR)
+	cd $(BUILDDIR) && $(MAKE) $(PARALLEL_MAKE_OPTS) install
+	cd $(BUILDDIR)/src/pl/plpython && $(MAKE) clean && echo 'LDFLAGS += -Wl,-rpath,\$$$$ORIGIN/../../ext/python/lib/ -Wl,--enable-new-dtags' >> Makefile && $(MAKE) $(PARALLEL_MAKE_OPTS) install && cd $(BUILDDIR)
 	cd $(BUILDDIR)/src/pl/plperl && $(MAKE) clean && echo "LDFLAGS += -Wl,-rpath,$(perl_archlibexp)/CORE -Wl,--enable-new-dtags" >> GNUmakefile && echo "LDFLAGS_SL += -Wl,-rpath,$(perl_archlibexp)/CORE -Wl,--enable-new-dtags" >> GNUmakefile && $(MAKE) $(PARALLEL_MAKE_OPTS) install && cd $(BUILDDIR)
 	#@$(MAKE) greenplum_path INSTLOC=$(INSTLOC)
 	#@$(MAKE) mgmtcopy INSTLOC=$(INSTLOC)
@@ -254,7 +254,6 @@ define BUILD_STEPS
 	cp -p $(GPMGMT)/bin/gpload $(INSTLOC)/bin/gpload
 	cp -p $(GPMGMT)/bin/gpload.py $(INSTLOC)/bin/gpload.py
 	$(MAKE) copylibs INSTLOC=$(INSTLOC)
-	cd $(GPMGMT)/bin && $(MAKE) pygresql INSTLOC=$(INSTLOC)
 	$(MAKE) clients INSTLOC=$(INSTLOC) CLIENTSINSTLOC=$(CLIENTSINSTLOC)
 endef
 endif
@@ -348,7 +347,6 @@ CLIENTS_HOME_DIR=$(BLD_HOME)
 endif
 CLIENTSINSTLOC=$(CLIENTS_HOME_DIR)/cloudberry-clients
 CLIENTSINSTLOC_BIN=$(CLIENTSINSTLOC)/bin
-CLIENTSINSTLOC_BINEXT=$(CLIENTSINSTLOC)/bin/ext/
 CLIENTSINSTLOC_EXT=$(CLIENTSINSTLOC)/ext
 CLIENTSINSTLOC_LIB=$(CLIENTSINSTLOC)/lib
 CLIENTSINSTLOC_LIB_PWARE=$(CLIENTSINSTLOC)/lib/pware
@@ -372,15 +370,6 @@ define tmpCLIENTS_FILESET_BIN
 endef
 CLIENTS_FILESET_BIN = $(strip $(tmpCLIENTS_FILESET_BIN))
 
-# pg.py, pgdb.py, _pg.so are from pygresql which does not install into a single module
-define tmpCLIENTS_FILESET_BINEXT
-	pg.py
-	pgdb.py
-	_pg*.so
-	yaml
-endef
-CLIENTS_FILESET_BINEXT = $(strip $(tmpCLIENTS_FILESET_BINEXT))
-
 BLD_PYTHON_FILESET=.
 
 BLD_OS:=$(shell uname -s)
@@ -418,8 +407,6 @@ else
 	# ---- copy GPDB fileset ----
 	mkdir -p $(CLIENTSINSTLOC_BIN)
 	(cd $(INSTLOC)/bin/ && $(TAR) cf - $(CLIENTS_FILESET_BIN)) | (cd $(CLIENTSINSTLOC_BIN)/ && $(TAR) xpf -)$(check_pipe_for_errors)
-	mkdir -p $(CLIENTSINSTLOC_BINEXT)
-	(cd $(GPMGMT)/bin/ext/ && $(TAR) cf - $(CLIENTS_FILESET_BINEXT)) | (cd $(CLIENTSINSTLOC_BINEXT)/ && $(TAR) xpf -)$(check_pipe_for_errors)
 ifneq "$(PYTHONHOME)" ""
 	mkdir -p $(CLIENTSINSTLOC_EXT)/python
 	(cd $(PYTHONHOME) && $(TAR) cf - $(BLD_PYTHON_FILESET)) | (cd $(CLIENTSINSTLOC_EXT)/python/ && $(TAR) xpf -)$(check_pipe_for_errors)

gpAux/client/install/src/windows/CreatePackage.bat

@@ -11,8 +11,6 @@ type nul > %GPDB_INSTALL_PATH%\bin\gppylib\__init__.py
 copy ..\..\..\..\..\gpMgmt\bin\gppylib\gpversion.py %GPDB_INSTALL_PATH%\bin\gppylib\
 perl -pi.bak -e "s,\$Revision\$,%VERSION%," %GPDB_INSTALL_PATH%\bin\gpload.py
 copy ..\..\..\..\..\gpMgmt\bin\gpload.bat %GPDB_INSTALL_PATH%\bin
-for %%f in (..\..\..\..\..\gpMgmt\bin\pythonSrc\ext\PyYAML-*.tar.gz) do tar -xf %%f
-for /D %%d in (PyYAML-*) do copy %%d\lib\yaml\* %GPDB_INSTALL_PATH%\lib\python\yaml
 perl -p -e "s,__VERSION_PLACEHOLDER__,%VERSION%," greenplum-clients.wxs > greenplum-clients-%VERSION%.wxs
 candle.exe -nologo greenplum-clients-%VERSION%.wxs -out greenplum-clients-%VERSION%.wixobj -dSRCDIR=%GPDB_INSTALL_PATH% -dVERSION=%VERSION%
 light.exe -nologo -sval greenplum-clients-%VERSION%.wixobj -out greenplum-clients-x86_64.msi

gpAux/client/install/src/windows/greenplum-clients.wxs

@@ -1139,35 +1139,6 @@ If you want to review or change any of your installation settings, click Back. C
               </Directory>
             </Directory>
             <Directory Id="TOPLEVELlib" Name="lib">
-              <Directory Id="PYTHONlib" Name="python">
-                <Component Id="PYGRESQLlib" DiskId="1" Guid="CDFE5977-8F11-43AF-95B7-374510FA872D">
-                  <File Id="__init__.py" Name="__init__.py" Source="$(var.SRCDIR)\lib\python\__init__.py" />
-                  <File Id="_pg.pyd" Name="_pg.pyd" Source="$(var.SRCDIR)\lib\python\_pg.pyd" />
-                  <File Id="pg.py" Name="pg.py" Source="$(var.SRCDIR)\lib\python\pg.py" />
-                  <File Id="pgdb.py" Name="pgdb.py" Source="$(var.SRCDIR)\lib\python\pgdb.py" />
-                </Component>
-                <Directory Id="YAMLlib" Name="yaml">
-                  <Component Id="YAMLlib" DiskId="1" Guid="77EAA4D5-888B-4633-BE68-8C81186EAA83">
-                    <File Id="composer.py" Name="composer.py" Source="$(var.SRCDIR)\lib\python\yaml\composer.py" />
-                    <File Id="constructor.py" Name="constructor.py" Source="$(var.SRCDIR)\lib\python\yaml\constructor.py" />
-                    <File Id="cyaml.py" Name="cyaml.py" Source="$(var.SRCDIR)\lib\python\yaml\cyaml.py" />
-                    <File Id="dumper.py" Name="dumper.py" Source="$(var.SRCDIR)\lib\python\yaml\dumper.py" />
-                    <File Id="emitter.py" Name="emitter.py" Source="$(var.SRCDIR)\lib\python\yaml\emitter.py" />
-                    <File Id="error.py" Name="error.py" Source="$(var.SRCDIR)\lib\python\yaml\error.py" />
-                    <File Id="events.py" Name="events.py" Source="$(var.SRCDIR)\lib\python\yaml\events.py" />
-                    <File Id="loader.py" Name="loader.py" Source="$(var.SRCDIR)\lib\python\yaml\loader.py" />
-                    <File Id="nodes.py" Name="nodes.py" Source="$(var.SRCDIR)\lib\python\yaml\nodes.py" />
-                    <File Id="parser.py" Name="parser.py" Source="$(var.SRCDIR)\lib\python\yaml\parser.py" />
-                    <File Id="reader.py" Name="reader.py" Source="$(var.SRCDIR)\lib\python\yaml\reader.py" />
-                    <File Id="representer.py" Name="representer.py" Source="$(var.SRCDIR)\lib\python\yaml\representer.py" />
-                    <File Id="resolver.py" Name="resolver.py" Source="$(var.SRCDIR)\lib\python\yaml\resolver.py" />
-                    <File Id="scanner.py" Name="scanner.py" Source="$(var.SRCDIR)\lib\python\yaml\scanner.py" />
-                    <File Id="serializer.py" Name="serializer.py" Source="$(var.SRCDIR)\lib\python\yaml\serializer.py" />
-                    <File Id="tokens.py" Name="tokens.py" Source="$(var.SRCDIR)\lib\python\yaml\tokens.py" />
-                    <File Id="__init__.py.yaml" Name="__init__.py" Source="$(var.SRCDIR)\lib\python\yaml\__init__.py" />
-                  </Component>
-                </Directory>
-              </Directory>
             </Directory>
           </Directory>
         </Directory>
@@ -1179,8 +1150,6 @@ If you want to review or change any of your installation settings, click Back. C
       <Feature Id="psql" Level="1" Title="Apache Cloudberry client command line tools" Description="Apache Cloudberry client command line tools" AllowAdvertise="no" Display="expand">
         <ComponentRef Id="TOPLEVELbin" />
         <ComponentRef Id="GPPYlib" />
-        <ComponentRef Id="PYGRESQLlib" />
-        <ComponentRef Id="YAMLlib" />
       </Feature>
     </Feature>
   </Product>

gpMgmt/Makefile

@@ -18,17 +18,6 @@ install: generate_greenplum_path_file
 	if [ -e bin/ext/__init__.py ]; then \
 	    cp -rp bin/ext/__init__.py $(DESTDIR)$(prefix)/lib/python ; \
 	fi
-	if [ -e bin/ext/psutil ]; then \
-	    cp -rp bin/ext/psutil $(DESTDIR)$(prefix)/lib/python ; \
-	fi
-	if [ -e bin/ext/pgdb.py ]; then \
-	    cp -rp bin/ext/pgdb.py $(DESTDIR)$(prefix)/lib/python && \
-	    cp -rp bin/ext/pg.py $(DESTDIR)$(prefix)/lib/python && \
-	    cp -rp bin/ext/_pg*.so $(DESTDIR)$(prefix)/lib/python ; \
-	fi
-	if [ -e bin/ext/yaml ]; then \
-	    cp -rp bin/ext/yaml $(DESTDIR)$(prefix)/lib/python ; \
-	fi
 
 clean distclean:
 	$(MAKE) -C bin $@

gpMgmt/bin/Makefile

@@ -70,63 +70,19 @@ PYLIB_DIR=$(SRC)/ext
 core:
 	python3 gpconfig_modules/parse_guc_metadata.py $(DESTDIR)$(prefix)
 
-ifeq ($(with_pythonsrc_ext), yes)
-install: installdirs installprograms core psutil pygresql pyyaml
-else
 install: installdirs installprograms core
-endif
 
 #
 # Python Libraries
 #
 
-#
-# PyGreSQL
-#
-PYGRESQL_VERSION=5.2
-PYGRESQL_DIR=PyGreSQL-$(PYGRESQL_VERSION)
-pygresql:
-	@echo "--- PyGreSQL"
-	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYGRESQL_DIR).tar.gz
-	cd $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/ && PATH=$(DESTDIR)$(bindir):$$PATH LDFLAGS='$(LDFLAGS) $(PYGRESQL_LDFLAGS)' python3 setup.py build
-	cp -r $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build/lib*-3*/* $(PYLIB_DIR)/
-
-
-#
-# PSUTIL
-#
-PSUTIL_VERSION=5.7.0
-PSUTIL_DIR=psutil-$(PSUTIL_VERSION)
-
-psutil:
-	@echo "--- psutil"
-ifeq "$(findstring $(BLD_ARCH),aix7_ppc_64 )" ""
-	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PSUTIL_DIR).tar.gz
-	cd $(PYLIB_SRC_EXT)/$(PSUTIL_DIR)/ && env -u CC python3 setup.py build
-	cp -r $(PYLIB_SRC_EXT)/$(PSUTIL_DIR)/build/lib.*/psutil $(PYLIB_DIR)
-endif
-
-#
-# PYYAML
-#
-PYYAML_VERSION=5.3.1
-PYYAML_DIR=PyYAML-$(PYYAML_VERSION)
-
-pyyaml:
-	@echo "--- pyyaml"
-	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYYAML_DIR).tar.gz
-	cd $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/ && env -u CC python3 setup.py build
-	cp -r $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/build/lib*-3*/* $(PYLIB_DIR)/
-
 #
 # PYLINT
 #
 
 PYLINT_VERSION=0.21.0
 PYLINT_DIR=pylint-$(PYLINT_VERSION)
 PYLINT_PYTHONPATH=$(PYLIB_DIR):$(PYLIB_SRC_EXT)/$(PYLINT_DIR)/build/lib/
-MOCK_VERSION=1.0.1
-MOCK_DIR=mock-$(MOCK_VERSION)
 SETUP_TOOLS_VERSION=36.6.0
 PARSE_VERSION=1.8.2
 SETUP_TOOLS_DIR=setuptools-$(SETUP_TOOLS_VERSION)
@@ -135,8 +91,6 @@ PYTHONSRC_INSTALL=$(PYLIB_SRC_EXT)/install
 PYTHON_VERSION=$(shell python3 -c "import sys; print ('%s.%s' % (sys.version_info[0:2]))")
 PYTHONSRC_INSTALL_SITE=$(PYLIB_SRC_EXT)/install/lib/python$(PYTHON_VERSION)/site-packages
 PYTHONSRC_INSTALL_PYTHON_PATH=$(PYTHONPATH):$(PYTHONSRC_INSTALL_SITE)
-# TODO: mock-1.0.1-py2.6.egg package should be updated.
-MOCK_BIN=$(PYTHONSRC_INSTALL)/lib/python$(PYTHON_VERSION)/site-packages/mock-1.0.1-py2.6.egg
 UBUNTU_PLATFORM=$(shell if lsb_release -a 2>/dev/null | grep -q 'Ubuntu' ; then echo "Ubuntu"; fi)
 
 pylint:
@@ -145,17 +99,6 @@ pylint:
 	@cd $(PYLIB_SRC_EXT)/$(PYLINT_DIR)/ && python3 setup.py build 1> /dev/null
 	@touch $(PYLIB_SRC_EXT)/$(PYLINT_DIR)/build/lib/__init__.py
 
-$(MOCK_BIN):
-	@echo "--- mock for platform $(UBUNTU_PLATFORM)"
-	@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
-       pip3 install mock;\
-     else\
-       mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
-	   cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(MOCK_DIR).tar.gz && \
-	   cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
-	   PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
-	 fi;
-
 PYTHON_FILES=`grep -l --exclude=Makefile --exclude=gplogfilter "/bin/env python3" *`\
 			 `grep -l "/bin/env python3" $(SRC)/../sbin/*`\
 			 `find ./gppylib -name "*.py"`\
@@ -167,7 +110,7 @@ checkcode: pylint
 	@echo -n "pylint_score=" > $(SRC)/../pylint_score.properties
 	@grep "Your code has been rated at" $(SRC)/../pylint.txt | sed -e "s|Your .* \(.*\)/.*|\1|" >> $(SRC)/../pylint_score.properties
 
-check: $(MOCK_BIN)
+check:
 	@echo "Running pure unit and also "unit" tests that require cluster to be up..."
 	@TMPDIR=/tmp PYTHONPATH=$(SERVER_SRC):$(SERVER_SBIN):$(PYTHONPATH):$(PYTHONSRC_INSTALL_PYTHON_PATH):$(SRC)/ext:$(SBIN_DIR):$(LIB_DIR):$(PYLIB_DIR)/mock-1.0.1 \
 	gppylib/gpunit discover --verbose -s $(SRC)/gppylib -p "test_unit*.py" 2> $(SRC)/../gpMgmt_testunit_results.log 1> $(SRC)/../gpMgmt_testunit_output.log
@@ -190,8 +133,6 @@ installcheck: installcheck-bash
 
 clean distclean:
 	rm -rf $(PYLIB_SRC_EXT)/$(PYLINT_DIR)
-	rm -rf $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build
-	rm -rf $(PYLIB_SRC)/$(PYGRESQL_DIR)/build
 	rm -rf *.pyc
 	rm -f analyzedbc gpactivatestandbyc gpaddmirrorsc gpcheckcatc \
 		  gpcheckperfc gpcheckresgroupimplc gpchecksubnetcfgc gpconfigc \