From 4609e878b5980fa983e6897cdf658ae2f1f48007 Mon Sep 17 00:00:00 2001 From: Mounika M <91211334+mounikagunnam@users.noreply.github.com> Date: Tue, 18 Apr 2023 10:40:41 +0530 Subject: [PATCH 1/2] Create the Greenplum 7 README for Rocky8 to build OSS GPDB (#15329) Authored-by : Mounika M --- README.Linux.md | 64 +++++++++++++++++++++++++ README.Rhel-Rocky.bash | 103 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 README.Linux.md create mode 100755 README.Rhel-Rocky.bash diff --git a/README.Linux.md b/README.Linux.md new file mode 100644 index 00000000000..6cd89f7c17e --- /dev/null +++ b/README.Linux.md @@ -0,0 +1,64 @@ +## For RHEL/Rocky8: + + - For EL versions (> 8.0): + - Install git + ```bash + sudo yum install git + ``` + - Install dependencies using README.Rhel-Rocky.bash script. + ```bash + ./README.Rhel-Rocky.bash + ``` + +## For Ubuntu: + +- Install Dependencies + When you run the README.Ubuntu.bash script for dependencies, you will be asked to configure realm for kerberos. + You can enter any realm, since this is just for testing, and during testing, it will reconfigure a local server/client. + If you want to skip this manual configuration, use: + `export DEBIAN_FRONTEND=noninteractive` + + ```bash + sudo ./README.Ubuntu.bash + ``` + +- Ubuntu 18.04 and newer should have use gcc 7 or newer, but you can also enable gcc-7 on older versions of Ubuntu: + + ```bash + sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y + sudo apt-get update + sudo apt-get install -y gcc-7 g++-7 + ``` + +## Common Platform Tasks: + +Make sure that you add `/usr/local/lib` to `/etc/ld.so.conf`, +then run command `ldconfig`. +1. Create gpadmin and setup ssh keys + Either use: + + ```bash + # Requires gpdb clone to be named gpdb_src + gpdb_src/concourse/scripts/setup_gpadmin_user.bash + ``` + to create the gpadmin user and set up keys, + + OR + + manually create ssh keys so you can do ssh localhost without a password, e.g., + + ``` + ssh-keygen + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + chmod 600 ~/.ssh/authorized_keys + ``` + +1. Verify that you can ssh to your machine name without a password. + + ```bash + ssh # e.g., ssh briarwood (You can use `hostname` to get the hostname of your machine.) + ``` + +1. Set up your system configuration by following the installation guide on [docs.greenplum.org](https://docs.greenplum.org) + + diff --git a/README.Rhel-Rocky.bash b/README.Rhel-Rocky.bash new file mode 100755 index 00000000000..ec23ea84503 --- /dev/null +++ b/README.Rhel-Rocky.bash @@ -0,0 +1,103 @@ +#!/bin/bash + +# Install needed packages. Please add to this list if you discover additional prerequisites +sudo yum group install -y "Development Tools" +INSTALL_PKGS="apr-devel bison bzip2-devel cmake3 epel-release flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libxml2-devel libzstd-devel openssl-devel python39 python39-devel python39-psutil python3-pip perl-ExtUtils-MakeMaker.noarch perl-ExtUtils-Embed.noarch readline-devel rsync xerces-c-devel zlib-devel" + +for i in $INSTALL_PKGS; do + sudo yum install -y $i +done + +# Needed for pygresql, or you can source greenplum_path.sh after compiling database and installing python-dependencies then +sudo yum install -y postgresql +sudo yum install -y postgresql-devel + +pip3.9 install pygresql +pip3.9 install -r python-dependencies.txt + +#Remove the python3 softlink because on Rocky8 its pointing to python3.6 by default. We need python3.9 for gpdb7. +sudo rm -rf /usr/bin/python3 +sudo ln -s /usr/bin/python3.9 /usr/bin/python3 + +#For all Greenplum Database host systems running RHEL, CentOs or Rocky8, SELinux must either be Disabled or configured to allow unconfined access to Greenplum processes, directories, and the gpadmin user. +setenforce 0 +sudo tee -a /etc/selinux/config << EOF +SELINUX=disabled +EOF + +#To prevent SELinux-related SSH authentication denials that could occur even with SELinux deactivated +sudo tee -a /etc/sssd/sssd.conf << EOF +selinux_provider=none +EOF + +sudo systemctl stop firewalld.service + +#Configure kernel settings so the system is optimized for Greenplum Database. +sudo tee -a /etc/sysctl.d/10-gpdb.conf << EOF +kernel.msgmax = 65536 +kernel.msgmnb = 65536 +kernel.msgmni = 2048 +kernel.sem = 500 2048000 200 8192 +kernel.shmmni = 1024 +kernel.core_uses_pid = 1 +kernel.core_pattern=/var/core/core.%h.%t +kernel.sysrq = 1 +net.core.netdev_max_backlog = 2000 +net.core.rmem_max = 4194304 +net.core.wmem_max = 4194304 +net.core.rmem_default = 4194304 +net.core.wmem_default = 4194304 +net.ipv4.tcp_rmem = 4096 4224000 16777216 +net.ipv4.tcp_wmem = 4096 4224000 16777216 +net.core.optmem_max = 4194304 +net.core.somaxconn = 10000 +net.ipv4.ip_forward = 0 +net.ipv4.tcp_congestion_control = cubic +net.ipv4.tcp_tw_recycle = 0 +net.core.default_qdisc = fq_codel +net.ipv4.tcp_mtu_probing = 0 +net.ipv4.conf.all.arp_filter = 1 +net.ipv4.conf.default.accept_source_route = 0 +net.ipv4.ip_local_port_range = 10000 65535 +net.ipv4.tcp_max_syn_backlog = 4096 +net.ipv4.tcp_syncookies = 1 +net.ipv4.ipfrag_high_thresh = 41943040 +net.ipv4.ipfrag_low_thresh = 31457280 +net.ipv4.ipfrag_time = 60 +net.ipv4.ip_local_reserved_ports=65330 +vm.overcommit_memory = 2 +vm.overcommit_ratio = 95 +vm.swappiness = 10 +vm.dirty_expire_centisecs = 500 +vm.dirty_writeback_centisecs = 100 +vm.zone_reclaim_mode = 0 +EOF + +RAM_IN_KB=`cat /proc/meminfo | grep MemTotal | awk '{print $2}'` +RAM_IN_BYTES=$(($RAM_IN_KB*1024)) +echo "vm.min_free_kbytes = $(($RAM_IN_BYTES*3/100/1024))" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null +echo "kernel.shmall = $(($RAM_IN_BYTES/2/4096))" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null +echo "kernel.shmmax = $(($RAM_IN_BYTES/2))" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null +if [ $RAM_IN_BYTES -le $((64*1024*1024*1024)) ]; then + echo "vm.dirty_background_ratio = 3" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null + echo "vm.dirty_ratio = 10" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null +else + echo "vm.dirty_background_ratio = 0" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null + echo "vm.dirty_ratio = 0" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null + echo "vm.dirty_background_bytes = 1610612736 # 1.5GB" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null + echo "vm.dirty_bytes = 4294967296 # 4GB" | sudo tee -a /etc/sysctl.d/10-gpdb.conf > /dev/null +fi + +sudo sysctl -p + +sudo tee -a /etc/security/limits.d/10-nproc.conf << EOF +* soft nofile 524288 +* hard nofile 524288 +* soft nproc 131072 +* hard nproc 131072 +* soft core unlimited +EOF + + +ulimit -n 65536 65536 + From 1a07f288136748ed26ceb6120f5f76bf23eb2675 Mon Sep 17 00:00:00 2001 From: Xing Guo Date: Fri, 20 Oct 2023 09:52:09 +0800 Subject: [PATCH 2/2] [Doc] Add missing dependency libyaml-devel for Rocky Linux. (#16612) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add missing dependency libyaml-devel for Rocky Linux. --------- Co-authored-by: Xuebin Su Co-authored-by: Xuebin Su (苏学斌) <12034000+xuebinsu@users.noreply.github.com> --- README.Linux.md | 64 ------------------- .../README.Rhel-Rocky.bash | 12 ++-- 2 files changed, 5 insertions(+), 71 deletions(-) delete mode 100644 README.Linux.md rename README.Rhel-Rocky.bash => readmes/README.Rhel-Rocky.bash (93%) diff --git a/README.Linux.md b/README.Linux.md deleted file mode 100644 index 6cd89f7c17e..00000000000 --- a/README.Linux.md +++ /dev/null @@ -1,64 +0,0 @@ -## For RHEL/Rocky8: - - - For EL versions (> 8.0): - - Install git - ```bash - sudo yum install git - ``` - - Install dependencies using README.Rhel-Rocky.bash script. - ```bash - ./README.Rhel-Rocky.bash - ``` - -## For Ubuntu: - -- Install Dependencies - When you run the README.Ubuntu.bash script for dependencies, you will be asked to configure realm for kerberos. - You can enter any realm, since this is just for testing, and during testing, it will reconfigure a local server/client. - If you want to skip this manual configuration, use: - `export DEBIAN_FRONTEND=noninteractive` - - ```bash - sudo ./README.Ubuntu.bash - ``` - -- Ubuntu 18.04 and newer should have use gcc 7 or newer, but you can also enable gcc-7 on older versions of Ubuntu: - - ```bash - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update - sudo apt-get install -y gcc-7 g++-7 - ``` - -## Common Platform Tasks: - -Make sure that you add `/usr/local/lib` to `/etc/ld.so.conf`, -then run command `ldconfig`. -1. Create gpadmin and setup ssh keys - Either use: - - ```bash - # Requires gpdb clone to be named gpdb_src - gpdb_src/concourse/scripts/setup_gpadmin_user.bash - ``` - to create the gpadmin user and set up keys, - - OR - - manually create ssh keys so you can do ssh localhost without a password, e.g., - - ``` - ssh-keygen - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys - chmod 600 ~/.ssh/authorized_keys - ``` - -1. Verify that you can ssh to your machine name without a password. - - ```bash - ssh # e.g., ssh briarwood (You can use `hostname` to get the hostname of your machine.) - ``` - -1. Set up your system configuration by following the installation guide on [docs.greenplum.org](https://docs.greenplum.org) - - diff --git a/README.Rhel-Rocky.bash b/readmes/README.Rhel-Rocky.bash similarity index 93% rename from README.Rhel-Rocky.bash rename to readmes/README.Rhel-Rocky.bash index ec23ea84503..8b7dc2cb832 100755 --- a/README.Rhel-Rocky.bash +++ b/readmes/README.Rhel-Rocky.bash @@ -2,17 +2,15 @@ # Install needed packages. Please add to this list if you discover additional prerequisites sudo yum group install -y "Development Tools" -INSTALL_PKGS="apr-devel bison bzip2-devel cmake3 epel-release flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libxml2-devel libzstd-devel openssl-devel python39 python39-devel python39-psutil python3-pip perl-ExtUtils-MakeMaker.noarch perl-ExtUtils-Embed.noarch readline-devel rsync xerces-c-devel zlib-devel" +INSTALL_PKGS="apr-devel bison bzip2-devel cmake3 epel-release flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libxml2-devel libzstd-devel openssl-devel python39 python39-devel python39-psutil python3-pip perl-ExtUtils-MakeMaker.noarch perl-ExtUtils-Embed.noarch readline-devel rsync xerces-c-devel zlib-devel python3-psutil python3-pyyaml python3-psycopg2" -for i in $INSTALL_PKGS; do - sudo yum install -y $i -done +sudo yum install -y $INSTALL_PKGS -# Needed for pygresql, or you can source greenplum_path.sh after compiling database and installing python-dependencies then -sudo yum install -y postgresql +sudo yum --enablerepo=powertools install -y libyaml-devel + +sudo yum install -y postgresql sudo yum install -y postgresql-devel -pip3.9 install pygresql pip3.9 install -r python-dependencies.txt #Remove the python3 softlink because on Rocky8 its pointing to python3.6 by default. We need python3.9 for gpdb7.