From 39c480d416caa8cda936579cc7b0336e896b5844 Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Thu, 28 Jan 2021 13:35:33 +0100 Subject: [PATCH 1/8] NGINX systemd open file limit Allowing the nginx webserver a higher number of open files. --- scripts/helper_functions.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/helper_functions.sh b/scripts/helper_functions.sh index 0f1da129..3211f225 100644 --- a/scripts/helper_functions.sh +++ b/scripts/helper_functions.sh @@ -171,6 +171,9 @@ function setup_moodle_mount_dependency_for_systemd_service cat < $systemdSvcOverrideFilePath [Unit] After=moodle.mount + +[Service] +LimitNOFILE=100000 EOF systemctl daemon-reload fi From f86ad99a12e22c37401fe489e864271cf86c404b Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Thu, 28 Jan 2021 14:06:24 +0100 Subject: [PATCH 2/8] Improving the installation script logging Making sure we are capturing all of the commands output for our setup script. --- scripts/setup_webserver.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index ca833f1b..461696bd 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -22,6 +22,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. set -ex +echo "### Script Start `date`###" moodle_on_azure_configs_json_path=${1} @@ -52,6 +53,9 @@ echo $phpVersion >> /tmp/vars.txt check_fileServerType_param $fileServerType { + set -ex + echo "### Function Start `date`###" + # make sure the system does automatic update sudo apt-get -y update sudo apt-get -y install unattended-upgrades @@ -682,4 +686,5 @@ EOF systemctl daemon-reload service varnish restart -} > /tmp/setup.log + echo "### Script End `date`###" +} 2>&1 | tee /tmp/setup.log From 6ca3e2b6a3bb2c4711f6e877367b8a4a962d935b Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Thu, 28 Jan 2021 14:27:20 +0100 Subject: [PATCH 3/8] Consolidating apt package installation In order to decrease the amount of time we are taking for installating packages we've consolidated it in a single command adding quiet options to the apt command. --- scripts/setup_webserver.sh | 100 +++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 37 deletions(-) diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index 461696bd..ecf8f900 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -45,10 +45,6 @@ echo $nfsByoIpExportPath >> /tmp/vars.txt echo $htmlLocalCopySwitch >> /tmp/vars.txt echo $phpVersion >> /tmp/vars.txt -# downloading and updating php packages from the repository -# sudo dpkg --configure –a - sudo add-apt-repository ppa:ondrej/php -y > /dev/null 2>&1 - sudo apt-get update > /dev/null 2>&1 check_fileServerType_param $fileServerType @@ -56,57 +52,87 @@ check_fileServerType_param $fileServerType set -ex echo "### Function Start `date`###" - # make sure the system does automatic update - sudo apt-get -y update - sudo apt-get -y install unattended-upgrades - - # install pre-requisites - # sudo apt-get -y install python-software-properties unzip rsyslog - sudo apt-get -y install software-properties-common - sudo apt-get -y install unzip - sudo apt-get -y install rsyslog - sudo apt-get -y install postgresql-client mysql-client git + # add azure-cli repository + curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null + AZ_REPO=$(lsb_release -cs) + echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list + + # add PHP-FPM repository + add-apt-repository ppa:ondrej/php -y > /dev/null 2>&1 + + apt-get -qq -o=Dpkg::Use-Pty=0 update + + # install pre-requisites including VARNISH and PHP-FPM + export DEBIAN_FRONTEND=noninteractive + apt-get --yes \ + --no-install-recommends \ + -qq -o=Dpkg::Use-Pty=0 \ + -o Dpkg::Options::="--force-confdef" \ + -o Dpkg::Options::="--force-confold" \ + install \ + azure-cli \ + ca-certificates \ + curl \ + apt-transport-https \ + lsb-release gnupg \ + software-properties-common \ + unzip \ + rsyslog \ + postgresql-client \ + mysql-client \ + git \ + unattended-upgrades \ + tuned \ + varnish \ + php$phpVersion \ + php$phpVersion-cli \ + php$phpVersion-curl \ + php$phpVersion-zip \ + php-pear \ + php$phpVersion-mbstring \ + mcrypt \ + php$phpVersion-dev \ + graphviz \ + aspell \ + php$phpVersion-soap \ + php$phpVersion-json \ + php$phpVersion-redis \ + php$phpVersion-bcmath \ + php$phpVersion-gd \ + php$phpVersion-pgsql \ + php$phpVersion-mysql \ + php$phpVersion-xmlrpc \ + php$phpVersion-intl \ + php$phpVersion-xml \ + php$phpVersion-bz2 + + # install azcopy + wget -q -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1 && mv ./azcopy /usr/bin/ if [ $fileServerType = "gluster" ]; then #configure gluster repository & install gluster client - sudo add-apt-repository ppa:gluster/glusterfs-3.10 -y - sudo apt-get -y update - sudo apt-get -y install glusterfs-client + add-apt-repository ppa:gluster/glusterfs-3.10 -y + apt-get -y update + apt-get -y -qq -o=Dpkg::Use-Pty=0 install glusterfs-client elif [ "$fileServerType" = "azurefiles" ]; then - sudo apt-get -y install cifs-utils + apt-get -y -qq -o=Dpkg::Use-Pty=0 install cifs-utils fi - - # install the base stack - # passing php versions $phpVersion - sudo apt-get -y install varnish php$phpVersion php$phpVersion-cli php$phpVersion-curl php$phpVersion-zip php-pear php$phpVersion-mbstring php$phpVersion-dev mcrypt - - # if webservertype is nginx then apache2 will be masked. - # service=apache2 - # if [ "$webServerType" = "nginx" ]; then - # if [ $(ps -ef | grep -v grep | grep $service | wc -l) > 0 ]; then - # echo “Stop the $service!!!” - # sudo systemctl stop $service - # sudo systemctl mask $service - # fi - # fi if [ "$webServerType" = "nginx" -o "$httpsTermination" = "VMSS" ]; then - sudo apt-get -y install nginx + apt-get --yes -qq -o=Dpkg::Use-Pty=0 install nginx fi if [ "$webServerType" = "apache" ]; then # install apache pacakges - sudo apt-get -y install apache2 libapache2-mod-php + apt-get --yes -qq -o=Dpkg::Use-Pty=0 install apache2 libapache2-mod-php else # for nginx-only option - sudo apt-get -y install php$phpVersion-fpm + apt-get --yes -qq -o=Dpkg::Use-Pty=0 install php$phpVersion-fpm fi # Moodle requirements - sudo apt-get install -y graphviz aspell php$phpVersion-soap php$phpVersion-json php$phpVersion-redis php$phpVersion-bcmath php$phpVersion-gd php$phpVersion-pgsql php$phpVersion-mysql php$phpVersion-xmlrpc php$phpVersion-intl php$phpVersion-xml php$phpVersion-bz2 if [ "$dbServerType" = "mssql" ]; then install_php_mssql_driver - fi # PHP Version From 9e752987ca1825a38da6b4a2833f60a8b0d4a09c Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Thu, 28 Jan 2021 14:44:18 +0100 Subject: [PATCH 4/8] Tuning linux kernel parameters Adding network kernel parameters and enabling tuned with the throughput-performance profile. --- scripts/setup_webserver.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index ecf8f900..69204cc5 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -109,6 +109,26 @@ check_fileServerType_param $fileServerType # install azcopy wget -q -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1 && mv ./azcopy /usr/bin/ + # kernel settings + cat < /etc/sysctl.d/99-network-performance.conf +net.core.somaxconn = 65536 +net.core.netdev_max_backlog = 5000 +net.core.rmem_max = 16777216 +net.core.wmem_max = 16777216 +net.ipv4.tcp_wmem = 4096 12582912 16777216 +net.ipv4.tcp_rmem = 4096 12582912 16777216 +net.ipv4.route.flush = 1 +net.ipv4.tcp_max_syn_backlog = 8096 +net.ipv4.tcp_tw_reuse = 1 +net.ipv4.ip_local_port_range = 10240 65535 +EOF + # apply the new kernel settings + sysctl -p /etc/sysctl.d/99-network-performance.conf + + # configuring tuned for throughput-performance + systemctl enable tuned + tuned-adm profile throughput-performance + if [ $fileServerType = "gluster" ]; then #configure gluster repository & install gluster client add-apt-repository ppa:gluster/glusterfs-3.10 -y From da47c6b6353af0e5fbf886621b9111fb5a90e2a9 Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Thu, 28 Jan 2021 14:45:57 +0100 Subject: [PATCH 5/8] Masking last two vcpu cores using irqbalance Allowing (mostly) network irqs to be scheduled on the last two vcpu cores. --- scripts/setup_webserver.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index 69204cc5..b0b53363 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -125,6 +125,13 @@ EOF # apply the new kernel settings sysctl -p /etc/sysctl.d/99-network-performance.conf + # scheduling IRQ interrupts on the last two cores of the cpu + # masking 0011 or 00000011 the result will always be 3 echo "obase=16;ibase=2;0011" | bc | tr '[:upper:]' '[:lower:]' + if [ -f /etc/default/irqbalance ]; then + sed -i "s/\#IRQBALANCE_BANNED_CPUS\=/IRQBALANCE_BANNED_CPUS\=3/g" /etc/default/irqbalance + systemctl restart irqbalance.service + fi + # configuring tuned for throughput-performance systemctl enable tuned tuned-adm profile throughput-performance From adcea9672af103ec222213672ed453175d780d72 Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Thu, 28 Jan 2021 14:49:56 +0100 Subject: [PATCH 6/8] Tuning varnish daemon settings Increasing allocated memory to 4096MB, increasing the minimum number of threads to 1000 and lowering the delay for creating a thread. --- scripts/setup_webserver.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index b0b53363..cf3a1d8d 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -478,8 +478,8 @@ EOF sudo service apache2 restart fi - # Configure varnish startup for 16.04 - VARNISHSTART="ExecStart=\/usr\/sbin\/varnishd -j unix,user=vcache -F -a :80 -T localhost:6082 -f \/etc\/varnish\/moodle.vcl -S \/etc\/varnish\/secret -s malloc,1024m -p thread_pool_min=200 -p thread_pool_max=4000 -p thread_pool_add_delay=2 -p timeout_linger=100 -p timeout_idle=30 -p send_timeout=1800 -p thread_pools=4 -p http_max_hdr=512 -p workspace_backend=512k" + # Configure varnish startup for 18.04 + VARNISHSTART="ExecStart=\/usr\/sbin\/varnishd -j unix,user=vcache -F -a :80 -T localhost:6082 -f \/etc\/varnish\/moodle.vcl -S \/etc\/varnish\/secret -s malloc,4096m -p thread_pool_min=1000 -p thread_pool_max=4000 -p thread_pool_add_delay=0.1 -p timeout_linger=10 -p timeout_idle=30 -p send_timeout=1800 -p thread_pools=2 -p http_max_hdr=512 -p workspace_backend=512k" sed -i "s/^ExecStart.*/${VARNISHSTART}/" /lib/systemd/system/varnish.service # Configure varnish VCL for moodle From c25e06c12efde9886efa807cdf12d2b092153060 Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Thu, 28 Jan 2021 15:36:15 +0100 Subject: [PATCH 7/8] Replacing rsync with azcopy In order to speed up the copying process rsync code was replaced by azcopy. --- scripts/setup_webserver.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index cf3a1d8d..8356b3c4 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -272,8 +272,24 @@ EOF htmlRootDir="/moodle/html/moodle" if [ "$htmlLocalCopySwitch" = "true" ]; then mkdir -p /var/www/html - rsync -av --delete /moodle/html/moodle /var/www/html + ACCOUNT_KEY="$storageAccountKey" + NAME="$storageAccountName" + END=`date -u -d "60 minutes" '+%Y-%m-%dT%H:%M:00Z'` htmlRootDir="/var/www/html/moodle" + + sas=$(az storage share generate-sas \ + -n moodle \ + --account-key $ACCOUNT_KEY \ + --account-name $NAME \ + --https-only \ + --permissions lr \ + --expiry $END -o tsv) + + export AZCOPY_CONCURRENCY_VALUE='48' + export AZCOPY_BUFFER_GB='4' + + azcopy --log-level ERROR copy "https://$NAME.file.core.windows.net/moodle/html/moodle/*?$sas" $htmlRootDir --recursive + chown www-data:www-data -R $htmlRootDir && sync setup_html_local_copy_cron_job fi From 8df2fbbe7739260a8c975ac7a07f4930ddec7132 Mon Sep 17 00:00:00 2001 From: Adrian Joian Date: Thu, 28 Jan 2021 15:39:47 +0100 Subject: [PATCH 8/8] NGINX and PHP-FPM general tuning Adding tuning options for nginx and php-fpm in order to increase performance under load. --- scripts/setup_webserver.sh | 119 +++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 43 deletions(-) diff --git a/scripts/setup_webserver.sh b/scripts/setup_webserver.sh index 8356b3c4..cb9fa2b6 100644 --- a/scripts/setup_webserver.sh +++ b/scripts/setup_webserver.sh @@ -46,6 +46,7 @@ echo $htmlLocalCopySwitch >> /tmp/vars.txt echo $phpVersion >> /tmp/vars.txt + check_fileServerType_param $fileServerType { @@ -203,13 +204,17 @@ EOF # Build nginx config cat < /etc/nginx/nginx.conf user www-data; -worker_processes 2; +worker_processes auto; pid /run/nginx.pid; events { - worker_connections 2048; + worker_connections 8192; + multi_accept on; + use epoll; } +worker_rlimit_nofile 100000; + http { sendfile on; @@ -229,13 +234,18 @@ http { access_log /var/log/nginx/access.log; error_log /var/log/nginx/error.log; + open_file_cache max=20000 inactive=20s; + open_file_cache_valid 30s; + open_file_cache_min_uses 2; + open_file_cache_errors on; + set_real_ip_from 127.0.0.1; real_ip_header X-Forwarded-For; #upgrading to TLSv1.2 and droping 1 & 1.1 - ssl_protocols TLSv1.2; - #ssl_prefer_server_ciphers on; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers off; #adding ssl ciphers - ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; gzip on; gzip_disable "msie6"; @@ -297,13 +307,16 @@ EOF # Configure nginx/https cat <> /etc/nginx/sites-enabled/${siteFQDN}.conf server { - listen 443 ssl; + listen 443 ssl http2; root ${htmlRootDir}; - index index.php index.html index.htm; + index index.php index.html index.htm; ssl on; ssl_certificate /moodle/certs/nginx.crt; ssl_certificate_key /moodle/certs/nginx.key; + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; # Log to syslog error_log syslog:server=localhost,facility=local1,severity=error,tag=moodle; @@ -326,12 +339,18 @@ server { proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; proxy_pass http://localhost:80; + proxy_next_upstream error timeout http_502 http_504; proxy_connect_timeout 3600; proxy_send_timeout 3600; proxy_read_timeout 3600; send_timeout 3600; } } + +upstream backend { + server unix:/run/php/php${PhpVer}-fpm.sock fail_timeout=1s; + server unix:/run/php/php${PhpVer}-fpm-backup.sock backup; +} EOF fi @@ -382,8 +401,8 @@ EOF fastcgi_buffers 16 16k; fastcgi_buffer_size 32k; - fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name; - fastcgi_pass unix:/run/php/php${PhpVer}-fpm.sock; + fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name; + fastcgi_pass backend; fastcgi_read_timeout 3600; fastcgi_index index.php; include fastcgi_params; @@ -391,7 +410,7 @@ EOF } EOF - fi # if [ "$webServerType" = "nginx" ]; + fi if [ "$webServerType" = "apache" ]; then # Configure Apache/php @@ -451,8 +470,8 @@ EOF sed -i "s/;opcache.save_comments.*/opcache.save_comments = 1/" $PhpIni sed -i "s/;opcache.enable_file_override.*/opcache.enable_file_override = 0/" $PhpIni sed -i "s/;opcache.enable.*/opcache.enable = 1/" $PhpIni - sed -i "s/;opcache.memory_consumption.*/opcache.memory_consumption = 256/" $PhpIni - sed -i "s/;opcache.max_accelerated_files.*/opcache.max_accelerated_files = 8000/" $PhpIni + sed -i "s/;opcache.memory_consumption.*/opcache.memory_consumption = 512/" $PhpIni + sed -i "s/;opcache.max_accelerated_files.*/opcache.max_accelerated_files = 20000/" $PhpIni # Remove the default site. Moodle is the only site we want rm -f /etc/nginx/sites-enabled/default @@ -467,33 +486,6 @@ EOF sudo service nginx restart fi - if [ "$webServerType" = "nginx" ]; then - # fpm config - overload this - cat < /etc/php/${PhpVer}/fpm/pool.d/www.conf -[www] -user = www-data -group = www-data -listen = /run/php/php${PhpVer}-fpm.sock -listen.owner = www-data -listen.group = www-data -pm = dynamic -pm.max_children = 3000 -pm.start_servers = 20 -pm.min_spare_servers = 20 -pm.max_spare_servers = 30 -EOF - - # Restart fpm - service php${PhpVer}-fpm restart - fi - - if [ "$webServerType" = "apache" ]; then - if [ "$htmlLocalCopySwitch" != "true" ]; then - setup_moodle_mount_dependency_for_systemd_service apache2 || exit 1 - fi - sudo service apache2 restart - fi - # Configure varnish startup for 18.04 VARNISHSTART="ExecStart=\/usr\/sbin\/varnishd -j unix,user=vcache -F -a :80 -T localhost:6082 -f \/etc\/varnish\/moodle.vcl -S \/etc\/varnish\/secret -s malloc,4096m -p thread_pool_min=1000 -p thread_pool_max=4000 -p thread_pool_add_delay=0.1 -p timeout_linger=10 -p timeout_idle=30 -p send_timeout=1800 -p thread_pools=2 -p http_max_hdr=512 -p workspace_backend=512k" sed -i "s/^ExecStart.*/${VARNISHSTART}/" /lib/systemd/system/varnish.service @@ -745,15 +737,56 @@ EOF # This code is stop apache2 which is installing in 18.04 service=apache2 if [ "$webServerType" = "nginx" ]; then - if [ $(ps -ef | grep -v grep | grep $service | wc -l) > 0 ]; then + if pgrep -x "$service" >/dev/null + then echo “Stop the $service!!!” - sudo systemctl stop $service - sudo systemctl mask $service + systemctl stop $service + else + systemctl mask $service fi fi # Restart Varnish systemctl daemon-reload - service varnish restart + systemctl restart varnish + + if [ "$webServerType" = "nginx" ]; then + # fpm config - overload this + cat < /etc/php/${PhpVer}/fpm/pool.d/www.conf +[www] +user = www-data +group = www-data +listen = /run/php/php${PhpVer}-fpm.sock +listen.owner = www-data +listen.group = www-data +pm = static +pm.max_children = 32 +pm.start_servers = 32 +pm.max_requests = 300000 +EOF + +cat < /etc/php/${PhpVer}/fpm/pool.d/backup.conf +[backup] +user = www-data +group = www-data +listen = /run/php/php${PhpVer}-fpm-backup.sock +listen.owner = www-data +listen.group = www-data +pm = static +pm.max_children = 16 +pm.start_servers = 16 +pm.max_requests = 300000 +EOF + + # Restart fpm + service php${PhpVer}-fpm restart + fi + + if [ "$webServerType" = "apache" ]; then + if [ "$htmlLocalCopySwitch" != "true" ]; then + setup_moodle_mount_dependency_for_systemd_service apache2 || exit 1 + fi + service apache2 restart + fi echo "### Script End `date`###" } 2>&1 | tee /tmp/setup.log