From e05999f16ce94d6522e78ba13a3b2f412ba9b226 Mon Sep 17 00:00:00 2001 From: JedwardsEdwards Date: Mon, 22 Feb 2021 12:22:49 +0000 Subject: [PATCH 1/2] Update multiple hosts docs with config script for torq install script --- docs/MultipleHosts.md | 91 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/docs/MultipleHosts.md b/docs/MultipleHosts.md index 02b6f9939..38e28ad9e 100644 --- a/docs/MultipleHosts.md +++ b/docs/MultipleHosts.md @@ -7,6 +7,8 @@ Each environment will be split across two hosts. This will use the TorQ Finance ## Basic Installation For this example we will use the TorQ Installation Script. Information on this can be found here [TorQ Installation Script](http://www.aquaq.co.uk/q/torq-installation-script/) +The majority of the below can be implemented through a environment-specific config script supplied to the installtorqapp.sh script through an edit to the installlatest.sh script as described at the above link. + ## Host Split To split the stack across hosts we do so in the process.csv file. Processes on Host A: @@ -21,20 +23,20 @@ Processes on Host B: - RDB (With quote table) - Gateway -For this we will use the following process.csv: +For this we will use the following process.csv (the STACKENV string will be replaced with the appropriate environment name as part of the install): ``` host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd -${PRIMARYHOSTA},{KDBBASEPORT}+1,discovery,discovery1\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/discovery.q,1,,q -${PRIMARYHOSTA},{KDBBASEPORT},segmentedtickerplant,stp1\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQAPPHOME}/database.q -tplogdir ${KDBTPLOG},q -${PRIMARYHOSTA},{KDBBASEPORT}+2,rdbt,rdb1\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,180,,${KDBCODE}/processes/rdb.q,1,-parentproctype rdb,q -${PRIMARYHOSTB},{KDBBASEPORT}+2,rdbq,rdb2\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,180,,${KDBCODE}/processes/rdb.q,1,-parentproctype rdb,q -${PRIMARYHOSTA},{KDBBASEPORT}+3,hdb,hdb1\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,,q -${PRIMARYHOSTA},{KDBBASEPORT}+4,hdb,hdb2\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,,q -${PRIMARYHOSTA},{KDBBASEPORT}+5,wdb,wdb1\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q,1,,q -${PRIMARYHOSTA},{KDBBASEPORT}+6,sort,sort1\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q,1,-s -2,q -${PRIMARYHOSTB},{KDBBASEPORT}+7,gateway,gateway1\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,,4000,${KDBCODE}/processes/gateway.q,1,,q -${PRIMARYHOSTB},{KDBBASEPORT}+8,segmentedchainedtickerplant,sctp1\_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-parentproctype segmentedtickerplant,q -${PRIMARYHOSTA},{KDBBASEPORT}+9,feed,feed1\_STACKENV,,1,0,,,${KDBAPPCODE}/tick/feed.q,1,,q +${PRIMARYHOSTA},{KDBBASEPORT}+1,discovery,discovery1_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/discovery.q,1,,q +${PRIMARYHOSTA},{KDBBASEPORT},segmentedtickerplant,stp1_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQAPPHOME}/database.q -tplogdir ${KDBTPLOG},q +${PRIMARYHOSTA},{KDBBASEPORT}+2,rdbt,rdb1_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,180,,${KDBCODE}/processes/rdb.q,1,-parentproctype rdb,q +${PRIMARYHOSTB},{KDBBASEPORT}+2,rdbq,rdb2_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,180,,${KDBCODE}/processes/rdb.q,1,-parentproctype rdb,q +${PRIMARYHOSTA},{KDBBASEPORT}+3,hdb,hdb1_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,,q +${PRIMARYHOSTA},{KDBBASEPORT}+4,hdb,hdb2_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,60,4000,${KDBHDB},1,,q +${PRIMARYHOSTA},{KDBBASEPORT}+5,wdb,wdb1_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q,1,,q +${PRIMARYHOSTA},{KDBBASEPORT}+6,sort,sort1_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q,1,-s -2,q +${PRIMARYHOSTB},{KDBBASEPORT}+7,gateway,gateway1_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,1,,4000,${KDBCODE}/processes/gateway.q,1,,q +${PRIMARYHOSTB},{KDBBASEPORT}+8,segmentedchainedtickerplant,sctp1_STACKENV,${TORQAPPHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-parentproctype segmentedtickerplant,q +${PRIMARYHOSTA},{KDBBASEPORT}+9,feed,feed1_STACKENV,,1,0,,,${KDBAPPCODE}/tick/feed.q,1,,q ``` ## Table Split @@ -51,14 +53,15 @@ rdbq.q: .rdb.subscribeto:`quote; ``` -For the chained tickerplant we want it to create its own log. To do so we will update the following file. +For the chained tickerplant we want it to create its own log, and to subscribe to the correct tp name. To do so we will update the following file. segmentedchainedtickerplant.q: ``` +.sctp.tickerplantname:stp1_prod; .sctp.loggingmode:`create; ``` ## Gateway Connectivity -We want the gateways to connect to both sides. To do this we will add the other environment's RDBs and HDBs to the nontorqprocess.csv file: +We want the gateways to connect to both sides. To do this we will add the other environment's RDBs and HDBs to the nontorqprocess.csv file (the STACKENV_S string will be replaced as with the process.csv): ``` host,port,proctype,procname {SECONDARYHOSTA},6002,rdbt,rdb1_STACKENV_S @@ -68,7 +71,7 @@ host,port,proctype,procname ``` ## Environment Variables -To run this across multiple hosts we need to set the different host names and what environment the host we are on is. To do so I added the following to the setenv.sh file: +To run this across multiple hosts we need to set the different host names and what environment the host we are on is. Hence the following should be added to the setenv.sh script: ``` export PRIMARYHOSTA=prod.a export PRIMARYHOSTB=prod.b @@ -76,9 +79,61 @@ export SECONDARYHOSTA=dr.a export SECONDARYHOSTB=dr.b export STACKENV=prod export STACKENV_S=dr -sed -i -e 's/STACKENV/'$STACKENV'/g' ${TORQAPPHOME}/appconfig/process.csv -sed -i -e 's/STACKENV_S/'$STACKENV_S'/g' ${TORQAPPHOME}/appconfig/nontorqprocess.csv ``` -Replacing prod.a,prod.b,dr.a and dr.b with the actual hostnames or IPs of the hosts. For the DR hosts I then flipped the values so that for example PRIMARYHOSTA was dr.a and STACKENV was dr. +Replacing prod.a,prod.b,dr.a and dr.b with the actual hostnames or IPs of the hosts. + +## Environment-Specific Config Script + +The below should provide an example of a config script to implement the majority of the above for the production host. It assumes that both a process.csv and nontorqprocess.csv file exist: + +``` +# configurable variables +PRIMARYHOSTA=prod.a +PRIMARYHOSTB=prod.b +SECONDARYHOSTA=dr.a +SECONDARYHOSTB=dr.b +STACKENV=prod +STACKENV_S=dr + + +# updates to appconfig +if [ -d ${1}appconfig ];then + + # overwrite process.csv + cp ./process.csv ${1}appconfig/ + + # make nontorqprocess.csv + cp ./nontorqprocess.csv ${1}appconfig/ + + # add rdbt settings file + echo '.rdb.subscribeto:`trade;' > ${1}appconfig/settings/rdbt.q + + # add rdbq settings file + echo '.rdb.tickerplanttypes:`segmentedchainedtickerplant; +.rdb.subscribeto:`quote;' > ${1}appconfig/settings/rdbq.q + + # append to sctp file + echo '.sctp.loggingmode:`create;' >> ${1}appconfig/settings/segmentedchainedtickerplant.q + find $1 -type f -name "segmentedchainedtickerplant.q" -exec sed -i -e "s/stp1/stp1_${STACKENV}/g" {} \; +fi + +# define environment variables to add to setenv.sh +envvars=" +export PRIMARYHOSTA=${PRIMARYHOSTA}\n\ +export PRIMARYHOSTB=${PRIMARYHOSTB}\n\ +export SECONDARYHOSTA=${SECONDARYHOSTA}\n\ +export SECONDARYHOSTB=${SECONDARYHOSTB}\n\ +export STACKENV=${STACKENV}\n\ +export STACKENV_S=${STACKENV_S}" + +# add environment variables +find $1 -type f -name "setenv.sh" -exec sed -i '$a\'"$envvars"'' {} \; + +# replace generics in process.csv and nontorqprocess.csv with specifics +find $1 -type f -name "process.csv" -exec sed -i -e "s/STACKENV/${STACKENV}/g" {} \; +find $1 -type f -name "nontorqprocess.csv" -exec sed -i -e "s/STACKENV_S/${STACKENV_S}/g" {} \; +``` + +For the DR the script would look identical but for the variables at the start flipped so that for example PRIMARYHOSTA was dr.a and STACKENV was dr. From 8437fdeeeb3b19a37504261da86932fe24cedf18 Mon Sep 17 00:00:00 2001 From: JedwardsEdwards Date: Tue, 23 Feb 2021 16:46:14 +0000 Subject: [PATCH 2/2] Update ordering and remove some of the config script steps for simplcity --- docs/MultipleHosts.md | 112 ++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 70 deletions(-) diff --git a/docs/MultipleHosts.md b/docs/MultipleHosts.md index 38e28ad9e..f94ec054c 100644 --- a/docs/MultipleHosts.md +++ b/docs/MultipleHosts.md @@ -4,13 +4,10 @@ The purpose of this is to give an example of a multiple host set up. We will have two environments, production and DR. Each environment will be split across two hosts. This will use the TorQ Finance Starter Pack as the base to work from. -## Basic Installation -For this example we will use the TorQ Installation Script. Information on this can be found here [TorQ Installation Script](http://www.aquaq.co.uk/q/torq-installation-script/) - -The majority of the below can be implemented through a environment-specific config script supplied to the installtorqapp.sh script through an edit to the installlatest.sh script as described at the above link. +We will run through the steps to implement this set-up in order below, making use of the TorQ install script where appropriate. ## Host Split -To split the stack across hosts we do so in the process.csv file. +First off we will want to split the stack across hosts, which we can do by configuring a custom process.csv file. Processes on Host A: - Feed - Tickerplant @@ -39,29 +36,8 @@ ${PRIMARYHOSTB},{KDBBASEPORT}+8,segmentedchainedtickerplant,sctp1_STACKENV,${TOR ${PRIMARYHOSTA},{KDBBASEPORT}+9,feed,feed1_STACKENV,,1,0,,,${KDBAPPCODE}/tick/feed.q,1,,q ``` -## Table Split -We want to split the RDBs so that one has the trade table and the other has the quote table. They will run on host A and B respectively. To do this we will update the variable .rdb.subscribe to specify which table each RDB wants to subscribe to. Along with this the RDB on host B will also connect to the chained tickerplant on the same host. To do this we will create two files both of which will be located in $TORQAPPHOME/appconfig/settings. They will be as follows: - -rdbt.q: -``` -.rdb.subscribeto:`trade; -``` - -rdbq.q: -``` -.rdb.tickerplanttypes:`segmentedchainedtickerplant; -.rdb.subscribeto:`quote; -``` - -For the chained tickerplant we want it to create its own log, and to subscribe to the correct tp name. To do so we will update the following file. - -segmentedchainedtickerplant.q: -``` -.sctp.tickerplantname:stp1_prod; -.sctp.loggingmode:`create; -``` ## Gateway Connectivity -We want the gateways to connect to both sides. To do this we will add the other environment's RDBs and HDBs to the nontorqprocess.csv file (the STACKENV_S string will be replaced as with the process.csv): +We want the gateways to connect to both sides, i.e. such that the production gateway connects to the DR databases as well as the production ones, and vice versa. To do this we will configure a custom nontorqprocess.csv file, adding the other environment's RDBs and HDBs (the STACKENV_S string will be replaced as with the process.csv): ``` host,port,proctype,procname {SECONDARYHOSTA},6002,rdbt,rdb1_STACKENV_S @@ -70,25 +46,13 @@ host,port,proctype,procname {SECONDARYHOSTA},6004,hdb,hdb2_STACKENV_S ``` -## Environment Variables -To run this across multiple hosts we need to set the different host names and what environment the host we are on is. Hence the following should be added to the setenv.sh script: -``` -export PRIMARYHOSTA=prod.a -export PRIMARYHOSTB=prod.b -export SECONDARYHOSTA=dr.a -export SECONDARYHOSTB=dr.b -export STACKENV=prod -export STACKENV_S=dr -``` - -Replacing prod.a,prod.b,dr.a and dr.b with the actual hostnames or IPs of the hosts. - -## Environment-Specific Config Script +## TorQ Install Script and Environment Specific Configuration Script +To make use of the process.csv/nontorqprocess.csv files we've defined above we'll also need to set some environment variables on stack startup, which can be done by updating the setenv.sh script. -The below should provide an example of a config script to implement the majority of the above for the production host. It assumes that both a process.csv and nontorqprocess.csv file exist: +To make this update, and make the other string replacements mentioned earlier we can supply a config script to the TorQ install script, which we'll run after this set to deploy TorQ and the TorQ Finace Starter Pack. Details on how to retrieve the TorQ install script, edit the install script once it has been retrieved to make use of this config script, and more information about the install script can be found here: [TorQ Installation Script](http://www.aquaq.co.uk/q/torq-installation-script/) +The below should provide an example of a config script to be run as part of the TorQ install script when deploying to the production servers. The six variables defined being our environment variables to add to the setenv.sh script, defining the different host names and what environment the host we are on is, replacing prod.a,prod.b,dr.a and dr.b with the actual hostnames or IPs of the hosts. ``` -# configurable variables PRIMARYHOSTA=prod.a PRIMARYHOSTB=prod.b SECONDARYHOSTA=dr.a @@ -96,29 +60,10 @@ SECONDARYHOSTB=dr.b STACKENV=prod STACKENV_S=dr +find $1 -type f -name "segmentedchainedtickerplant.q" -exec sed -i -e "s/stp1/stp1_${STACKENV}/g" {} \; +find ./ -type f -name "process.csv" -exec sed -i -e "s/STACKENV/${STACKENV}/g" {} \; +find ./ -type f -name "nontorqprocess.csv" -exec sed -i -e "s/STACKENV_S/${STACKENV_S}/g" {} \; -# updates to appconfig -if [ -d ${1}appconfig ];then - - # overwrite process.csv - cp ./process.csv ${1}appconfig/ - - # make nontorqprocess.csv - cp ./nontorqprocess.csv ${1}appconfig/ - - # add rdbt settings file - echo '.rdb.subscribeto:`trade;' > ${1}appconfig/settings/rdbt.q - - # add rdbq settings file - echo '.rdb.tickerplanttypes:`segmentedchainedtickerplant; -.rdb.subscribeto:`quote;' > ${1}appconfig/settings/rdbq.q - - # append to sctp file - echo '.sctp.loggingmode:`create;' >> ${1}appconfig/settings/segmentedchainedtickerplant.q - find $1 -type f -name "segmentedchainedtickerplant.q" -exec sed -i -e "s/stp1/stp1_${STACKENV}/g" {} \; -fi - -# define environment variables to add to setenv.sh envvars=" export PRIMARYHOSTA=${PRIMARYHOSTA}\n\ export PRIMARYHOSTB=${PRIMARYHOSTB}\n\ @@ -127,13 +72,40 @@ export SECONDARYHOSTB=${SECONDARYHOSTB}\n\ export STACKENV=${STACKENV}\n\ export STACKENV_S=${STACKENV_S}" -# add environment variables find $1 -type f -name "setenv.sh" -exec sed -i '$a\'"$envvars"'' {} \; +``` -# replace generics in process.csv and nontorqprocess.csv with specifics -find $1 -type f -name "process.csv" -exec sed -i -e "s/STACKENV/${STACKENV}/g" {} \; -find $1 -type f -name "nontorqprocess.csv" -exec sed -i -e "s/STACKENV_S/${STACKENV_S}/g" {} \; +Note that when implementing this on the DR servers that the primary and secondary values should be swapped, as below: ``` +PRIMARYHOSTA=dr.a +PRIMARYHOSTB=dr.b +SECONDARYHOSTA=prod.a +SECONDARYHOSTB=prod.b +STACKENV=dr +STACKENV_S=prod +``` + +Once the config script has been set up and the TorQ install script has been set up to make use of it, you should have the installlatest.sh script, the config script, the process.csv file, and the nontorqprocess.csv file in your install directory and the install script can be run -For the DR the script would look identical but for the variables at the start flipped so that for example PRIMARYHOSTA was dr.a and STACKENV was dr. +Once that has completed the process.csv/nontorqprocess.csv files can be moved to the $TORQAPPHOME/appconfig folder. +## Table Split +We want to split the RDBs so that one has the trade table and the other has the quote table. They will run on host A and B respectively. To do this we will update the variable .rdb.subscribe to specify which table each RDB wants to subscribe to. Along with this the RDB on host B will also connect to the chained tickerplant on the same host. To do this we will create two files both of which will be located in $TORQAPPHOME/appconfig/settings. They will be as follows: + +rdbt.q: +``` +.rdb.subscribeto:`trade; +``` + +rdbq.q: +``` +.rdb.tickerplanttypes:`segmentedchainedtickerplant; +.rdb.subscribeto:`quote; +``` + +For the chained tickerplant we want it to create its own log. To do so we will update the following file. + +segmentedchainedtickerplant.q: +``` +.sctp.loggingmode:`create; +```