From 1feb3a2793edda32ed247e3de816390ef725e9ed Mon Sep 17 00:00:00 2001 From: ConorSwain Date: Wed, 29 Jan 2025 08:29:54 +0000 Subject: [PATCH 1/4] Fix partbyenum to enumerate against NONE symbol --- code/processes/idb.q | 2 +- code/processes/wdb.q | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/code/processes/idb.q b/code/processes/idb.q index fd67ccc8d..098413b9d 100644 --- a/code/processes/idb.q +++ b/code/processes/idb.q @@ -107,5 +107,5 @@ reload:.idb.intradayreload; /- helper function to support queries against the sym column maptoint:{[symbol] - sym?symbol + sym?`NONE^symbol }; diff --git a/code/processes/wdb.q b/code/processes/wdb.q index 960ae8f74..92452c23e 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -114,16 +114,20 @@ mergemaxrows:{[tabname] mergenumrows^mergenumtab[tabname]} tablelist:{[] sortedlist:exec tablename from `bytes xdesc tabsizes; (sortedlist union tables[`.]) except ignorelist} +/- function to check for null syms +checknullsym:{[s] -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} s,(::)} + /- function to upsert to specified directory upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] /- enumerate current extra partition against the hdb sym file - if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)?first expt;]; + /- if extra partition is null, send to a partition enumerated against `NONE symbol + if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`NONE^ checknullsym[expt]]]; /- create directory location for selected partiton /- replace non-alphanumeric characters in symbols with _ /- convert to symbols and replace any null values with `NONE directory:$[writedownmode~`partbyenum; ` sv .Q.par[dir;pt;`$string i],tablename,`; - ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `NONE^ -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} expt,(::)),`]; + ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `NONE^ checknullsym[expt]),`]; .lg.o[`save;"saving ",(string tablename)," data to partition ",string directory]; /- selecting rows of table with matching partition r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt);{(x;y;(),z)}[in;;]'[expttype;expt]];0b;()]; From e84185c3d35582952f77dab8c9e43cd94a540edf Mon Sep 17 00:00:00 2001 From: ConorSwainDI Date: Thu, 30 Jan 2025 16:10:46 +0000 Subject: [PATCH 2/4] Added tests; Modified NONE sym to TORQNULLSYMBOL --- code/processes/idb.q | 2 +- code/processes/wdb.q | 4 +-- tests/wdb/nullpartbyenum/config/sort.csv | 4 +++ tests/wdb/nullpartbyenum/database.q | 2 ++ tests/wdb/nullpartbyenum/nullpartbyenum.csv | 40 +++++++++++++++++++++ tests/wdb/nullpartbyenum/process.csv | 7 ++++ tests/wdb/nullpartbyenum/run.sh | 23 ++++++++++++ tests/wdb/nullpartbyenum/settings.q | 21 +++++++++++ 8 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 tests/wdb/nullpartbyenum/config/sort.csv create mode 100644 tests/wdb/nullpartbyenum/database.q create mode 100644 tests/wdb/nullpartbyenum/nullpartbyenum.csv create mode 100644 tests/wdb/nullpartbyenum/process.csv create mode 100755 tests/wdb/nullpartbyenum/run.sh create mode 100644 tests/wdb/nullpartbyenum/settings.q diff --git a/code/processes/idb.q b/code/processes/idb.q index 098413b9d..d07b6f275 100644 --- a/code/processes/idb.q +++ b/code/processes/idb.q @@ -107,5 +107,5 @@ reload:.idb.intradayreload; /- helper function to support queries against the sym column maptoint:{[symbol] - sym?`NONE^symbol + sym?`TORQNULLSYMBOL^symbol }; diff --git a/code/processes/wdb.q b/code/processes/wdb.q index 92452c23e..3f03f9d9e 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -121,13 +121,13 @@ checknullsym:{[s] -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} s upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] /- enumerate current extra partition against the hdb sym file /- if extra partition is null, send to a partition enumerated against `NONE symbol - if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`NONE^ checknullsym[expt]]]; + if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`TORQNULLSYMBOL^ checknullsym[expt]]]; /- create directory location for selected partiton /- replace non-alphanumeric characters in symbols with _ /- convert to symbols and replace any null values with `NONE directory:$[writedownmode~`partbyenum; ` sv .Q.par[dir;pt;`$string i],tablename,`; - ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `NONE^ checknullsym[expt]),`]; + ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ checknullsym[expt]),`]; .lg.o[`save;"saving ",(string tablename)," data to partition ",string directory]; /- selecting rows of table with matching partition r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt);{(x;y;(),z)}[in;;]'[expttype;expt]];0b;()]; diff --git a/tests/wdb/nullpartbyenum/config/sort.csv b/tests/wdb/nullpartbyenum/config/sort.csv new file mode 100644 index 000000000..1113f6251 --- /dev/null +++ b/tests/wdb/nullpartbyenum/config/sort.csv @@ -0,0 +1,4 @@ +tabname,att,column,sort +quote,p,sym,1 +trade,p,sym,1 +default,p,sym,1 \ No newline at end of file diff --git a/tests/wdb/nullpartbyenum/database.q b/tests/wdb/nullpartbyenum/database.q new file mode 100644 index 000000000..44d463e29 --- /dev/null +++ b/tests/wdb/nullpartbyenum/database.q @@ -0,0 +1,2 @@ +quote:([]time:`timestamp$(); sym:`g#`symbol$(); bid:`float$(); ask:`float$(); bsize:`long$(); asize:`long$(); mode:`char$(); ex:`char$(); src:`symbol$()) +trade:([]time:`timestamp$(); sym:`g#`symbol$(); price:`float$(); size:`int$(); stop:`boolean$(); cond:`char$(); ex:`char$();side:`symbol$()) diff --git a/tests/wdb/nullpartbyenum/nullpartbyenum.csv b/tests/wdb/nullpartbyenum/nullpartbyenum.csv new file mode 100644 index 000000000..9d96e1ac5 --- /dev/null +++ b/tests/wdb/nullpartbyenum/nullpartbyenum.csv @@ -0,0 +1,40 @@ +action,ms,bytes,lang,code,repeat,minver,comment +beforeany,0,0,q,system "sleep 5",1,,"Wait for processes to start" +before,0,0,q,.servers.startup[],1,,"Start connection management" +before,0,0,q,system "sleep 2",1,,"Wait for connections" +before,0,0,q,stpHandle:gethandle[`stp1],1,,"Open handle to STP" +before,0,0,q,wdbEnumHandle:gethandle[`wdbenum],1,,"Open handle to WDB enum" +before,0,0,q,idbHandle:gethandle[`idbenum],1,,"Open handle to IDB" +before,0,0,q,hdbEnumHandle:gethandle[`hdbenum],1,,"Open handle to partbyenum HDB" +before,0,0,q,"wdbEnumSym:` sv (wdbEnumHandle(`.wdb.hdbsettings;`hdbdir)),`sym",1,,"Get location of partbyenum hdb sym file" +before,0,0,q,wdbEnumHandle(set;`.wdb.numtab;`quote`trade!12 15),1,,"Set low treshold numbers for merging on partbyenum WDB" + +before,0,0,q,stpHandle(`.stplg.init;testlogdb),1,,"Create test stplog test directory" +before,0,0,q,logdir:1_string stpHandle(`.stplg.dldir),1,,"Get location of log directory" +before,0,0,q,.proc.sys "sleep 3",1,,"Wait for connections to get set up" +before,0,0,q,.os.md temphdbpartbyenumdir,1,,"Creating HDB directory for partbyenum DB" + +run,0,0,q,"stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))",1,,"Send trade and quote updates to STP" +run,0,0,q,.proc.sys "sleep 2",1,,"Wait for updates to publish" +true,0,0,q,count[first testtrade]~wdbEnumHandle "count trade",1,,"Check trade update was correctly published to partbyenum WDB" +true,0,0,q,count[first testquote]~wdbEnumHandle "count quote",1,,"Check quote update was correctly published to partbyenum WDB" + +run,0,0,q,"stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))",1,,"Send trade and quote updates to STP to force a writedown" +run,0,0,q,.proc.sys "sleep 3",1,,"Wait for updates to publish and writedown to happen" +true,0,0,q,"all (distinct `TORQNULLSYMBOL^(testtrade[0]),(testquote[0])) in get wdbEnumSym",1,,"Check if all symbols are enumerated against partbyenum hdb sym, fill null sym in test data with TORQNULLSYMBOL to confirm this has been enumerated against" + +true,0,0,q,all {any x like/: y}[;folder_patterns] exec ptdir from wdbEnumHandle `.merge.partsizes,1,,"Check if all partitions are placed in the correct enumerated tables in the partbyenum WDB dir" + +true,0,0,q,(2*count[where null [first[testquote]]]) ~ idbHandle"count select from quote where int=maptoint[`]",1,,"Check maptoint function is querying null sym correctly on quote" +true,0,0,q,(2*count[where null [first[testquote]]]) ~ idbHandle"count select from trade where int=maptoint[`]",1,,"Check maptoint function is querying null sym correctly on trade" + +run,0,0,q,wdbEnumHandle(`.u.end;`.wdb.currentpartition),1,,"Trigger EOD on partbyenum WDB" +run,0,0,q,.proc.sys "sleep 5",1,,"Wait for merge to HDB to happen" +true,0,0,q,(2*count[first testtrade])~hdbEnumHandle "count select from trade",1,,"Check trade in hdb" +true,0,0,q,(2*count[first testquote])~hdbEnumHandle "count select from quote",1,,"Check quote in hdb" + +after,0,0,q,hclose each (wdbEnumHandle;stpHandle;idbHandle;hdbEnumHandle);,1,,"Close handles" +after,0,0,q,kill9proc each ("stp1";"wdbenum";"hdbenum";"idbenum";"sort1"),1,,"Stop all procs" +after,0,0,q,.os.deldir logdir,1,,"Delete test segmented tickerplant logs" +after,0,0,q,.os.deldir 1_string temphdbpartbyenumdir,1,,"Delete temp WDB" +after,0,0,q,.os.deldir 1_string wdbpartbyenumdir,1,,"Delete temp WDB" diff --git a/tests/wdb/nullpartbyenum/process.csv b/tests/wdb/nullpartbyenum/process.csv new file mode 100644 index 000000000..8094ae9ee --- /dev/null +++ b/tests/wdb/nullpartbyenum/process.csv @@ -0,0 +1,7 @@ +host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd +localhost,{KDBBASEPORT}+100,discovery,discovery1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/discovery.q,1,,q +localhost,{KDBBASEPORT}+101,wdb,wdbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q -.wdb.writedownmode partbyenum -.wdb.mode saveandsort -.wdb.sortcsv ${KDBTESTS}/wdb/nullpartbyenum/config/sort.csv -.wdb.savedir ${KDBTESTS}/wdb/nullpartbyenum/tempwdbpartbyenum/ -.wdb.hdbdir ${KDBTESTS}/wdb/nullpartbyenum/temphdbpartbyenum/ -.wdb.settimer 0D00:00:02,1,,q +localhost,{KDBBASEPORT}+103,hdb,hdbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBTESTS}/wdb/nullpartbyenum/temphdbpartbyenum/,1,,q +localhost,{KDBBASEPORT}+105,segmentedtickerplant,stp1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQHOME}/database.q,q +localhost,{KDBBASEPORT}+106,idb,idbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/idb.q,1,,q +localhost,{KDBBASEPORT}+108,sort,sort1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q ${KDBTESTS}/merge/wdb/config/wdbsettings.q,1,-s -2, diff --git a/tests/wdb/nullpartbyenum/run.sh b/tests/wdb/nullpartbyenum/run.sh new file mode 100755 index 000000000..98395621a --- /dev/null +++ b/tests/wdb/nullpartbyenum/run.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Handle command-line arguments +source $KDBTESTS/flagparse.sh + +# Path to test directory +testpath=${KDBTESTS}/wdb/nullpartbyenum + +# Start procs +${TORQHOME}/torq.sh start discovery1 stp1 wdbenum hdbenum idbenum sort1 -csv ${testpath}/process.csv + +# Start test proc +/usr/bin/rlwrap q ${TORQHOME}/torq.q \ + -proctype test -procname test1 \ + -test ${testpath} \ + -load ${KDBTESTS}/helperfunctions.q ${testpath}/settings.q \ + -testresults ${KDBTESTS}/stp/results/ \ + -runtime $run \ + -procfile ${testpath}/process.csv \ + $debug $stop $write $quiet + +# Shut down procs +${TORQHOME}/torq.sh stop discovery1 -csv ${testpath}/process.csv diff --git a/tests/wdb/nullpartbyenum/settings.q b/tests/wdb/nullpartbyenum/settings.q new file mode 100644 index 000000000..6453c25df --- /dev/null +++ b/tests/wdb/nullpartbyenum/settings.q @@ -0,0 +1,21 @@ +// IPC connection parameters +.servers.CONNECTIONS:`wdb`segmentedtickerplant`tickerplant`hdb`idb`sort; +.servers.USERPASS:`admin:admin; + +// Paths to process CSV and test STP log directory +processcsv:getenv[`KDBTESTS],"/wdb/nullpartbyenum/process.csv"; +wdbpartbyenumdir:hsym `$getenv[`KDBTESTS],"/wdb/nullpartbyenum/tempwdbpartbyenum/"; +temphdbpartbyenumdir:hsym `$getenv[`KDBTESTS],"/wdb/nullpartbyenum/temphdbpartbyenum/"; +testlogdb:"testlog"; + +// Test updates +testtrade:((3#`GOOG),``,5?`4;10?100.0;10?100i;10#0b;10?.Q.A;10?.Q.A;10#`buy); +testquote:((8?`4),``;(5?50.0),50+5?50.0;10?100.0;10?100i;10?100i;10?.Q.A;10?.Q.A;10#`3); + +// expected WDB folder structure +folder_patterns:{"*",x,"*"}each 1_/:string ` sv/: cross[hsym each `$string til count distinct testtrade[0],testquote[0];`trade`quote]; + + +// Function projections (using functions from helperfunctions.q) +startproc:startorstopproc["start";;processcsv]; +stopproc:startorstopproc["stop";;processcsv]; From 6a46d0e1356d6df405f966136e65738671321881 Mon Sep 17 00:00:00 2001 From: ConorSwainDI Date: Fri, 31 Jan 2025 14:11:22 +0000 Subject: [PATCH 3/4] Rename sym list check function --- code/processes/wdb.q | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/processes/wdb.q b/code/processes/wdb.q index 3f03f9d9e..f5ef6ae83 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -114,20 +114,20 @@ mergemaxrows:{[tabname] mergenumrows^mergenumtab[tabname]} tablelist:{[] sortedlist:exec tablename from `bytes xdesc tabsizes; (sortedlist union tables[`.]) except ignorelist} -/- function to check for null syms -checknullsym:{[s] -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} s,(::)} +/- function that ensures a list of syms is returned no matter what is passed to it +ensuresymlist:{[s] -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} s,(::)} /- function to upsert to specified directory upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] /- enumerate current extra partition against the hdb sym file /- if extra partition is null, send to a partition enumerated against `NONE symbol - if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`TORQNULLSYMBOL^ checknullsym[expt]]]; + if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`TORQNULLSYMBOL^ ensuresymlist[expt]]]; /- create directory location for selected partiton /- replace non-alphanumeric characters in symbols with _ /- convert to symbols and replace any null values with `NONE directory:$[writedownmode~`partbyenum; ` sv .Q.par[dir;pt;`$string i],tablename,`; - ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ checknullsym[expt]),`]; + ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ ensuresymlist[expt]),`]; .lg.o[`save;"saving ",(string tablename)," data to partition ",string directory]; /- selecting rows of table with matching partition r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt);{(x;y;(),z)}[in;;]'[expttype;expt]];0b;()]; From c279ac1b1e2c580d15aec6f699687860d0d76a61 Mon Sep 17 00:00:00 2001 From: ConorSwainDI Date: Fri, 31 Jan 2025 14:15:20 +0000 Subject: [PATCH 4/4] Change comment to reflect new null placeholder sym name --- code/processes/wdb.q | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/processes/wdb.q b/code/processes/wdb.q index f5ef6ae83..b8f4b531c 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -120,11 +120,11 @@ ensuresymlist:{[s] -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} /- function to upsert to specified directory upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] /- enumerate current extra partition against the hdb sym file - /- if extra partition is null, send to a partition enumerated against `NONE symbol + /- if extra partition is null, send to a partition enumerated against `TORQNULLSYMBOL symbol if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`TORQNULLSYMBOL^ ensuresymlist[expt]]]; /- create directory location for selected partiton /- replace non-alphanumeric characters in symbols with _ - /- convert to symbols and replace any null values with `NONE + /- convert to symbols and replace any null values with `TORQNULLSYMBOL directory:$[writedownmode~`partbyenum; ` sv .Q.par[dir;pt;`$string i],tablename,`; ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ ensuresymlist[expt]),`];