diff --git a/code/processes/idb.q b/code/processes/idb.q index fd67ccc8d..d07b6f275 100644 --- a/code/processes/idb.q +++ b/code/processes/idb.q @@ -107,5 +107,5 @@ reload:.idb.intradayreload; /- helper function to support queries against the sym column maptoint:{[symbol] - sym?symbol + sym?`TORQNULLSYMBOL^symbol }; diff --git a/code/processes/wdb.q b/code/processes/wdb.q index 960ae8f74..b8f4b531c 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -114,16 +114,20 @@ mergemaxrows:{[tabname] mergenumrows^mergenumtab[tabname]} tablelist:{[] sortedlist:exec tablename from `bytes xdesc tabsizes; (sortedlist union tables[`.]) except ignorelist} +/- function that ensures a list of syms is returned no matter what is passed to it +ensuresymlist:{[s] -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} s,(::)} + /- function to upsert to specified directory upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] /- enumerate current extra partition against the hdb sym file - if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)?first expt;]; + /- if extra partition is null, send to a partition enumerated against `TORQNULLSYMBOL symbol + if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`TORQNULLSYMBOL^ ensuresymlist[expt]]]; /- create directory location for selected partiton /- replace non-alphanumeric characters in symbols with _ - /- convert to symbols and replace any null values with `NONE + /- convert to symbols and replace any null values with `TORQNULLSYMBOL directory:$[writedownmode~`partbyenum; ` sv .Q.par[dir;pt;`$string i],tablename,`; - ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `NONE^ -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} expt,(::)),`]; + ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ ensuresymlist[expt]),`]; .lg.o[`save;"saving ",(string tablename)," data to partition ",string directory]; /- selecting rows of table with matching partition r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt);{(x;y;(),z)}[in;;]'[expttype;expt]];0b;()]; diff --git a/tests/wdb/nullpartbyenum/config/sort.csv b/tests/wdb/nullpartbyenum/config/sort.csv new file mode 100644 index 000000000..1113f6251 --- /dev/null +++ b/tests/wdb/nullpartbyenum/config/sort.csv @@ -0,0 +1,4 @@ +tabname,att,column,sort +quote,p,sym,1 +trade,p,sym,1 +default,p,sym,1 \ No newline at end of file diff --git a/tests/wdb/nullpartbyenum/database.q b/tests/wdb/nullpartbyenum/database.q new file mode 100644 index 000000000..44d463e29 --- /dev/null +++ b/tests/wdb/nullpartbyenum/database.q @@ -0,0 +1,2 @@ +quote:([]time:`timestamp$(); sym:`g#`symbol$(); bid:`float$(); ask:`float$(); bsize:`long$(); asize:`long$(); mode:`char$(); ex:`char$(); src:`symbol$()) +trade:([]time:`timestamp$(); sym:`g#`symbol$(); price:`float$(); size:`int$(); stop:`boolean$(); cond:`char$(); ex:`char$();side:`symbol$()) diff --git a/tests/wdb/nullpartbyenum/nullpartbyenum.csv b/tests/wdb/nullpartbyenum/nullpartbyenum.csv new file mode 100644 index 000000000..9d96e1ac5 --- /dev/null +++ b/tests/wdb/nullpartbyenum/nullpartbyenum.csv @@ -0,0 +1,40 @@ +action,ms,bytes,lang,code,repeat,minver,comment +beforeany,0,0,q,system "sleep 5",1,,"Wait for processes to start" +before,0,0,q,.servers.startup[],1,,"Start connection management" +before,0,0,q,system "sleep 2",1,,"Wait for connections" +before,0,0,q,stpHandle:gethandle[`stp1],1,,"Open handle to STP" +before,0,0,q,wdbEnumHandle:gethandle[`wdbenum],1,,"Open handle to WDB enum" +before,0,0,q,idbHandle:gethandle[`idbenum],1,,"Open handle to IDB" +before,0,0,q,hdbEnumHandle:gethandle[`hdbenum],1,,"Open handle to partbyenum HDB" +before,0,0,q,"wdbEnumSym:` sv (wdbEnumHandle(`.wdb.hdbsettings;`hdbdir)),`sym",1,,"Get location of partbyenum hdb sym file" +before,0,0,q,wdbEnumHandle(set;`.wdb.numtab;`quote`trade!12 15),1,,"Set low treshold numbers for merging on partbyenum WDB" + +before,0,0,q,stpHandle(`.stplg.init;testlogdb),1,,"Create test stplog test directory" +before,0,0,q,logdir:1_string stpHandle(`.stplg.dldir),1,,"Get location of log directory" +before,0,0,q,.proc.sys "sleep 3",1,,"Wait for connections to get set up" +before,0,0,q,.os.md temphdbpartbyenumdir,1,,"Creating HDB directory for partbyenum DB" + +run,0,0,q,"stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))",1,,"Send trade and quote updates to STP" +run,0,0,q,.proc.sys "sleep 2",1,,"Wait for updates to publish" +true,0,0,q,count[first testtrade]~wdbEnumHandle "count trade",1,,"Check trade update was correctly published to partbyenum WDB" +true,0,0,q,count[first testquote]~wdbEnumHandle "count quote",1,,"Check quote update was correctly published to partbyenum WDB" + +run,0,0,q,"stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))",1,,"Send trade and quote updates to STP to force a writedown" +run,0,0,q,.proc.sys "sleep 3",1,,"Wait for updates to publish and writedown to happen" +true,0,0,q,"all (distinct `TORQNULLSYMBOL^(testtrade[0]),(testquote[0])) in get wdbEnumSym",1,,"Check if all symbols are enumerated against partbyenum hdb sym, fill null sym in test data with TORQNULLSYMBOL to confirm this has been enumerated against" + +true,0,0,q,all {any x like/: y}[;folder_patterns] exec ptdir from wdbEnumHandle `.merge.partsizes,1,,"Check if all partitions are placed in the correct enumerated tables in the partbyenum WDB dir" + +true,0,0,q,(2*count[where null [first[testquote]]]) ~ idbHandle"count select from quote where int=maptoint[`]",1,,"Check maptoint function is querying null sym correctly on quote" +true,0,0,q,(2*count[where null [first[testquote]]]) ~ idbHandle"count select from trade where int=maptoint[`]",1,,"Check maptoint function is querying null sym correctly on trade" + +run,0,0,q,wdbEnumHandle(`.u.end;`.wdb.currentpartition),1,,"Trigger EOD on partbyenum WDB" +run,0,0,q,.proc.sys "sleep 5",1,,"Wait for merge to HDB to happen" +true,0,0,q,(2*count[first testtrade])~hdbEnumHandle "count select from trade",1,,"Check trade in hdb" +true,0,0,q,(2*count[first testquote])~hdbEnumHandle "count select from quote",1,,"Check quote in hdb" + +after,0,0,q,hclose each (wdbEnumHandle;stpHandle;idbHandle;hdbEnumHandle);,1,,"Close handles" +after,0,0,q,kill9proc each ("stp1";"wdbenum";"hdbenum";"idbenum";"sort1"),1,,"Stop all procs" +after,0,0,q,.os.deldir logdir,1,,"Delete test segmented tickerplant logs" +after,0,0,q,.os.deldir 1_string temphdbpartbyenumdir,1,,"Delete temp WDB" +after,0,0,q,.os.deldir 1_string wdbpartbyenumdir,1,,"Delete temp WDB" diff --git a/tests/wdb/nullpartbyenum/process.csv b/tests/wdb/nullpartbyenum/process.csv new file mode 100644 index 000000000..8094ae9ee --- /dev/null +++ b/tests/wdb/nullpartbyenum/process.csv @@ -0,0 +1,7 @@ +host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd +localhost,{KDBBASEPORT}+100,discovery,discovery1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/discovery.q,1,,q +localhost,{KDBBASEPORT}+101,wdb,wdbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q -.wdb.writedownmode partbyenum -.wdb.mode saveandsort -.wdb.sortcsv ${KDBTESTS}/wdb/nullpartbyenum/config/sort.csv -.wdb.savedir ${KDBTESTS}/wdb/nullpartbyenum/tempwdbpartbyenum/ -.wdb.hdbdir ${KDBTESTS}/wdb/nullpartbyenum/temphdbpartbyenum/ -.wdb.settimer 0D00:00:02,1,,q +localhost,{KDBBASEPORT}+103,hdb,hdbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBTESTS}/wdb/nullpartbyenum/temphdbpartbyenum/,1,,q +localhost,{KDBBASEPORT}+105,segmentedtickerplant,stp1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQHOME}/database.q,q +localhost,{KDBBASEPORT}+106,idb,idbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/idb.q,1,,q +localhost,{KDBBASEPORT}+108,sort,sort1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q ${KDBTESTS}/merge/wdb/config/wdbsettings.q,1,-s -2, diff --git a/tests/wdb/nullpartbyenum/run.sh b/tests/wdb/nullpartbyenum/run.sh new file mode 100755 index 000000000..98395621a --- /dev/null +++ b/tests/wdb/nullpartbyenum/run.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Handle command-line arguments +source $KDBTESTS/flagparse.sh + +# Path to test directory +testpath=${KDBTESTS}/wdb/nullpartbyenum + +# Start procs +${TORQHOME}/torq.sh start discovery1 stp1 wdbenum hdbenum idbenum sort1 -csv ${testpath}/process.csv + +# Start test proc +/usr/bin/rlwrap q ${TORQHOME}/torq.q \ + -proctype test -procname test1 \ + -test ${testpath} \ + -load ${KDBTESTS}/helperfunctions.q ${testpath}/settings.q \ + -testresults ${KDBTESTS}/stp/results/ \ + -runtime $run \ + -procfile ${testpath}/process.csv \ + $debug $stop $write $quiet + +# Shut down procs +${TORQHOME}/torq.sh stop discovery1 -csv ${testpath}/process.csv diff --git a/tests/wdb/nullpartbyenum/settings.q b/tests/wdb/nullpartbyenum/settings.q new file mode 100644 index 000000000..6453c25df --- /dev/null +++ b/tests/wdb/nullpartbyenum/settings.q @@ -0,0 +1,21 @@ +// IPC connection parameters +.servers.CONNECTIONS:`wdb`segmentedtickerplant`tickerplant`hdb`idb`sort; +.servers.USERPASS:`admin:admin; + +// Paths to process CSV and test STP log directory +processcsv:getenv[`KDBTESTS],"/wdb/nullpartbyenum/process.csv"; +wdbpartbyenumdir:hsym `$getenv[`KDBTESTS],"/wdb/nullpartbyenum/tempwdbpartbyenum/"; +temphdbpartbyenumdir:hsym `$getenv[`KDBTESTS],"/wdb/nullpartbyenum/temphdbpartbyenum/"; +testlogdb:"testlog"; + +// Test updates +testtrade:((3#`GOOG),``,5?`4;10?100.0;10?100i;10#0b;10?.Q.A;10?.Q.A;10#`buy); +testquote:((8?`4),``;(5?50.0),50+5?50.0;10?100.0;10?100i;10?100i;10?.Q.A;10?.Q.A;10#`3); + +// expected WDB folder structure +folder_patterns:{"*",x,"*"}each 1_/:string ` sv/: cross[hsym each `$string til count distinct testtrade[0],testquote[0];`trade`quote]; + + +// Function projections (using functions from helperfunctions.q) +startproc:startorstopproc["start";;processcsv]; +stopproc:startorstopproc["stop";;processcsv];