Fix partbyenum to enumerate against placeholding symbol

code/processes/idb.q

@@ -107,5 +107,5 @@ reload:.idb.intradayreload;
 
 /- helper function to support queries against the sym column
 maptoint:{[symbol]
-    sym?symbol
+    sym?`TORQNULLSYMBOL^symbol
  };

code/processes/wdb.q

@@ -114,16 +114,20 @@ mergemaxrows:{[tabname] mergenumrows^mergenumtab[tabname]}
 tablelist:{[] sortedlist:exec tablename from `bytes xdesc tabsizes;
     (sortedlist union tables[`.]) except ignorelist}
 
+/- function that ensures a list of syms is returned no matter what is passed to it
+ensuresymlist:{[s] -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} s,(::)}
+
 /- function to upsert to specified directory
 upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode]
     /- enumerate current extra partition against the hdb sym file
-    if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)?first expt;];
+    /- if extra partition is null, send to a partition enumerated against `TORQNULLSYMBOL symbol
+    if[writedownmode~`partbyenum;i:`long$(` sv hdbsettings[`hdbdir],`sym)? first[`TORQNULLSYMBOL^ ensuresymlist[expt]]];
     /- create directory location for selected partiton
     /- replace non-alphanumeric characters in symbols with _
-    /- convert to symbols and replace any null values with `NONE
+    /- convert to symbols and replace any null values with `TORQNULLSYMBOL
     directory:$[writedownmode~`partbyenum;
                 ` sv .Q.par[dir;pt;`$string i],tablename,`;
-                ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `NONE^ -1 _ `${@[x; where not ((type each x) in (10 -10h));string]} expt,(::)),`];
+                ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ ensuresymlist[expt]),`];
     .lg.o[`save;"saving ",(string tablename)," data to partition ",string directory];
     /- selecting rows of table with matching partition
     r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt);{(x;y;(),z)}[in;;]'[expttype;expt]];0b;()];

tests/wdb/nullpartbyenum/config/sort.csv

@@ -0,0 +1,4 @@
+tabname,att,column,sort
+quote,p,sym,1
+trade,p,sym,1
+default,p,sym,1

tests/wdb/nullpartbyenum/database.q

@@ -0,0 +1,2 @@
+quote:([]time:`timestamp$(); sym:`g#`symbol$(); bid:`float$(); ask:`float$(); bsize:`long$(); asize:`long$(); mode:`char$(); ex:`char$(); src:`symbol$())
+trade:([]time:`timestamp$(); sym:`g#`symbol$(); price:`float$(); size:`int$(); stop:`boolean$(); cond:`char$(); ex:`char$();side:`symbol$())

tests/wdb/nullpartbyenum/nullpartbyenum.csv

@@ -0,0 +1,40 @@
+action,ms,bytes,lang,code,repeat,minver,comment
+beforeany,0,0,q,system "sleep 5",1,,"Wait for processes to start"
+before,0,0,q,.servers.startup[],1,,"Start connection management"
+before,0,0,q,system "sleep 2",1,,"Wait for connections"
+before,0,0,q,stpHandle:gethandle[`stp1],1,,"Open handle to STP"
+before,0,0,q,wdbEnumHandle:gethandle[`wdbenum],1,,"Open handle to WDB enum"
+before,0,0,q,idbHandle:gethandle[`idbenum],1,,"Open handle to IDB"
+before,0,0,q,hdbEnumHandle:gethandle[`hdbenum],1,,"Open handle to partbyenum HDB"
+before,0,0,q,"wdbEnumSym:` sv (wdbEnumHandle(`.wdb.hdbsettings;`hdbdir)),`sym",1,,"Get location of partbyenum hdb sym file"
+before,0,0,q,wdbEnumHandle(set;`.wdb.numtab;`quote`trade!12 15),1,,"Set low treshold numbers for merging on partbyenum WDB"
+
+before,0,0,q,stpHandle(`.stplg.init;testlogdb),1,,"Create test stplog test directory"
+before,0,0,q,logdir:1_string stpHandle(`.stplg.dldir),1,,"Get location of log directory"
+before,0,0,q,.proc.sys "sleep 3",1,,"Wait for connections to get set up"
+before,0,0,q,.os.md temphdbpartbyenumdir,1,,"Creating HDB directory for partbyenum DB"
+
+run,0,0,q,"stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))",1,,"Send trade and quote updates to STP"
+run,0,0,q,.proc.sys "sleep 2",1,,"Wait for updates to publish"
+true,0,0,q,count[first testtrade]~wdbEnumHandle "count trade",1,,"Check trade update was correctly published to partbyenum WDB"
+true,0,0,q,count[first testquote]~wdbEnumHandle "count quote",1,,"Check quote update was correctly published to partbyenum WDB"
+
+run,0,0,q,"stpHandle @/: `.u.upd ,/: ((`trade;testtrade);(`quote;testquote))",1,,"Send trade and quote updates to STP to force a writedown"
+run,0,0,q,.proc.sys "sleep 3",1,,"Wait for updates to publish and writedown to happen"
+true,0,0,q,"all (distinct `TORQNULLSYMBOL^(testtrade[0]),(testquote[0])) in get wdbEnumSym",1,,"Check if all symbols are enumerated against partbyenum hdb sym, fill null sym in test data with TORQNULLSYMBOL to confirm this has been enumerated against"
+
+true,0,0,q,all {any x like/: y}[;folder_patterns] exec ptdir from wdbEnumHandle `.merge.partsizes,1,,"Check if all partitions are placed in the correct enumerated tables in the partbyenum WDB dir"
+
+true,0,0,q,(2*count[where null [first[testquote]]]) ~ idbHandle"count select from quote where int=maptoint[`]",1,,"Check maptoint function is querying null sym correctly on quote"
+true,0,0,q,(2*count[where null [first[testquote]]]) ~ idbHandle"count select from trade where int=maptoint[`]",1,,"Check maptoint function is querying null sym correctly on trade"
+
+run,0,0,q,wdbEnumHandle(`.u.end;`.wdb.currentpartition),1,,"Trigger EOD on partbyenum WDB"
+run,0,0,q,.proc.sys "sleep 5",1,,"Wait for merge to HDB to happen"
+true,0,0,q,(2*count[first testtrade])~hdbEnumHandle "count select from trade",1,,"Check trade in hdb"
+true,0,0,q,(2*count[first testquote])~hdbEnumHandle "count select from quote",1,,"Check quote in hdb"
+
+after,0,0,q,hclose each (wdbEnumHandle;stpHandle;idbHandle;hdbEnumHandle);,1,,"Close handles"
+after,0,0,q,kill9proc each ("stp1";"wdbenum";"hdbenum";"idbenum";"sort1"),1,,"Stop all procs"
+after,0,0,q,.os.deldir logdir,1,,"Delete test segmented tickerplant logs"
+after,0,0,q,.os.deldir 1_string temphdbpartbyenumdir,1,,"Delete temp WDB"
+after,0,0,q,.os.deldir 1_string wdbpartbyenumdir,1,,"Delete temp WDB"

tests/wdb/nullpartbyenum/process.csv

@@ -0,0 +1,7 @@
+host,port,proctype,procname,U,localtime,g,T,w,load,startwithall,extras,qcmd
+localhost,{KDBBASEPORT}+100,discovery,discovery1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/discovery.q,1,,q
+localhost,{KDBBASEPORT}+101,wdb,wdbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q -.wdb.writedownmode partbyenum  -.wdb.mode saveandsort -.wdb.sortcsv ${KDBTESTS}/wdb/nullpartbyenum/config/sort.csv -.wdb.savedir ${KDBTESTS}/wdb/nullpartbyenum/tempwdbpartbyenum/ -.wdb.hdbdir ${KDBTESTS}/wdb/nullpartbyenum/temphdbpartbyenum/ -.wdb.settimer 0D00:00:02,1,,q
+localhost,{KDBBASEPORT}+103,hdb,hdbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBTESTS}/wdb/nullpartbyenum/temphdbpartbyenum/,1,,q
+localhost,{KDBBASEPORT}+105,segmentedtickerplant,stp1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,0,,,${KDBCODE}/processes/segmentedtickerplant.q,1,-schemafile ${TORQHOME}/database.q,q
+localhost,{KDBBASEPORT}+106,idb,idbenum,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/idb.q,1,,q
+localhost,{KDBBASEPORT}+108,sort,sort1,${TORQHOME}/appconfig/passwords/accesslist.txt,1,1,,,${KDBCODE}/processes/wdb.q ${KDBTESTS}/merge/wdb/config/wdbsettings.q,1,-s -2,

tests/wdb/nullpartbyenum/run.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Handle command-line arguments
+source $KDBTESTS/flagparse.sh
+
+# Path to test directory
+testpath=${KDBTESTS}/wdb/nullpartbyenum
+
+# Start procs
+${TORQHOME}/torq.sh start discovery1 stp1 wdbenum hdbenum idbenum sort1 -csv ${testpath}/process.csv
+
+# Start test proc
+/usr/bin/rlwrap q ${TORQHOME}/torq.q \
+  -proctype test -procname test1 \
+  -test ${testpath} \
+  -load ${KDBTESTS}/helperfunctions.q ${testpath}/settings.q \
+  -testresults ${KDBTESTS}/stp/results/ \
+  -runtime $run \
+  -procfile ${testpath}/process.csv \
+  $debug $stop $write $quiet
+
+# Shut down procs
+${TORQHOME}/torq.sh stop discovery1 -csv ${testpath}/process.csv

tests/wdb/nullpartbyenum/settings.q

@@ -0,0 +1,21 @@
+// IPC connection parameters
+.servers.CONNECTIONS:`wdb`segmentedtickerplant`tickerplant`hdb`idb`sort;
+.servers.USERPASS:`admin:admin;
+
+// Paths to process CSV and test STP log directory
+processcsv:getenv[`KDBTESTS],"/wdb/nullpartbyenum/process.csv";
+wdbpartbyenumdir:hsym `$getenv[`KDBTESTS],"/wdb/nullpartbyenum/tempwdbpartbyenum/";
+temphdbpartbyenumdir:hsym `$getenv[`KDBTESTS],"/wdb/nullpartbyenum/temphdbpartbyenum/";
+testlogdb:"testlog";
+
+// Test updates
+testtrade:((3#`GOOG),``,5?`4;10?100.0;10?100i;10#0b;10?.Q.A;10?.Q.A;10#`buy);
+testquote:((8?`4),``;(5?50.0),50+5?50.0;10?100.0;10?100i;10?100i;10?.Q.A;10?.Q.A;10#`3);
+
+// expected WDB folder structure
+folder_patterns:{"*",x,"*"}each 1_/:string ` sv/:  cross[hsym each `$string til count distinct testtrade[0],testquote[0];`trade`quote];
+
+
+// Function projections (using functions from helperfunctions.q)
+startproc:startorstopproc["start";;processcsv];
+stopproc:startorstopproc["stop";;processcsv];