From d348f130676b9ac1840edb4750406582194e71fe Mon Sep 17 00:00:00 2001 From: Joseph Muldoon Date: Wed, 19 Nov 2025 16:41:16 +0000 Subject: [PATCH 1/3] Only upsert once for each distinct character group --- code/processes/wdb.q | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/code/processes/wdb.q b/code/processes/wdb.q index 60892d152..550ac4a81 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -135,7 +135,7 @@ mapfctoint:{[val] upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] /- enumerate first extra partition value if[writedownmode~`partbyenum;i:maptoint first expt]; - if[writedownmode~`partbyfirstchar;i:mapfctoint first expt]; + if[writedownmode~`partbyfirstchar;i:mapfctoint first first expt]; /- create directory location for selected partition /- replace non-alphanumeric characters in symbols with _ /- convert to symbols and replace any null values with `TORQNULLSYMBOL @@ -144,7 +144,9 @@ upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ ensuresymlist[expt]),`]; .lg.o[`save;"saving ",(string tablename)," data to partition ",string directory]; /- selecting rows of table with matching partition - r:?[tabdata;$[writedownmode in `partbyenum`partbyfirstchar;enlist(in;first expttype;expt);{(x;y;(),z)}[in;;]'[expttype;expt]];0b;()]; + r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt); + writedownmode~`partbyfirstchar;enlist(in;first expttype;enlist raze expt); + {(x;y;(),z)}[in;;]'[expttype;expt]];0b;()]; /- upsert selected data matched on partition to specific directory .[upsert;(directory;r);{[e] .lg.e[`savetablesbypart;"Failed to save table to disk : ",e];'e}]; .lg.o[`track;"appending details to partsizes"]; @@ -168,6 +170,8 @@ savetablesbypart:{[dir;pt;forcesave;tablename;writedownmode] if[writedownmode~`partbyenum;.merge.checkenumerabletype[tablename;extrapartitiontype]]; /- get list of distinct combinations for partition directories extrapartitions:.merge.getextrapartitions[tablename;extrapartitiontype]; + if[writedownmode~`partbyfirstchar; + extrapartitions:value extrapartitions group {`$first x} each string raze extrapartitions]; /- enumerate data to be upserted enumdata:.Q.en[hdbsettings[`hdbdir];0!.save.manipulate[tablename;`. tablename]]; .lg.o[`save;"enumerated ",(string tablename)," table"]; From af9fb4ad4d077db4ac4b69ce513b8445b5543614 Mon Sep 17 00:00:00 2001 From: Joseph Muldoon Date: Thu, 20 Nov 2025 10:17:40 +0000 Subject: [PATCH 2/3] .Q.fu optimisation. Co-authored-by: Ryan Cheale --- code/processes/wdb.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/processes/wdb.q b/code/processes/wdb.q index 550ac4a81..65f59406f 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -171,7 +171,7 @@ savetablesbypart:{[dir;pt;forcesave;tablename;writedownmode] /- get list of distinct combinations for partition directories extrapartitions:.merge.getextrapartitions[tablename;extrapartitiontype]; if[writedownmode~`partbyfirstchar; - extrapartitions:value extrapartitions group {`$first x} each string raze extrapartitions]; + extrapartitions:value extrapartitions group .Q.fu[{first each string x}; raze extrapartitions]]; /- enumerate data to be upserted enumdata:.Q.en[hdbsettings[`hdbdir];0!.save.manipulate[tablename;`. tablename]]; .lg.o[`save;"enumerated ",(string tablename)," table"]; From f6907224973befcc84a19cd5aeffd1401d51e042 Mon Sep 17 00:00:00 2001 From: Joseph Muldoon Date: Fri, 21 Nov 2025 13:16:19 +0000 Subject: [PATCH 3/3] Reduced changes to upserttopartition --- code/processes/wdb.q | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/code/processes/wdb.q b/code/processes/wdb.q index 65f59406f..f22b6f238 100644 --- a/code/processes/wdb.q +++ b/code/processes/wdb.q @@ -135,7 +135,7 @@ mapfctoint:{[val] upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] /- enumerate first extra partition value if[writedownmode~`partbyenum;i:maptoint first expt]; - if[writedownmode~`partbyfirstchar;i:mapfctoint first first expt]; + if[writedownmode~`partbyfirstchar;i:mapfctoint first expt]; /- create directory location for selected partition /- replace non-alphanumeric characters in symbols with _ /- convert to symbols and replace any null values with `TORQNULLSYMBOL @@ -144,8 +144,7 @@ upserttopartition:{[dir;tablename;tabdata;pt;expttype;expt;writedownmode] ` sv .Q.par[dir;pt;tablename],(`$"_"^.Q.an .Q.an?"_" sv string `TORQNULLSYMBOL^ ensuresymlist[expt]),`]; .lg.o[`save;"saving ",(string tablename)," data to partition ",string directory]; /- selecting rows of table with matching partition - r:?[tabdata;$[writedownmode~`partbyenum;enlist(in;first expttype;expt); - writedownmode~`partbyfirstchar;enlist(in;first expttype;enlist raze expt); + r:?[tabdata;$[writedownmode in `partbyenum`partbyfirstchar;enlist(in;first expttype;enlist expt); {(x;y;(),z)}[in;;]'[expttype;expt]];0b;()]; /- upsert selected data matched on partition to specific directory .[upsert;(directory;r);{[e] .lg.e[`savetablesbypart;"Failed to save table to disk : ",e];'e}]; @@ -171,7 +170,7 @@ savetablesbypart:{[dir;pt;forcesave;tablename;writedownmode] /- get list of distinct combinations for partition directories extrapartitions:.merge.getextrapartitions[tablename;extrapartitiontype]; if[writedownmode~`partbyfirstchar; - extrapartitions:value extrapartitions group .Q.fu[{first each string x}; raze extrapartitions]]; + extrapartitions:value extrapartitions group .Q.fu[{first each string x}; extrapartitions:raze extrapartitions]]; /- enumerate data to be upserted enumdata:.Q.en[hdbsettings[`hdbdir];0!.save.manipulate[tablename;`. tablename]]; .lg.o[`save;"enumerated ",(string tablename)," table"];