Feature prefilledplugin

DataMonk/v1.5.3/quartz.properties

@@ -0,0 +1,7 @@
+org.quartz.scheduler.instanceName=PriorityExampleScheduler
+# Set thread count to 1 to force Triggers scheduled for the same time to
+# to be ordered by priority.
+org.quartz.threadPool.threadCount=1
+org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool
+
+org.quartz.jobStore.class=org.quartz.simpl.RAMJobStore

DataMonk/v1.5.3/start.cmd

@@ -0,0 +1,5 @@
+setlocal
+set ZOOMAIN=com.linemetrics.monk.DataMonk
+echo off
+java -cp "service.jar" %ZOOMAIN% 2> error.log > service.log
+endlocal

DataMonk/v1.5.3/system.properties

@@ -0,0 +1,44 @@
+# ----------------------------
+# API Configuration
+# ----------------------------
+
+api.endpoint=http://bapi.linemetrics.com:8002
+api.hash=ABCDEFG...
+
+job.1.info.scheduler_mask=0 0 8-17 ? * MON-SAT
+job.1.info.timezone=Europe/Vienna
+job.1.info.batch_size=PT1m
+job.1.info.duration=PT1H
+
+job.1.datastream=123
+job.1.datastream=456
+
+job.1.processor.type=com.linemetrics.monk.processor.plugins.compress.CompressorPlugin
+job.1.processor.compression_mode=SUM
+job.1.processor.compression_size=PT15M
+job.1.processor.compression_batch_item_count=15
+
+job.1.store.1.type=com.linemetrics.monk.store.plugins.csv.StorePlugin
+job.1.store.1.csv_number_locale=de_AT
+job.1.store.1.csv_file_path=./
+job.1.store.1.csv_file_template=${job.start:YYYY-mm-dd}.csv
+job.1.store.1.csv_header_template=Das ist der Header meiner CSV mit einer Meta Info ${meta.clientID}
+job.1.store.1.csv_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};${item.value:0.00};${meta.customer_id}
+job.1.store.1.csv_line_separator=<LF>
+
+job.1.store.2.type=com.linemetrics.monk.store.plugins.csv.PrefilledPlugin
+job.1.store.2.csv_number_locale=de_AT
+job.1.store.2.csv_time_scope=PT1D
+job.1.store.2.csv_time_slice=PT15M
+job.1.store.2.csv_file_path=./
+job.1.store.2.csv_file_template_scope=DAY
+job.1.store.2.csv_file_template=${job.start:YYYY-mm-dd}.csv
+job.1.store.2.csv_header_template=Das ist der Header meiner CSV mit einer Meta Info ${meta.clientID}
+job.1.store.2.csv_empty_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};
+job.1.store.2.csv_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};${item.value:0.00};${meta.customer_id}
+
+activated_jobs=1
+
+meta.job.1.clientID=Customer #1
+meta.datastream.123.customer_id=1234
+meta.datastream.456.customer_id=4567

DataMonk/v1.5.4/quartz.properties

@@ -0,0 +1,7 @@
+org.quartz.scheduler.instanceName=PriorityExampleScheduler
+# Set thread count to 1 to force Triggers scheduled for the same time to
+# to be ordered by priority.
+org.quartz.threadPool.threadCount=1
+org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool
+
+org.quartz.jobStore.class=org.quartz.simpl.RAMJobStore

DataMonk/v1.5.4/start.cmd

@@ -0,0 +1,5 @@
+setlocal
+set ZOOMAIN=com.linemetrics.monk.DataMonk
+echo off
+java -cp "service.jar" %ZOOMAIN% 2> error.log > service.log
+endlocal

DataMonk/v1.5.4/system.properties

@@ -0,0 +1,45 @@
+# ----------------------------
+# API Configuration
+# ----------------------------
+
+api.endpoint=http://bapi.linemetrics.com:8002
+api.hash=ABCDEFG...
+
+job.1.info.scheduler_mask=0 0 8-17 ? * MON-SAT
+job.1.info.timezone=Europe/Vienna
+job.1.info.batch_size=PT1m
+job.1.info.duration=PT1H
+job.1.info.start_delay=30000
+
+job.1.datastream=123
+job.1.datastream=456
+
+job.1.processor.type=com.linemetrics.monk.processor.plugins.compress.CompressorPlugin
+job.1.processor.compression_mode=SUM
+job.1.processor.compression_size=PT15M
+job.1.processor.compression_batch_item_count=15
+
+job.1.store.1.type=com.linemetrics.monk.store.plugins.csv.StorePlugin
+job.1.store.1.csv_number_locale=de_AT
+job.1.store.1.csv_file_path=./
+job.1.store.1.csv_file_template=${job.start:YYYY-mm-dd}.csv
+job.1.store.1.csv_header_template=Das ist der Header meiner CSV mit einer Meta Info ${meta.clientID}
+job.1.store.1.csv_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};${item.value:0.00};${meta.customer_id}
+job.1.store.1.csv_line_separator=<LF>
+
+job.1.store.2.type=com.linemetrics.monk.store.plugins.csv.PrefilledPlugin
+job.1.store.2.csv_number_locale=de_AT
+job.1.store.2.csv_time_scope=PT1D
+job.1.store.2.csv_time_slice=PT15M
+job.1.store.2.csv_file_path=./
+job.1.store.2.csv_file_template_scope=DAY
+job.1.store.2.csv_file_template=${job.start:YYYY-mm-dd}.csv
+job.1.store.2.csv_header_template=Das ist der Header meiner CSV mit einer Meta Info ${meta.clientID}
+job.1.store.2.csv_empty_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};
+job.1.store.2.csv_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};${item.value:0.00};${meta.customer_id}
+
+activated_jobs=1
+
+meta.job.1.clientID=Customer #1
+meta.datastream.123.customer_id=1234
+meta.datastream.456.customer_id=4567

DataMonk/v1.5.5/quartz.properties

@@ -0,0 +1,7 @@
+org.quartz.scheduler.instanceName=PriorityExampleScheduler
+# Set thread count to 1 to force Triggers scheduled for the same time to
+# to be ordered by priority.
+org.quartz.threadPool.threadCount=1
+org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool
+
+org.quartz.jobStore.class=org.quartz.simpl.RAMJobStore

DataMonk/v1.5.5/start.cmd

@@ -0,0 +1,5 @@
+setlocal
+set ZOOMAIN=com.linemetrics.monk.DataMonk
+echo off
+java -cp "service.jar" %ZOOMAIN% 2> error.log > service.log
+endlocal

DataMonk/v1.5.5/system.properties

@@ -0,0 +1,45 @@
+# ----------------------------
+# API Configuration
+# ----------------------------
+
+api.endpoint=http://bapi.linemetrics.com:8002
+api.hash=ABCDEFG...
+
+job.1.info.scheduler_mask=0 0 8-17 ? * MON-SAT
+job.1.info.timezone=Europe/Vienna
+job.1.info.batch_size=PT1m
+job.1.info.duration=PT1H
+job.1.info.start_delay=30000
+
+job.1.datastream=123
+job.1.datastream=456
+
+job.1.processor.type=com.linemetrics.monk.processor.plugins.compress.CompressorPlugin
+job.1.processor.compression_mode=SUM
+job.1.processor.compression_size=PT15M
+job.1.processor.compression_batch_item_count=15
+
+job.1.store.1.type=com.linemetrics.monk.store.plugins.csv.StorePlugin
+job.1.store.1.csv_number_locale=de_AT
+job.1.store.1.csv_file_path=./
+job.1.store.1.csv_file_template=${job.start:YYYY-mm-dd}.csv
+job.1.store.1.csv_header_template=Das ist der Header meiner CSV mit einer Meta Info ${meta.clientID}
+job.1.store.1.csv_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};${item.value:0.00};${meta.customer_id}
+job.1.store.1.csv_line_separator=<LF>
+
+job.1.store.2.type=com.linemetrics.monk.store.plugins.csv.PrefilledPlugin
+job.1.store.2.csv_number_locale=de_AT
+job.1.store.2.csv_time_scope=PT1D
+job.1.store.2.csv_time_slice=PT15M
+job.1.store.2.csv_file_path=./
+job.1.store.2.csv_file_template_scope=DAY
+job.1.store.2.csv_file_template=${job.start:YYYY-mm-dd}.csv
+job.1.store.2.csv_header_template=Das ist der Header meiner CSV mit einer Meta Info ${meta.clientID}
+job.1.store.2.csv_empty_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};
+job.1.store.2.csv_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};${item.value:0.00};${meta.customer_id}
+
+activated_jobs=1
+
+meta.job.1.clientID=Customer #1
+meta.datastream.123.customer_id=1234
+meta.datastream.456.customer_id=4567

README.md

@@ -67,6 +67,7 @@ Die Synchronisierungseinstellungen unterteilen sich in 4 Bereiche:
     job.1.info.timezone=Europe/Vienna
     job.1.info.batch_size=PT1m
     job.1.info.duration=PT1H
+    job.1.info.start_delay=10000
 
 Alle Einstellungen die zu einem Sync Vorgang gehören, müssen die selbe `JOB ID` beinhalten. Die `JOB ID` ist
 ein ganzzahliger numerischer Wert, der selbst definiert werden kann und lediglich in der Konfiguration konsistent sein
@@ -90,6 +91,8 @@ Mögliche Werte: `PT1M`, `PT1H`, `P1D`
 
 `duration` legt fest, was für ein Zeitraum geladen werden soll. Angaben im ISO_8601.
 
+`start_delay` ist optional und legt fest, mit welcher Verzögerung (in ms) ein Job gestartet werden soll. Damit kann sichergestellt werden das die angeforderten Daten bereits verfügbar sind.
+
 Aus den Einstellungen lässt sich interpretieren, dass Montag-Samstag zwischen 8:00 und 17:00 Uhr zu
 jeder vollen Stunde die vergangene Stunde als Minuten Datenpunkte abgeholt wird.
 
@@ -167,6 +170,7 @@ Beispiel Konfiguration für das Prefilled Plugin
     job.1.store.2.csv_time_slice=PT15M
     job.1.store.2.csv_number_locale=de_AT
     job.1.store.2.csv_file_path=exports/
+    job.1.store.2.csv_file_template_scope=DAY
     job.1.store.2.csv_file_template=${job.start:YYYY-mm-dd}.csv
     job.1.store.2.csv_header_template=Das ist der Header meiner CSV
     job.1.store.2.csv_empty_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};
@@ -184,6 +188,9 @@ Das Attribut `csv_empty_line_template` legt das Format fest, wie die leeren Date
 für den gesamten Zeitraum geschrieben werden, aussehen sollen. Diese Zeilen werden später durch die tatsächlichen Werte bzw.
 durch die Konfiguration von `csv_line_template` ersetzt.
 
+Das Attribut `csv_file_template_scope` ist optional und legt den Zeitraum fest für den die Zeilen im CSV File erstellt werden. Mögliche Werte sind HOUR, DAY, MONTH.
+Wird z.B. der Wert `DAY` verwendet, werden unter Beachtung der Time Slices Zeilen von 00:00-23:59 angelegt. 
+
 #### Daten-Weiterleitung an LineMetrics v3
 
 Das Bridge Plugin ermöglicht das Weiterleiten von Daten aus der v2 Umgebung in die v3 Umgebung. Wichtig hier ist, dass zuvor
@@ -355,6 +362,7 @@ Definition laut ISO8601:
     job.1.info.timezone=Europe/Vienna
     job.1.info.batch_size=PT1m
     job.1.info.duration=PT1H
+    job.1.info.start_delay=10000
 
     job.1.datastream=123
     job.1.datastream=456
@@ -375,6 +383,7 @@ Definition laut ISO8601:
     job.1.store.2.csv_time_slice=PT15M
     job.1.store.2.csv_number_locale=de_AT
     job.1.store.2.csv_file_path=exports/
+    job.1.store.2.csv_file_template_scope=DAY
     job.1.store.2.csv_file_template=${job.start:YYYY-mm-dd}.csv
     job.1.store.2.csv_header_template=Das ist der Header meiner CSV
     job.1.store.2.csv_empty_line_template=${item.start:YYYY-mm-dd HH:mm:ss};${item.end:YYYY-mm-dd HH:mm:ss};
@@ -394,6 +403,7 @@ Definition laut ISO8601:
     job.1.info.timezone=Europe/Vienna
     job.1.info.batch_size=PT1m
     job.1.info.duration=PT1H
+    job.1.info.start_delay=10000
 
     job.1.datastream=123
     job.1.datastream=456

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.linemetrics.monk</groupId>
     <artifactId>core</artifactId>
-    <version>1.5.2</version>
+    <version>1.5.5</version>
     <packaging>jar</packaging>
 
     <licenses>

src/main/java/com/linemetrics/monk/config/dao/DirectorJob.java

@@ -66,6 +66,12 @@ public long getDurationInMillis() {
         return new Period(getDuration()).toStandardDuration().getMillis();
     }
 
+    public long getStartDelay(){
+        return this.properties.containsKey("start_delay")
+                ? (Long)this.properties.get("start_delay")
+                : 10000;
+    }
+
     public String getProperties() {
         return properties == null ? "{}" : properties.toJSONString();
     }

src/main/java/com/linemetrics/monk/config/file/SystemConfigStore.java

@@ -75,6 +75,7 @@ public List<DirectorJob> getDirectorJobs() throws ConfigException {
                 put("timezone",       propertyConfiguration.getString("job." + key + ".info.timezone"));
                 put("batch_size",     propertyConfiguration.getString("job." + key + ".info.batch_size"));
                 put("duration",       propertyConfiguration.getString("job." + key + ".info.duration"));
+                put("start_delay",    propertyConfiguration.containsKey("job." + key + ".info.start_delay")?propertyConfiguration.getLong("job." + key + ".info.start_delay"):10000);
             }});
             directorJobs.add(job);
         }

src/main/java/com/linemetrics/monk/director/DirectorInitiator.java

@@ -27,7 +27,8 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution
                 jobExecutionContext.getScheduledFireTime().getTime() - job.getDurationInMillis(),
                 jobExecutionContext.getScheduledFireTime().getTime(),
                 job.getBatchSizeInMillis(),
-                job.getTimeZone()
+                job.getTimeZone(),
+                job.getStartDelay()
             );
 
             DirectorRunner.getInstance().addContext(ctx);

src/main/java/com/linemetrics/monk/director/DirectorRunner.java

@@ -65,10 +65,9 @@ public void run() {
                         Thread.sleep(WAIT_FOR_NEW_CONTEXT);
                         continue;
                     } else {
-                        /** Wait for 10s in order to be sure, that data is processed */
-                        Thread.sleep(10000);
+                        /** Wait for defined seconds (default=10s) in order to be sure, that data is processed */
+                        Thread.sleep(ctx.getStartDelay());
                     }
-
                     String logPrefix = "[" + ctx.getJobId() + "] ";
 
                     this.initializeEnvironment(ctx.getJobId());

src/main/java/com/linemetrics/monk/director/RunnerContext.java

@@ -11,17 +11,20 @@ public class RunnerContext {
     Long timeTo;
     Long batchSize;
     String timezone;
+    Long startDelay;
 
     public RunnerContext(
             int jobId,
             long timeFrom, long timeTo,
-            long batchSize, String timezone) {
+            long batchSize, String timezone,
+            long startDelay) {
 
         this.jobId      = jobId;
         this.timeFrom   = timeFrom;
         this.timeTo     = timeTo;
         this.batchSize  = batchSize;
         this.timezone   = timezone;
+        this.startDelay = startDelay;
     }
 
     public static RunnerContext parse(String json) throws MonkException {
@@ -36,7 +39,8 @@ public static RunnerContext parse(String json) throws MonkException {
             (Long)jsonObj.get("timeFrom"),
             (Long)jsonObj.get("timeTo"),
             (Long)jsonObj.get("batchSize"),
-            (String)jsonObj.get("timezone")
+            (String)jsonObj.get("timezone"),
+            (Long)jsonObj.get("startDelay")
         );
     }
 
@@ -47,6 +51,7 @@ public String toJSONString() {
         obj.put("timeTo", getTimeTo());
         obj.put("batchSize", getBatchSize());
         obj.put("timezone", getTimezone());
+        obj.put("startDelay", getStartDelay());
         return obj.toJSONString();
     }
 
@@ -69,4 +74,8 @@ public long getBatchSize() {
     public String getTimezone() {
         return timezone;
     }
+
+    public Long getStartDelay() {
+        return startDelay;
+    }
 }